You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2025/01/21 19:46:04 UTC
(opennlp) branch main updated: OPENNLP-1695: Add more tests for classes in formats package (#742)
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new c5cc1f00 OPENNLP-1695: Add more tests for classes in formats package (#742)
c5cc1f00 is described below
commit c5cc1f00c251cbe03bdcfa375fd4c80ca3cfdb48
Author: Martin Wiesner <ma...@users.noreply.github.com>
AuthorDate: Tue Jan 21 20:45:59 2025 +0100
OPENNLP-1695: Add more tests for classes in formats package (#742)
- introduces AbstractSampleStreamFactoryTest as common base class
- reduces code duplication in format factory classes
- adds a ton of new test classes for format factories (+ sample data)
- adds two more Evalita samples taken from Appendix of: https://www.evalita.it/wp-content/uploads/2021/11/Guidelines_evalita09_NER.pdf
- adds two OntoNotes samples from the public, official v5.0 release notes (sec 6.4 + 6.8), see: https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf
---
.../opennlp/tools/chunker/ChunkSampleStream.java | 12 +-
.../tools/formats/AbstractSampleStreamFactory.java | 48 ++++++
.../tools/formats/BioNLP2004NameSampleStream.java | 38 +++--
.../formats/BioNLP2004NameSampleStreamFactory.java | 30 ++--
.../tools/formats/ChunkerSampleStreamFactory.java | 28 +---
.../formats/Conll02NameSampleStreamFactory.java | 13 +-
.../formats/Conll03NameSampleStreamFactory.java | 12 +-
.../formats/ConllXPOSSampleStreamFactory.java | 26 ++--
.../formats/ConllXSentenceSampleStreamFactory.java | 16 +-
.../formats/ConllXTokenSampleStreamFactory.java | 14 +-
.../tools/formats/DocumentSampleStreamFactory.java | 27 +---
.../tools/formats/EvalitaNameSampleStream.java | 59 ++++----
.../formats/EvalitaNameSampleStreamFactory.java | 13 +-
.../LanguageDetectorSampleStreamFactory.java | 31 +---
.../formats/LemmatizerSampleStreamFactory.java | 28 +---
.../tools/formats/NameSampleDataStreamFactory.java | 29 +---
.../tools/formats/ParseSampleStreamFactory.java | 26 +---
.../tools/formats/SentenceSampleStreamFactory.java | 28 +---
.../tools/formats/TokenSampleStreamFactory.java | 28 +---
.../tools/formats/TwentyNewsgroupSampleStream.java | 9 ++
.../TwentyNewsgroupSampleStreamFactory.java | 54 ++++---
.../tools/formats/WordTagSampleStreamFactory.java | 27 +---
.../tools/formats/ad/ADChunkSampleStream.java | 4 +-
.../formats/ad/ADChunkSampleStreamFactory.java | 32 +---
.../tools/formats/ad/ADNameSampleStream.java | 4 +-
.../formats/ad/ADNameSampleStreamFactory.java | 32 +---
.../tools/formats/ad/ADPOSSampleStream.java | 2 +-
.../tools/formats/ad/ADPOSSampleStreamFactory.java | 33 +---
.../tools/formats/ad/ADSentenceSampleStream.java | 15 +-
.../formats/ad/ADSentenceSampleStreamFactory.java | 34 +----
.../opennlp/tools/formats/ad/ADSentenceStream.java | 40 +----
.../formats/ad/ADTokenSampleStreamFactory.java | 12 +-
.../formats/ad/PortugueseContractionUtility.java | 9 +-
.../formats/brat/AnnotationConfiguration.java | 114 +++++++++-----
.../tools/formats/brat/BratAnnotationStream.java | 11 +-
.../opennlp/tools/formats/brat/BratDocument.java | 35 ++---
.../tools/formats/brat/BratNameSampleStream.java | 44 +++---
.../formats/brat/BratNameSampleStreamFactory.java | 47 +++---
.../tools/formats/brat/SegmenterObjectStream.java | 1 +
.../conllu/ConlluLemmaSampleStreamFactory.java | 15 +-
.../conllu/ConlluPOSSampleStreamFactory.java | 16 +-
.../tools/formats/conllu/ConlluSentence.java | 4 +-
.../conllu/ConlluSentenceSampleStreamFactory.java | 16 +-
.../conllu/ConlluTokenSampleStreamFactory.java | 17 +--
.../convert/NameToSentenceSampleStreamFactory.java | 9 +-
.../convert/NameToTokenSampleStreamFactory.java | 11 +-
.../convert/POSToSentenceSampleStreamFactory.java | 11 +-
.../convert/POSToTokenSampleStreamFactory.java | 11 +-
.../convert/ParseToPOSSampleStreamFactory.java | 19 ++-
.../ParseToSentenceSampleStreamFactory.java | 18 ++-
.../convert/ParseToTokenSampleStreamFactory.java | 17 ++-
.../frenchtreebank/ConstitDocumentHandler.java | 3 +-
.../ConstitParseSampleStreamFactory.java | 19 +--
.../IrishSentenceBankSentenceStreamFactory.java | 21 +--
.../IrishSentenceBankTokenSampleStreamFactory.java | 21 +--
.../LeipzigLanguageSampleStreamFactory.java | 32 ++--
.../tools/formats/leipzig/SampleShuffleStream.java | 17 ++-
.../tools/formats/leipzig/SampleSkipStream.java | 14 +-
.../letsmt/LetsmtSentenceStreamFactory.java | 22 +--
.../java/opennlp/tools/formats/masc/Masc.java} | 15 +-
.../opennlp/tools/formats/masc/MascDocument.java | 1 -
.../masc/MascNamedEntitySampleStreamFactory.java | 53 ++++---
.../formats/masc/MascPOSSampleStreamFactory.java | 52 ++++---
.../opennlp/tools/formats/masc/MascSentence.java | 4 +-
.../masc/MascSentenceSampleStreamFactory.java | 53 ++++---
.../java/opennlp/tools/formats/masc/MascToken.java | 3 +
.../formats/masc/MascTokenSampleStreamFactory.java | 54 ++++---
.../java/opennlp/tools/formats/masc/MascWord.java | 3 +
.../formats/moses/MosesSentenceSampleStream.java | 12 +-
.../moses/MosesSentenceSampleStreamFactory.java | 35 ++---
.../formats/muc/Muc6NameSampleStreamFactory.java | 28 +++-
.../opennlp/tools/formats/muc/MucElementNames.java | 6 +-
.../tools/formats/muc/MucNameSampleStream.java | 2 +-
.../formats/nkjp/NKJPSegmentationDocument.java | 58 +++----
.../formats/nkjp/NKJPSentenceSampleStream.java | 4 +-
.../nkjp/NKJPSentenceSampleStreamFactory.java | 23 +--
.../tools/formats/nkjp/NKJPTextDocument.java | 6 +-
.../ontonotes/OntoNotesNameSampleStream.java | 111 +++++++-------
.../OntoNotesNameSampleStreamFactory.java | 25 ++-
.../ontonotes/OntoNotesPOSSampleStreamFactory.java | 19 ++-
.../ontonotes/OntoNotesParseSampleStream.java | 2 +-
.../OntoNotesParseSampleStreamFactory.java | 30 +++-
.../tools/namefind/NameSampleDataStream.java | 10 +-
.../opennlp/tools/parser/ParseSampleStream.java | 4 +
.../opennlp/tools/postag/WordTagSampleStream.java | 12 +-
.../formats/AbstractSampleStreamFactoryTest.java | 68 +++++++++
.../tools/formats/AbstractSampleStreamTest.java | 3 +-
.../BioNLP2004NameSampleStreamFactoryTest.java | 113 ++++++++++++++
.../formats/ChunkerSampleStreamFactoryTest.java | 79 ++++++++++
.../Conll02NameSampleStreamFactoryTest.java | 127 ++++++++++++++++
.../Conll03NameSampleStreamFactoryTest.java | 127 ++++++++++++++++
.../formats/ConllXPOSSampleStreamFactoryTest.java | 78 ++++++++++
.../ConllXSentenceSampleStreamFactoryTest.java | 99 ++++++++++++
.../ConllXTokenSampleStreamFactoryTest.java | 98 ++++++++++++
.../EvalitaNameSampleStreamFactoryTest.java | 107 +++++++++++++
.../tools/formats/EvalitaNameSampleStreamTest.java | 88 ++++++++---
.../LanguageDetectorSampleStreamFactoryTest.java | 79 ++++++++++
.../formats/LemmatizerSampleStreamFactoryTest.java | 79 ++++++++++
.../formats/NameSampleDataStreamFactoryTest.java | 85 +++++++++++
.../formats/ParseSampleStreamFactoryTest.java | 80 ++++++++++
.../formats/SentenceSampleStreamFactoryTest.java | 79 ++++++++++
.../formats/TokenSampleStreamFactoryTest.java | 79 ++++++++++
.../TwentyNewsgroupSampleStreamFactoryTest.java | 151 +++++++++++++++++++
.../formats/WordTagSampleStreamFactoryTest.java | 83 ++++++++++
.../formats/ad/ADChunkSampleStreamFactoryTest.java | 102 +++++++++++++
.../formats/ad/ADPOSSampleStreamFactoryTest.java | 105 +++++++++++++
.../tools/formats/ad/ADParagraphStreamTest.java | 2 +-
.../ad/ADSentenceSampleStreamFactoryTest.java | 105 +++++++++++++
.../formats/ad/ADTokenSampleStreamFactoryTest.java | 108 +++++++++++++
.../tools/formats/ad/ADTokenSampleStreamTest.java | 6 +-
.../formats/ad/AbstractADSampleStreamTest.java | 4 +-
.../formats/brat/BratAnnotationStreamTest.java | 39 ++---
.../brat/BratNameSampleStreamFactoryTest.java | 167 +++++++++++++++++++++
.../formats/brat/BratNameSampleStreamTest.java | 17 ++-
.../conllu/ConlluLemmaSampleStreamFactoryTest.java | 113 ++++++++++++++
.../conllu/ConlluPOSSampleStreamFactoryTest.java | 113 ++++++++++++++
.../ConlluSentenceSampleStreamFactoryTest.java | 99 ++++++++++++
.../conllu/ConlluTokenSampleStreamFactoryTest.java | 82 ++++++++++
.../NameToSentenceSampleStreamFactoryTest.java | 101 +++++++++++++
.../NameToTokenSampleStreamFactoryTest.java | 101 +++++++++++++
.../POSToSentenceSampleStreamFactoryTest.java | 101 +++++++++++++
.../convert/POSToTokenSampleStreamFactoryTest.java | 101 +++++++++++++
.../convert/ParseToPOSSampleStreamFactoryTest.java | 81 ++++++++++
.../ParseToSentenceSampleStreamFactoryTest.java | 101 +++++++++++++
.../ParseToTokenSampleStreamFactoryTest.java | 101 +++++++++++++
.../ConstitParseSampleStreamFactoryTest.java | 93 ++++++++++++
...IrishSentenceBankSentenceStreamFactoryTest.java | 83 ++++++++++
...shSentenceBankTokenSampleStreamFactoryTest.java | 83 ++++++++++
.../LeipzigLanguageSampleStreamFactoryTest.java | 110 ++++++++++++++
.../letsmt/LetsmtSentenceStreamFactoryTest.java | 84 +++++++++++
.../MascNamedEntitySampleStreamFactoryTest.java | 102 +++++++++++++
.../masc/MascPOSSampleStreamFactoryTest.java | 102 +++++++++++++
.../masc/MascSentenceSampleStreamFactoryTest.java | 102 +++++++++++++
.../masc/MascTokenSampleStreamFactoryTest.java | 102 +++++++++++++
.../MosesSentenceSampleStreamFactoryTest.java | 82 ++++++++++
.../muc/Muc6NameSampleStreamFactoryTest.java | 114 ++++++++++++++
.../nkjp/NKJPSentenceSampleStreamFactoryTest.java | 102 +++++++++++++
.../OntoNotesNameSampleStreamFactoryTest.java | 96 ++++++++++++
.../OntoNotesPOSSampleStreamFactoryTest.java | 96 ++++++++++++
.../OntoNotesParseSampleStreamFactoryTest.java | 102 +++++++++++++
.../20newsgroup/sci.electronics/52794.sample | 59 ++++++++
.../opennlp/tools/formats/{ => ad}/ad.sample | 0
.../opennlp/tools/formats/bionlp2004-01.sample | 33 ++++
.../opennlp/tools/formats/brat/brat-ann.conf | 7 +
.../opennlp/tools/formats/chunker-01.sample | 16 ++
...lita-ner-it.sample => evalita-ner-it-01.sample} | 0
.../opennlp/tools/formats/evalita-ner-it-02.sample | 29 ++++
.../opennlp/tools/formats/evalita-ner-it-03.sample | 22 +++
.../tools/formats/evalita-ner-it-broken.sample | 2 +
.../tools/formats/evalita-ner-it-incorrect.sample | 3 +
.../opennlp/tools/formats/lang-detect-01.sample | 1 +
.../opennlp/tools/formats/lemma-01.sample | 1 +
.../opennlp/tools/formats/moses/moses-tiny.sample | 3 +
.../opennlp/tools/formats/muc/LDC2003T13.sgm | 73 +++++++++
.../opennlp/tools/formats/name-data-01.sample | 1 +
.../formats/ontonotes/ontonotes-sample-01.name | 9 ++
.../formats/ontonotes/ontonotes-sample-02.parse | 29 ++++
.../opennlp/tools/formats/parse-01.sample | 1 +
.../opennlp/tools/formats/sentences-01.sample | 2 +
.../opennlp/tools/formats/tokens-01.sample | 1 +
.../opennlp/tools/formats/word-tags-01.sample | 1 +
161 files changed, 6132 insertions(+), 1068 deletions(-)
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleStream.java
index 2bc23eed..53f6e979 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleStream.java
@@ -31,8 +31,8 @@ import opennlp.tools.util.ObjectStream;
* Parses the conll 2000 shared task shallow parser training data.
* <p>
* Data format is specified on the conll page:<br>
- * <a href="http://www.cnts.ua.ac.be/conll2000/chunking/">
- * http://www.cnts.ua.ac.be/conll2000/chunking/</a>
+ * <a href="https://www.cnts.ua.ac.be/conll2000/chunking/">
+ * https://www.cnts.ua.ac.be/conll2000/chunking/</a>
*/
public class ChunkSampleStream extends FilterObjectStream<String, ChunkSample> {
@@ -57,7 +57,7 @@ public class ChunkSampleStream extends FilterObjectStream<String, ChunkSample> {
for (String line = samples.read(); line != null && !line.isEmpty(); line = samples.read()) {
String[] parts = line.split(" ");
if (parts.length != 3) {
- logger.error("Skipping corrupt line: {}", line);
+ logger.warn("Skipping corrupt line: {}", line);
}
else {
toks.add(parts[0]);
@@ -66,11 +66,11 @@ public class ChunkSampleStream extends FilterObjectStream<String, ChunkSample> {
}
}
- if (toks.size() > 0) {
+ if (!toks.isEmpty()) {
return new ChunkSample(toks.toArray(new String[0]),
tags.toArray(new String[0]), preds.toArray(new String[0]));
+ } else {
+ return null;
}
-
- return null;
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/AbstractSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/AbstractSampleStreamFactory.java
index 663980ac..da9a36d1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/AbstractSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/AbstractSampleStreamFactory.java
@@ -17,7 +17,15 @@
package opennlp.tools.formats;
+import java.io.IOException;
+
+import opennlp.tools.cmdline.ArgumentParser;
+import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.util.InputStreamFactory;
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.PlainTextByLineStream;
/**
* Base class for sample stream factories.
@@ -40,4 +48,44 @@ public abstract class AbstractSampleStreamFactory<T,P> implements ObjectStreamFa
public Class<P> getParameters() {
return params;
}
+
+ /**
+ * Creates an {@link ObjectStream} for the specified arguments and
+ * the generic type {@code P}.
+ *
+ * @param args A set of command line arguments.
+ * @return The created {@link ObjectStream} instance.
+ */
+ protected <P extends BasicFormatParams> ObjectStream<String> readData(String[] args,
+ Class<P> parametersClass) {
+ P params = validateBasicFormatParameters(args, parametersClass);
+ ObjectStream<String> lineStream = null;
+ try {
+ InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
+ lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
+ } catch (IOException ex) {
+ CmdLineUtil.handleCreateObjectStreamError(ex);
+ }
+ return lineStream;
+ }
+
+ /**
+ * Validates the specified arguments ({@code args}) given the
+ * context the generic type {@code P} which provides at least all
+ * {@link BasicFormatParams}.
+ *
+ * @implNote Additional checks for the basic {@code -data} argument are conducted, that is
+ * wether the file exists or not.
+ *
+ * @param args A set of command line arguments.
+ * @return The parsed (basic format) parameter instance.
+ */
+ protected <P extends BasicFormatParams> P validateBasicFormatParameters(String[] args, Class<P> clazz) {
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ P params = ArgumentParser.parse(args, clazz);
+ CmdLineUtil.checkInputFile("Data", params.getData());
+ return params;
+ }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
index e2ad4ef4..5bdff932 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
@@ -45,25 +45,24 @@ import opennlp.tools.util.StringUtil;
* <p>
* Data can be found on this
* <a href="http://www.geniaproject.org/shared-tasks/bionlp-jnlpba-shared-task-2004">website</a>,
- * or in
- * <a href="https://github.com/spyysalo/jnlpba">this repository</a>.
+ * or in this
+ * <a href="https://github.com/spyysalo/jnlpba">GitHub repository</a>.
* <p>
- * The BioNLP/NLPBA 2004 data were originally published here:
- * <p>
- * <a href="http://www-tsujii.is.s.u-tokyo.ac.jp/GENIA/ERtask/report.html">
- * http://www-tsujii.is.s.u-tokyo.ac.jp/GENIA/ERtask/report.html</a>,
+ * The BioNLP/NLPBA 2004 data were originally published
+ * <a href="http://www-tsujii.is.s.u-tokyo.ac.jp/GENIA/ERtask/report.html">here</a>,
* <p>
* yet this page was gone when last checked in December 2022.
* <p>
- * It looks like this repo contains a copy of the data located on the original page:
- * The BioNLP 2004 seems to be related to http://www.geniaproject.org/shared-tasks/bionlp-jnlpba-shared-task-2004
- * <p>
* <b>Note:</b>
* Do not use this class, internal use only!
*/
@Internal
public class BioNLP2004NameSampleStream implements ObjectStream<NameSample> {
+ private static final String CODEC_TAG_O = "O";
+ private static final String CODEC_TAG_B = "B-";
+ private static final String CODEC_TAG_I = "I-";
+
public static final int GENERATE_DNA_ENTITIES = 0x01;
public static final int GENERATE_PROTEIN_ENTITIES = 0x01 << 1;
public static final int GENERATE_CELLTYPE_ENTITIES = 0x01 << 2;
@@ -96,7 +95,6 @@ public class BioNLP2004NameSampleStream implements ObjectStream<NameSample> {
boolean isClearAdaptiveData = false;
// Empty line indicates end of sentence
-
String line;
while ((line = lineStream.read()) != null && !StringUtil.isEmpty(line.trim())) {
@@ -121,7 +119,7 @@ public class BioNLP2004NameSampleStream implements ObjectStream<NameSample> {
}
}
- if (sentence.size() > 0) {
+ if (!sentence.isEmpty()) {
// convert name tags into spans
List<Span> names = new ArrayList<>();
@@ -133,34 +131,32 @@ public class BioNLP2004NameSampleStream implements ObjectStream<NameSample> {
String tag = tags.get(i);
if (tag.endsWith("DNA") && (types & GENERATE_DNA_ENTITIES) == 0)
- tag = "O";
+ tag = CODEC_TAG_O;
if (tag.endsWith("protein") && (types & GENERATE_PROTEIN_ENTITIES) == 0)
- tag = "O";
+ tag = CODEC_TAG_O;
if (tag.endsWith("cell_type") && (types & GENERATE_CELLTYPE_ENTITIES) == 0)
- tag = "O";
+ tag = CODEC_TAG_O;
if (tag.endsWith("cell_line") && (types & GENERATE_CELLTYPE_ENTITIES) == 0)
- tag = "O";
+ tag = CODEC_TAG_O;
if (tag.endsWith("RNA") && (types & GENERATE_RNA_ENTITIES) == 0)
- tag = "O";
+ tag = CODEC_TAG_O;
- if (tag.startsWith("B-")) {
+ if (tag.startsWith(CODEC_TAG_B)) {
if (beginIndex != -1) {
names.add(new Span(beginIndex, endIndex, tags.get(beginIndex).substring(2)));
- beginIndex = -1;
- endIndex = -1;
}
beginIndex = i;
endIndex = i + 1;
}
- else if (tag.startsWith("I-")) {
+ else if (tag.startsWith(CODEC_TAG_I)) {
endIndex++;
}
- else if (tag.equals("O")) {
+ else if (tag.equals(CODEC_TAG_O)) {
if (beginIndex != -1) {
names.add(new Span(beginIndex, endIndex, tags.get(beginIndex).substring(2)));
beginIndex = -1;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java
index 422cd4c6..0b7bfe3c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java
@@ -19,7 +19,6 @@ package opennlp.tools.formats;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
@@ -30,46 +29,46 @@ import opennlp.tools.util.ObjectStream;
/**
* @see BioNLP2004NameSampleStream
*/
-public class BioNLP2004NameSampleStreamFactory<P> extends AbstractSampleStreamFactory<NameSample, P> {
+public class BioNLP2004NameSampleStreamFactory extends
+ AbstractSampleStreamFactory<NameSample, BioNLP2004NameSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ParameterDescription(valueName = "DNA,protein,cell_type,cell_line,RNA")
String getTypes();
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(NameSample.class,
- "bionlp2004", new BioNLP2004NameSampleStreamFactory<>(Parameters.class));
+ "bionlp2004", new BioNLP2004NameSampleStreamFactory(Parameters.class));
}
- protected BioNLP2004NameSampleStreamFactory(Class<P> params) {
+ protected BioNLP2004NameSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
int typesToGenerate = 0;
-
- if (params.getTypes().contains("DNA")) {
+ String types = params.getTypes();
+ if (types.contains("DNA")) {
typesToGenerate = typesToGenerate |
BioNLP2004NameSampleStream.GENERATE_DNA_ENTITIES;
}
- else if (params.getTypes().contains("protein")) {
+ if (types.contains("protein")) {
typesToGenerate = typesToGenerate |
BioNLP2004NameSampleStream.GENERATE_PROTEIN_ENTITIES;
}
- else if (params.getTypes().contains("cell_type")) {
+ if (types.contains("cell_type")) {
typesToGenerate = typesToGenerate |
BioNLP2004NameSampleStream.GENERATE_CELLTYPE_ENTITIES;
}
- else if (params.getTypes().contains("cell_line")) {
+ if (types.contains("cell_line")) {
typesToGenerate = typesToGenerate |
BioNLP2004NameSampleStream.GENERATE_CELLLINE_ENTITIES;
}
- else if (params.getTypes().contains("RNA")) {
+ if (types.contains("RNA")) {
typesToGenerate = typesToGenerate |
BioNLP2004NameSampleStream.GENERATE_RNA_ENTITIES;
}
@@ -77,8 +76,9 @@ public class BioNLP2004NameSampleStreamFactory<P> extends AbstractSampleStreamFa
try {
return new BioNLP2004NameSampleStream(
CmdLineUtil.createInputStreamFactory(params.getData()), typesToGenerate);
- } catch (IOException e) {
- throw new IllegalStateException(e);
+ } catch (IOException ex) {
+ CmdLineUtil.handleCreateObjectStreamError(ex);
}
+ return null;
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ChunkerSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ChunkerSampleStreamFactory.java
index 36f8b58e..8925d196 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ChunkerSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ChunkerSampleStreamFactory.java
@@ -17,49 +17,33 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
import opennlp.tools.chunker.ChunkSample;
import opennlp.tools.chunker.ChunkSampleStream;
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link ChunkSampleStream}s.
*/
-public class ChunkerSampleStreamFactory<P> extends AbstractSampleStreamFactory<ChunkSample, P> {
+public class ChunkerSampleStreamFactory extends
+ AbstractSampleStreamFactory<ChunkSample, ChunkerSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(ChunkSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new ChunkerSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new ChunkerSampleStreamFactory(Parameters.class));
}
- protected ChunkerSampleStreamFactory(Class<P> params) {
+ protected ChunkerSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<ChunkSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
-
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new ChunkSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStreamFactory.java
index d417df0c..91cca667 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStreamFactory.java
@@ -19,7 +19,6 @@ package opennlp.tools.formats;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
@@ -37,9 +36,10 @@ import opennlp.tools.util.ObjectStream;
* @see Conll02NameSampleStream
*/
@Internal
-public class Conll02NameSampleStreamFactory<P> extends LanguageSampleStreamFactory<NameSample, P> {
+public class Conll02NameSampleStreamFactory extends
+ LanguageSampleStreamFactory<NameSample, Conll02NameSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ParameterDescription(valueName = "spa|nld")
String getLang();
@@ -49,17 +49,17 @@ public class Conll02NameSampleStreamFactory<P> extends LanguageSampleStreamFacto
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(NameSample.class,
- "conll02", new Conll02NameSampleStreamFactory<>(Parameters.class));
+ "conll02", new Conll02NameSampleStreamFactory(Parameters.class));
}
- protected Conll02NameSampleStreamFactory(Class<P> params) {
+ protected Conll02NameSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
LANGUAGE lang;
if ("nl".equals(params.getLang()) || "nld".equals(params.getLang())) {
@@ -93,7 +93,6 @@ public class Conll02NameSampleStreamFactory<P> extends LanguageSampleStreamFacto
Conll02NameSampleStream.GENERATE_MISC_ENTITIES;
}
-
try {
return new Conll02NameSampleStream(lang,
CmdLineUtil.createInputStreamFactory(params.getData()), typesToGenerate);
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStreamFactory.java
index dae580cb..d1a61509 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStreamFactory.java
@@ -19,7 +19,6 @@ package opennlp.tools.formats;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
@@ -32,9 +31,10 @@ import opennlp.tools.util.ObjectStream;
/**
* @see Conll03NameSampleStream
*/
-public class Conll03NameSampleStreamFactory<P> extends LanguageSampleStreamFactory<NameSample, P> {
+public class Conll03NameSampleStreamFactory extends
+ LanguageSampleStreamFactory<NameSample, Conll03NameSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ParameterDescription(valueName = "eng|deu")
String getLang();
@@ -44,17 +44,17 @@ public class Conll03NameSampleStreamFactory<P> extends LanguageSampleStreamFacto
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(NameSample.class,
- "conll03", new Conll03NameSampleStreamFactory<>(Parameters.class));
+ "conll03", new Conll03NameSampleStreamFactory(Parameters.class));
}
- protected Conll03NameSampleStreamFactory(Class<P> params) {
+ protected Conll03NameSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
// TODO: support the other languages with this CoNLL.
LANGUAGE lang;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStreamFactory.java
index 18b48b5a..a7a586d9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStreamFactory.java
@@ -18,13 +18,10 @@
package opennlp.tools.formats;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
-import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.commons.Internal;
import opennlp.tools.postag.POSSample;
@@ -35,40 +32,35 @@ import opennlp.tools.util.ObjectStream;
* <b>Note:</b>
* Do not use this class, internal use only!
*
+ * @see POSSample
* @see ConllXPOSSampleStream
*/
@Internal
-public class ConllXPOSSampleStreamFactory<P> extends AbstractSampleStreamFactory<POSSample, P> {
+public class ConllXPOSSampleStreamFactory extends
+ AbstractSampleStreamFactory<POSSample, ConllXPOSSampleStreamFactory.Parameters> {
public static final String CONLLX_FORMAT = "conllx";
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(POSSample.class,
- CONLLX_FORMAT, new ConllXPOSSampleStreamFactory<>(Parameters.class));
+ CONLLX_FORMAT, new ConllXPOSSampleStreamFactory(Parameters.class));
}
- protected ConllXPOSSampleStreamFactory(Class<P> params) {
+ protected ConllXPOSSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<POSSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- InputStreamFactory inFactory =
- CmdLineUtil.createInputStreamFactory(params.getData());
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
try {
+ InputStreamFactory inFactory = CmdLineUtil.createInputStreamFactory(params.getData());
return new ConllXPOSSampleStream(inFactory, StandardCharsets.UTF_8);
- } catch (UnsupportedEncodingException e) {
- // this shouldn't happen
- throw new TerminateToolException(-1, "UTF-8 encoding is not supported: " + e.getMessage(), e);
- }
- catch (IOException e) {
- // That will throw an exception
+ } catch (IOException e) {
CmdLineUtil.handleCreateObjectStreamError(e);
return null;
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactory.java
index 505f94f2..2d622e0f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactory.java
@@ -29,29 +29,31 @@ import opennlp.tools.util.ObjectStream;
/**
* <b>Note:</b>
* Do not use this class, internal use only!
+ *
+ * @see SentenceSample
+ * @see POSToSentenceSampleStream
*/
@Internal
-public class ConllXSentenceSampleStreamFactory<P> extends
- DetokenizerSampleStreamFactory<SentenceSample, P> {
+public class ConllXSentenceSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<SentenceSample, ConllXSentenceSampleStreamFactory.Parameters> {
- interface Parameters extends ConllXPOSSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends ConllXPOSSampleStreamFactory.Parameters, DetokenizerParameter {
// TODO: make chunk size configurable
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
ConllXPOSSampleStreamFactory.CONLLX_FORMAT,
- new ConllXSentenceSampleStreamFactory<>(Parameters.class));
+ new ConllXSentenceSampleStreamFactory(Parameters.class));
}
- protected ConllXSentenceSampleStreamFactory(Class<P> params) {
+ protected ConllXSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<POSSample> posSampleStream = StreamFactoryRegistry.getFactory(POSSample.class,
ConllXPOSSampleStreamFactory.CONLLX_FORMAT).create(
ArgumentParser.filter(args, ConllXPOSSampleStreamFactory.Parameters.class));
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXTokenSampleStreamFactory.java
index c894be0a..81c1eb21 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXTokenSampleStreamFactory.java
@@ -29,25 +29,29 @@ import opennlp.tools.util.ObjectStream;
/**
* <b>Note:</b>
* Do not use this class, internal use only!
+ *
+ * @see TokenSample
+ * @see POSToTokenSampleStream
*/
@Internal
-public class ConllXTokenSampleStreamFactory<P> extends DetokenizerSampleStreamFactory<TokenSample, P> {
+public class ConllXTokenSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<TokenSample, ConllXTokenSampleStreamFactory.Parameters> {
- interface Parameters extends ConllXPOSSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends ConllXPOSSampleStreamFactory.Parameters, DetokenizerParameter {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class,
- ConllXPOSSampleStreamFactory.CONLLX_FORMAT, new ConllXTokenSampleStreamFactory<>(Parameters.class));
+ ConllXPOSSampleStreamFactory.CONLLX_FORMAT, new ConllXTokenSampleStreamFactory(Parameters.class));
}
- protected ConllXTokenSampleStreamFactory(Class<P> params) {
+ protected ConllXTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<POSSample> samples = StreamFactoryRegistry.getFactory(POSSample.class,
ConllXPOSSampleStreamFactory.CONLLX_FORMAT).create(
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/DocumentSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/DocumentSampleStreamFactory.java
index 2ecf5664..1775eaa4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/DocumentSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/DocumentSampleStreamFactory.java
@@ -17,48 +17,33 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.doccat.DocumentSampleStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link DocumentSampleStream}s.
*/
-public class DocumentSampleStreamFactory<P> extends AbstractSampleStreamFactory<DocumentSample, P> {
+public class DocumentSampleStreamFactory extends
+ AbstractSampleStreamFactory<DocumentSample, DocumentSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(DocumentSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new DocumentSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new DocumentSampleStreamFactory(Parameters.class));
}
- protected DocumentSampleStreamFactory(Class<P> params) {
+ protected DocumentSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<DocumentSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new DocumentSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
index 76574500..a1382c5d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
@@ -32,7 +32,7 @@ import opennlp.tools.util.Span;
import opennlp.tools.util.StringUtil;
/**
- * Parser for the Italian NER training files of the Evalita 2007 and 2009 NER shared tasks.
+ * Parser for the Italian NER training files of the Evalita 2007 and 2009 NER shared tasks.
* <p>
* The data does not contain article boundaries,
* adaptive data will be cleared for every sentence.
@@ -46,12 +46,12 @@ import opennlp.tools.util.StringUtil;
* 2. The Entity type tag: PER (for Person), ORG (for Organization),
* GPE (for Geo-Political Entity), or LOC (for Location).
* <p>
- * Each file consists of four columns separated by a blank, containing
- * respectively the token, the Elsnet PoS-tag, the Adige news story to
- * which the token belongs, and the Named Entity tag.
+ * Each file consists of four columns separated by a blank, containing respectively the token, the
+ * <a href="https://www.evalita.it/wp-content/uploads/2021/11/elsnet-tagset-IT.pdf">Elsnet</a>
+ * PoS-tag, the Adige news story to which the token belongs, and the Named Entity tag.
* <p>
* Data can be found on this
- * <a href="http://www.evalita.it">web site</a>.
+ * <a href="https://www.evalita.it">web site</a>.
* <p>
* <b>Note:</b>
* Do not use this class, internal use only!
@@ -59,6 +59,15 @@ import opennlp.tools.util.StringUtil;
@Internal
public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
+ public static final String DOCSTART = "-DOCSTART-";
+ private static final String CODEC_TAG_O = "O";
+ private static final String CODEC_TAG_B = "B-";
+ private static final String CODEC_TAG_I = "I-";
+ private static final String ENT_TYPE_PER = "PER"; // Person
+ private static final String ENT_TYPE_LOC = "LOC"; // Location
+ private static final String ENT_TYPE_GPE = "GPE"; // Geo-Political Entity
+ private static final String ENT_TYPE_ORG = "ORG"; // Organization
+
public enum LANGUAGE {
IT
}
@@ -68,8 +77,6 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
public static final int GENERATE_LOCATION_ENTITIES = 0x01 << 2;
public static final int GENERATE_GPE_ENTITIES = 0x01 << 3;
- public static final String DOCSTART = "-DOCSTART-";
-
private final LANGUAGE lang;
private final ObjectStream<String> lineStream;
@@ -82,7 +89,7 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
}
public EvalitaNameSampleStream(LANGUAGE lang, InputStreamFactory in, int types) throws IOException {
- this(lang, new PlainTextByLineStream(in, StandardCharsets.UTF_8),types);
+ this(lang, new PlainTextByLineStream(in, StandardCharsets.UTF_8), types);
}
private static Span extract(int begin, int end, String beginTag) throws InvalidFormatException {
@@ -90,17 +97,16 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
String type = beginTag.substring(2);
type = switch (type) {
- case "PER" -> "person";
- case "LOC" -> "location";
- case "GPE" -> "gpe";
- case "ORG" -> "organization";
+ case ENT_TYPE_PER -> "person";
+ case ENT_TYPE_LOC -> "location";
+ case ENT_TYPE_GPE -> "gpe";
+ case ENT_TYPE_ORG -> "organization";
default -> throw new InvalidFormatException("Unknown type: " + type);
};
return new Span(begin, end, type);
}
-
@Override
public NameSample read() throws IOException {
@@ -110,7 +116,6 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
boolean isClearAdaptiveData = false;
// Empty line indicates end of sentence
-
String line;
while ((line = lineStream.read()) != null && !StringUtil.isEmpty(line)) {
@@ -140,7 +145,7 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
if (LANGUAGE.IT.equals(lang))
isClearAdaptiveData = true;
- if (sentence.size() > 0) {
+ if (!sentence.isEmpty()) {
// convert name tags into spans
List<Span> names = new ArrayList<>();
@@ -151,33 +156,31 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample> {
String tag = tags.get(i);
- if (tag.endsWith("PER") && (types & GENERATE_PERSON_ENTITIES) == 0)
- tag = "O";
+ if (tag.endsWith(ENT_TYPE_PER) && (types & GENERATE_PERSON_ENTITIES) == 0)
+ tag = CODEC_TAG_O;
- if (tag.endsWith("ORG") && (types & GENERATE_ORGANIZATION_ENTITIES) == 0)
- tag = "O";
+ if (tag.endsWith(ENT_TYPE_ORG) && (types & GENERATE_ORGANIZATION_ENTITIES) == 0)
+ tag = CODEC_TAG_O;
- if (tag.endsWith("LOC") && (types & GENERATE_LOCATION_ENTITIES) == 0)
- tag = "O";
+ if (tag.endsWith(ENT_TYPE_LOC) && (types & GENERATE_LOCATION_ENTITIES) == 0)
+ tag = CODEC_TAG_O;
- if (tag.endsWith("GPE") && (types & GENERATE_GPE_ENTITIES) == 0)
- tag = "O";
+ if (tag.endsWith(ENT_TYPE_GPE) && (types & GENERATE_GPE_ENTITIES) == 0)
+ tag = CODEC_TAG_O;
- if (tag.startsWith("B-")) {
+ if (tag.startsWith(CODEC_TAG_B)) {
if (beginIndex != -1) {
names.add(extract(beginIndex, endIndex, tags.get(beginIndex)));
- beginIndex = -1;
- endIndex = -1;
}
beginIndex = i;
endIndex = i + 1;
}
- else if (tag.startsWith("I-")) {
+ else if (tag.startsWith(CODEC_TAG_I)) {
endIndex++;
}
- else if (tag.equals("O")) {
+ else if (tag.equals(CODEC_TAG_O)) {
if (beginIndex != -1) {
names.add(extract(beginIndex, endIndex, tags.get(beginIndex)));
beginIndex = -1;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStreamFactory.java
index 7fa9db40..066861f3 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStreamFactory.java
@@ -19,7 +19,6 @@ package opennlp.tools.formats;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
@@ -37,9 +36,10 @@ import opennlp.tools.util.ObjectStream;
* @see EvalitaNameSampleStream
*/
@Internal
-public class EvalitaNameSampleStreamFactory<P> extends LanguageSampleStreamFactory<NameSample, P> {
+public class EvalitaNameSampleStreamFactory extends
+ LanguageSampleStreamFactory<NameSample, EvalitaNameSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ParameterDescription(valueName = "it")
String getLang();
@@ -49,17 +49,16 @@ public class EvalitaNameSampleStreamFactory<P> extends LanguageSampleStreamFacto
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(NameSample.class,
- "evalita", new EvalitaNameSampleStreamFactory<>(Parameters.class));
+ "evalita", new EvalitaNameSampleStreamFactory(Parameters.class));
}
- protected EvalitaNameSampleStreamFactory(Class<P> params) {
+ protected EvalitaNameSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
LANGUAGE lang;
if ("it".equals(params.getLang())) {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactory.java
index bda9d482..9aedc4bb 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactory.java
@@ -17,51 +17,34 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
-import opennlp.tools.doccat.DocumentSampleStream;
import opennlp.tools.langdetect.LanguageDetectorSampleStream;
import opennlp.tools.langdetect.LanguageSample;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
- * Factory producing OpenNLP {@link DocumentSampleStream}s.
+ * Factory producing OpenNLP {@link LanguageDetectorSampleStream lang detector sample streams}.
*/
-public class LanguageDetectorSampleStreamFactory<P>
- extends AbstractSampleStreamFactory<LanguageSample, P> {
+public class LanguageDetectorSampleStreamFactory extends
+ AbstractSampleStreamFactory<LanguageSample, LanguageDetectorSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(LanguageSample.class,
StreamFactoryRegistry.DEFAULT_FORMAT,
- new LanguageDetectorSampleStreamFactory<>(Parameters.class));
+ new LanguageDetectorSampleStreamFactory(Parameters.class));
}
- protected LanguageDetectorSampleStreamFactory(Class<P> params) {
+ protected LanguageDetectorSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<LanguageSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new LanguageDetectorSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/LemmatizerSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/LemmatizerSampleStreamFactory.java
index dfb137e7..f2209170 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/LemmatizerSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/LemmatizerSampleStreamFactory.java
@@ -17,49 +17,33 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.lemmatizer.LemmaSample;
import opennlp.tools.lemmatizer.LemmaSampleStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link LemmaSampleStream}s.
*/
-public class LemmatizerSampleStreamFactory<P> extends AbstractSampleStreamFactory<LemmaSample, P> {
+public class LemmatizerSampleStreamFactory extends
+ AbstractSampleStreamFactory<LemmaSample, LemmatizerSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(LemmaSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new LemmatizerSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new LemmatizerSampleStreamFactory(Parameters.class));
}
- protected LemmatizerSampleStreamFactory(Class<P> params) {
+ protected LemmatizerSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<LemmaSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
-
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new LemmaSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java
index 508359bb..efc786e1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java
@@ -17,50 +17,35 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.namefind.NameSample;
import opennlp.tools.namefind.NameSampleDataStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link NameSampleDataStream}s.
*/
-public class NameSampleDataStreamFactory<P> extends AbstractSampleStreamFactory<NameSample, P> {
+@Internal
+public class NameSampleDataStreamFactory extends
+ AbstractSampleStreamFactory<NameSample, NameSampleDataStreamFactory.Parameters> {
public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(NameSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new NameSampleDataStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new NameSampleDataStreamFactory(Parameters.class));
}
- protected NameSampleDataStreamFactory(Class<P> params) {
+ protected NameSampleDataStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
-
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new NameSampleDataStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ParseSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ParseSampleStreamFactory.java
index 6ed45743..2ce9ec36 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ParseSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ParseSampleStreamFactory.java
@@ -17,49 +17,33 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.ParseSampleStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link ParseSampleStream}s.
*/
-public class ParseSampleStreamFactory<P> extends AbstractSampleStreamFactory<Parse, P> {
+public class ParseSampleStreamFactory extends
+ AbstractSampleStreamFactory<Parse, ParseSampleStreamFactory.Parameters> {
public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(Parse.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new ParseSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new ParseSampleStreamFactory(Parameters.class));
}
- protected ParseSampleStreamFactory(Class<P> params) {
+ protected ParseSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<Parse> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new ParseSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/SentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/SentenceSampleStreamFactory.java
index e002bbb1..61b8e6bb 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/SentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/SentenceSampleStreamFactory.java
@@ -17,50 +17,34 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.sentdetect.SentenceSampleStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link SentenceSampleStream}s.
*/
-public class SentenceSampleStreamFactory<P> extends AbstractSampleStreamFactory<SentenceSample, P> {
+public class SentenceSampleStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, SentenceSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new SentenceSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new SentenceSampleStreamFactory(Parameters.class));
}
- protected SentenceSampleStreamFactory(Class<P> params) {
+ protected SentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new SentenceSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/TokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/TokenSampleStreamFactory.java
index ffbd1e6c..75dc62d4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/TokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/TokenSampleStreamFactory.java
@@ -17,49 +17,33 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.tokenize.TokenSample;
import opennlp.tools.tokenize.TokenSampleStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link TokenSampleStream}s.
*/
-public class TokenSampleStreamFactory<P> extends LanguageSampleStreamFactory<TokenSample, P> {
+public class TokenSampleStreamFactory extends
+ LanguageSampleStreamFactory<TokenSample, TokenSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new TokenSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new TokenSampleStreamFactory(Parameters.class));
}
- protected TokenSampleStreamFactory(Class<P> params) {
+ protected TokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new TokenSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStream.java
index eb3ab1df..c69319ce 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStream.java
@@ -28,6 +28,15 @@ import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.util.ObjectStream;
+/**
+ * An {@link ObjectStream} implementation for the Twenty Newsgroups text corpus.
+ * <p>
+ * The document collection was created and donated by: <i>Tom Mitchell,
+ * School of Computer Science, Carnegie Mellon University</i>.
+ * <p>
+ * Details and the data can be found via this DOI:
+ * <a href="https://dx.doi.org/10.24432/C5C323">10.24432/C5C323</a>.
+ */
public class TwentyNewsgroupSampleStream implements ObjectStream<DocumentSample> {
private final Tokenizer tokenizer;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactory.java
index edf3d5d3..85179eae 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactory.java
@@ -26,35 +26,55 @@ import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.cmdline.params.EncodingParameter;
import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.tokenize.SimpleTokenizer;
+import opennlp.tools.tokenize.ThreadSafeTokenizerME;
import opennlp.tools.tokenize.Tokenizer;
-import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.tokenize.WhitespaceTokenizer;
import opennlp.tools.util.ObjectStream;
-public class TwentyNewsgroupSampleStreamFactory<P> extends AbstractSampleStreamFactory<DocumentSample,P> {
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see TwentyNewsgroupSampleStream
+ */
+public class TwentyNewsgroupSampleStreamFactory extends
+ AbstractSampleStreamFactory<DocumentSample, TwentyNewsgroupSampleStreamFactory.Parameters> {
+
+ public interface Parameters extends EncodingParameter {
+ @ArgumentParser.ParameterDescription(valueName = "dataDir",
+ description = "dir containing the 20newsgroup folders")
+ File getDataDir();
+
+ @ArgumentParser.ParameterDescription(valueName = "modelFile")
+ @ArgumentParser.OptionalParameter
+ File getTokenizerModel();
+
+ @ArgumentParser.ParameterDescription(valueName = "name")
+ @ArgumentParser.OptionalParameter
+ String getRuleBasedTokenizer();
+ }
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(DocumentSample.class,
"20newsgroup",
- new TwentyNewsgroupSampleStreamFactory<>(TwentyNewsgroupSampleStreamFactory.Parameters.class));
+ new TwentyNewsgroupSampleStreamFactory(TwentyNewsgroupSampleStreamFactory.Parameters.class));
}
- protected TwentyNewsgroupSampleStreamFactory(Class<P> params) {
+ protected TwentyNewsgroupSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<DocumentSample> create(String[] args) {
-
- TwentyNewsgroupSampleStreamFactory.Parameters params =
- ArgumentParser.parse(args, TwentyNewsgroupSampleStreamFactory.Parameters.class);
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ Parameters params = ArgumentParser.parse(args, Parameters.class);
Tokenizer tokenizer = WhitespaceTokenizer.INSTANCE;
-
if (params.getTokenizerModel() != null) {
try {
- tokenizer = new TokenizerME(new TokenizerModel(params.getTokenizerModel()));
+ tokenizer = new ThreadSafeTokenizerME(new TokenizerModel(params.getTokenizerModel()));
} catch (IOException e) {
throw new TerminateToolException(-1, "Failed to load tokenizer model!", e);
}
@@ -74,24 +94,10 @@ public class TwentyNewsgroupSampleStreamFactory<P> extends AbstractSampleStreamF
}
try {
- return new TwentyNewsgroupSampleStream(
- tokenizer, params.getDataDir().toPath());
+ return new TwentyNewsgroupSampleStream(tokenizer, params.getDataDir().toPath());
} catch (IOException e) {
throw new TerminateToolException(-1, "IO error while opening sample data: " + e.getMessage(), e);
}
}
- interface Parameters extends EncodingParameter {
- @ArgumentParser.ParameterDescription(valueName = "dataDir",
- description = "dir containing the 20newsgroup folders")
- File getDataDir();
-
- @ArgumentParser.ParameterDescription(valueName = "modelFile")
- @ArgumentParser.OptionalParameter
- File getTokenizerModel();
-
- @ArgumentParser.ParameterDescription(valueName = "name")
- @ArgumentParser.OptionalParameter
- String getRuleBasedTokenizer();
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/WordTagSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/WordTagSampleStreamFactory.java
index 4972b4d1..6b92c41a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/WordTagSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/WordTagSampleStreamFactory.java
@@ -17,52 +17,37 @@
package opennlp.tools.formats;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.commons.Internal;
import opennlp.tools.postag.POSSample;
import opennlp.tools.postag.WordTagSampleStream;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* <b>Note:</b>
* Do not use this class, internal use only!
*/
@Internal
-public class WordTagSampleStreamFactory<P> extends AbstractSampleStreamFactory<POSSample, P> {
+public class WordTagSampleStreamFactory extends
+ AbstractSampleStreamFactory<POSSample, WordTagSampleStreamFactory.Parameters> {
public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(POSSample.class,
- StreamFactoryRegistry.DEFAULT_FORMAT, new WordTagSampleStreamFactory<>(Parameters.class));
+ StreamFactoryRegistry.DEFAULT_FORMAT, new WordTagSampleStreamFactory(Parameters.class));
}
- protected WordTagSampleStreamFactory(Class<P> params) {
+ protected WordTagSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<POSSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new WordTagSampleStream(lineStream);
}
+
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
index cdcbd9d1..6c51ee68 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
@@ -104,14 +104,14 @@ public class ADChunkSampleStream implements ObjectStream<ChunkSample> {
index++;
// skip this one
} else {
- Node root = paragraph.getRoot();
+ Node root = paragraph.root();
List<String> sentence = new ArrayList<>();
List<String> tags = new ArrayList<>();
List<String> target = new ArrayList<>();
processRoot(root, sentence, tags, target);
- if (sentence.size() > 0) {
+ if (!sentence.isEmpty()) {
index++;
return new ChunkSample(sentence, tags, target);
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStreamFactory.java
index 49922f8c..95183dad 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStreamFactory.java
@@ -17,21 +17,16 @@
package opennlp.tools.formats.ad;
-import java.io.File;
-import java.io.IOException;
import java.nio.charset.Charset;
import opennlp.tools.chunker.ChunkSample;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.OptionalParameter;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.commons.Internal;
import opennlp.tools.formats.LanguageSampleStreamFactory;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* A Factory to create a Arvores Deitadas ChunkStream from the command line
@@ -41,18 +36,16 @@ import opennlp.tools.util.PlainTextByLineStream;
* Do not use this class, internal use only!
*/
@Internal
-public class ADChunkSampleStreamFactory<P> extends LanguageSampleStreamFactory<ChunkSample, P> {
+public class ADChunkSampleStreamFactory extends
+ LanguageSampleStreamFactory<ChunkSample, ADChunkSampleStreamFactory.Parameters> {
- interface Parameters {
+ public interface Parameters extends BasicFormatParams {
//all have to be repeated, because encoding is not optional,
//according to the check if (encoding == null) { below (now removed)
@ParameterDescription(valueName = "charsetName",
description = "encoding for reading and writing text, if absent the system default is used.")
Charset getEncoding();
- @ParameterDescription(valueName = "sampleData", description = "data to be used, usually a file name.")
- File getData();
-
@ParameterDescription(valueName = "language", description = "language which is being processed.")
String getLang();
@@ -67,26 +60,17 @@ public class ADChunkSampleStreamFactory<P> extends LanguageSampleStreamFactory<C
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(ChunkSample.class,
- "ad", new ADChunkSampleStreamFactory<>(Parameters.class));
+ "ad", new ADChunkSampleStreamFactory(Parameters.class));
}
- protected ADChunkSampleStreamFactory(Class<P> params) {
+ protected ADChunkSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<ChunkSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
- language = params.getLang();
-
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
ADChunkSampleStream sampleStream = new ADChunkSampleStream(lineStream);
if (params.getStart() != null && params.getStart() > -1) {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
index dae804be..d2db0635 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
@@ -206,7 +206,7 @@ public class ADNameSampleStream implements ObjectStream<NameSample> {
textID = currentTextID;
}
- Node root = paragraph.getRoot();
+ Node root = paragraph.root();
List<String> sentence = new ArrayList<>();
List<Span> names = new ArrayList<>();
process(root, sentence, names);
@@ -438,7 +438,7 @@ public class ADNameSampleStream implements ObjectStream<NameSample> {
private int getTextID(Sentence paragraph) {
- final String meta = paragraph.getMetadata();
+ final String meta = paragraph.metadata();
Type corpusType;
Pattern metaPattern;
int textIdMeta2 = -1;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStreamFactory.java
index 525b4097..b1af32eb 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStreamFactory.java
@@ -17,21 +17,16 @@
package opennlp.tools.formats.ad;
-import java.io.File;
-import java.io.IOException;
import java.nio.charset.Charset;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.OptionalParameter;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.commons.Internal;
import opennlp.tools.formats.LanguageSampleStreamFactory;
import opennlp.tools.namefind.NameSample;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* A Factory to create a Arvores Deitadas NameSampleDataStream from the command line
@@ -41,18 +36,16 @@ import opennlp.tools.util.PlainTextByLineStream;
* Do not use this class, internal use only!
*/
@Internal
-public class ADNameSampleStreamFactory<P> extends LanguageSampleStreamFactory<NameSample, P> {
+public class ADNameSampleStreamFactory extends
+ LanguageSampleStreamFactory<NameSample, ADNameSampleStreamFactory.Parameters> {
- interface Parameters {
+ public interface Parameters extends BasicFormatParams {
//all have to be repeated, because encoding is not optional,
//according to the check if (encoding == null) { below (now removed)
@ParameterDescription(valueName = "charsetName",
description = "encoding for reading and writing text, if absent the system default is used.")
Charset getEncoding();
- @ParameterDescription(valueName = "sampleData", description = "data to be used, usually a file name.")
- File getData();
-
@ParameterDescription(valueName = "split",
description = "if true all hyphenated tokens will be separated (default true)")
@OptionalParameter(defaultValue = "true")
@@ -64,27 +57,18 @@ public class ADNameSampleStreamFactory<P> extends LanguageSampleStreamFactory<Na
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(NameSample.class,
- "ad", new ADNameSampleStreamFactory<>(Parameters.class));
+ "ad", new ADNameSampleStreamFactory(Parameters.class));
}
- protected ADNameSampleStreamFactory(Class<P> params) {
+ protected ADNameSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
language = params.getLang();
-
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new ADNameSampleStream(lineStream, params.getSplitHyphenatedTokens());
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
index 742e27e6..8b7bd964 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
@@ -84,7 +84,7 @@ public class ADPOSSampleStream implements ObjectStream<POSSample> {
public POSSample read() throws IOException {
Sentence paragraph;
if ((paragraph = this.adSentenceStream.read()) != null) {
- Node root = paragraph.getRoot();
+ Node root = paragraph.root();
List<String> sentence = new ArrayList<>();
List<String> tags = new ArrayList<>();
process(root, sentence, tags);
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStreamFactory.java
index 80dff476..88771d71 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStreamFactory.java
@@ -17,38 +17,30 @@
package opennlp.tools.formats.ad;
-import java.io.File;
-import java.io.IOException;
import java.nio.charset.Charset;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.OptionalParameter;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.commons.Internal;
import opennlp.tools.formats.LanguageSampleStreamFactory;
import opennlp.tools.postag.POSSample;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* <b>Note:</b>
* Do not use this class, internal use only!
*/
@Internal
-public class ADPOSSampleStreamFactory<P> extends
- LanguageSampleStreamFactory<POSSample, P> {
+public class ADPOSSampleStreamFactory extends
+ LanguageSampleStreamFactory<POSSample, ADPOSSampleStreamFactory.Parameters> {
- interface Parameters {
+ public interface Parameters extends BasicFormatParams {
@ParameterDescription(valueName = "charsetName",
description = "encoding for reading and writing text, if absent the system default is used.")
Charset getEncoding();
- @ParameterDescription(valueName = "sampleData", description = "data to be used, usually a file name.")
- File getData();
-
@ParameterDescription(valueName = "language", description = "language which is being processed.")
String getLang();
@@ -64,27 +56,18 @@ public class ADPOSSampleStreamFactory<P> extends
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(POSSample.class, "ad",
- new ADPOSSampleStreamFactory<>(Parameters.class));
+ new ADPOSSampleStreamFactory(Parameters.class));
}
- protected ADPOSSampleStreamFactory(Class<P> params) {
+ protected ADPOSSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<POSSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
language = params.getLang();
-
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new ADPOSSampleStream(lineStream, params.getExpandME(), params.getIncludeFeatures());
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
index d5a3401b..c78bf2b9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
@@ -99,9 +99,9 @@ public class ADSentenceSampleStream implements ObjectStream<SentenceSample> {
do {
do {
if (!isTitle || isIncludeTitles) {
- if (hasPunctuation(sent.getText())) {
+ if (hasPunctuation(sent.text())) {
int start = document.length();
- document.append(sent.getText());
+ document.append(sent.text());
sentences.add(new Span(start, document.length()));
document.append(" ");
}
@@ -116,7 +116,7 @@ public class ADSentenceSampleStream implements ObjectStream<SentenceSample> {
while (isSameText);
String doc;
- if (document.length() > 0) {
+ if (!document.isEmpty()) {
doc = document.substring(0, document.length() - 1);
} else {
doc = document.toString();
@@ -127,7 +127,7 @@ public class ADSentenceSampleStream implements ObjectStream<SentenceSample> {
private boolean hasPunctuation(String text) {
text = text.trim();
- if (text.length() > 0) {
+ if (!text.isEmpty()) {
char lastChar = text.charAt(text.length() - 1);
return Arrays.binarySearch(ptEosCharacters, lastChar) >= 0;
}
@@ -135,13 +135,12 @@ public class ADSentenceSampleStream implements ObjectStream<SentenceSample> {
}
// there are some different types of metadata depending on the corpus.
- // TODO Merge this patterns
- private static final Pattern META_1 = Pattern
- .compile("^(?:[a-zA-Z\\-]*(\\d+)).*?p=(\\d+).*");
+ // TODO Merge these patterns
+ private static final Pattern META_1 = Pattern.compile("^(?:[a-zA-Z\\-]*(\\d+)).*?p=(\\d+).*");
private void updateMeta() {
if (this.sent != null) {
- String meta = this.sent.getMetadata();
+ String meta = this.sent.metadata();
Matcher m = META_1.matcher(meta);
int currentText;
int currentPara;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStreamFactory.java
index 48748c92..50c8f593 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStreamFactory.java
@@ -17,37 +17,29 @@
package opennlp.tools.formats.ad;
-import java.io.File;
-import java.io.IOException;
import java.nio.charset.Charset;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.OptionalParameter;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.commons.Internal;
import opennlp.tools.formats.LanguageSampleStreamFactory;
import opennlp.tools.sentdetect.SentenceSample;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* <b>Note:</b>
* Do not use this class, internal use only!
*/
@Internal
-public class ADSentenceSampleStreamFactory<P> extends
- LanguageSampleStreamFactory<SentenceSample, P> {
+public class ADSentenceSampleStreamFactory extends
+ LanguageSampleStreamFactory<SentenceSample, ADSentenceSampleStreamFactory.Parameters> {
- interface Parameters {
+ public interface Parameters extends BasicFormatParams {
@ParameterDescription(valueName = "charsetName", description = "encoding for reading and writing text.")
Charset getEncoding();
- @ParameterDescription(valueName = "sampleData", description = "data to be used, usually a file name.")
- File getData();
-
@ParameterDescription(valueName = "language", description = "language which is being processed.")
String getLang();
@@ -59,29 +51,19 @@ public class ADSentenceSampleStreamFactory<P> extends
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class, "ad",
- new ADSentenceSampleStreamFactory<>(Parameters.class));
+ new ADSentenceSampleStreamFactory(Parameters.class));
}
- protected ADSentenceSampleStreamFactory(Class<P> params) {
+ protected ADSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
language = params.getLang();
-
boolean includeTitle = params.getIncludeTitles();
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new ADSentenceSampleStream(lineStream, includeTitle);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
index 27e91749..1445f153 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
@@ -51,38 +51,8 @@ import opennlp.tools.util.ObjectStream;
@Internal
public class ADSentenceStream extends FilterObjectStream<String, ADSentenceStream.Sentence> {
- public static class Sentence {
-
- private String text;
- private Node root;
- private String metadata;
-
+ public record Sentence (String text, Node root, String metadata) {
public static final String META_LABEL_FINAL = "final";
-
- public String getText() {
- return text;
- }
-
- public void setText(String text) {
- this.text = text;
- }
-
- public Node getRoot() {
- return root;
- }
-
- public void setRoot(Node root) {
- this.root = root;
- }
-
- public void setMetadata(String metadata) {
- this.metadata = metadata;
- }
-
- public String getMetadata() {
- return metadata;
- }
-
}
/**
@@ -116,7 +86,7 @@ public class ADSentenceStream extends FilterObjectStream<String, ADSentenceStrea
* @return A {@link Sentence} instance parsed from {@code sentenceString}.
*/
public Sentence parse(String sentenceString, int para, boolean isTitle, boolean isBox) {
- Sentence sentence = new Sentence();
+ Sentence sentence;
Node root = new Node();
try (BufferedReader reader = new BufferedReader(new StringReader(sentenceString))) {
// first line is <s ...>
@@ -153,8 +123,7 @@ public class ADSentenceStream extends FilterObjectStream<String, ADSentenceStrea
meta = line.substring(0, start) + " p=" + para + titleTag + boxTag + metaFromSource;
}
}
- sentence.setText(text);
- sentence.setMetadata(meta);
+ sentence = new Sentence(text, root, meta);
// now we look for the root node
do {
line = reader.readLine();
@@ -232,10 +201,9 @@ public class ADSentenceStream extends FilterObjectStream<String, ADSentenceStrea
} catch (Exception e) {
logger.warn("Caught exception for the given sentence: '{}'", sentenceString, e);
- return sentence;
+ return null;
}
// second line should be SOURCE
- sentence.setRoot(root);
return sentence;
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactory.java
index 7a93006a..a720b1ab 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactory.java
@@ -32,24 +32,24 @@ import opennlp.tools.util.ObjectStream;
* Do not use this class, internal use only!
*/
@Internal
-public class ADTokenSampleStreamFactory<P> extends
- DetokenizerSampleStreamFactory<TokenSample, P> {
+public class ADTokenSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<TokenSample, ADTokenSampleStreamFactory.Parameters> {
- interface Parameters extends ADNameSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends ADNameSampleStreamFactory.Parameters, DetokenizerParameter {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class, "ad",
- new ADTokenSampleStreamFactory<>(Parameters.class));
+ new ADTokenSampleStreamFactory(Parameters.class));
}
- protected ADTokenSampleStreamFactory(Class<P> params) {
+ protected ADTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<NameSample> samples = StreamFactoryRegistry.getFactory(
NameSample.class, "ad").create(
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/PortugueseContractionUtility.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/PortugueseContractionUtility.java
index c734ba46..36b1f79a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/PortugueseContractionUtility.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/PortugueseContractionUtility.java
@@ -40,6 +40,7 @@ import opennlp.tools.util.StringUtil;
public class PortugueseContractionUtility {
protected static final Map<String, String> CONTRACTIONS;
+ private static final String SYMBOL_PLUS = "+";
static {
Map<String, String> elems = new HashMap<>();
@@ -162,7 +163,7 @@ public class PortugueseContractionUtility {
* @return The merged contraction.
*/
public static String toContraction(String left, String right) {
- String key = left + "+" + right;
+ String key = left + SYMBOL_PLUS + right;
if (CONTRACTIONS.containsKey(key)) {
return CONTRACTIONS.get(key);
} else {
@@ -171,7 +172,7 @@ public class PortugueseContractionUtility {
for (int i = 0; i < parts.length - 1; i++) {
sb.append(parts[i]).append(" ");
}
- key = parts[parts.length - 1] + "+" + right;
+ key = parts[parts.length - 1] + SYMBOL_PLUS + right;
if (CONTRACTIONS.containsKey(key)) {
sb.append(CONTRACTIONS.get(key));
return sb.toString();
@@ -180,7 +181,7 @@ public class PortugueseContractionUtility {
if (right.contains("_")) {
parts = right.split("_");
- key = left + "+" + parts[0];
+ key = left + SYMBOL_PLUS + parts[0];
if (CONTRACTIONS.containsKey(key)) {
sb.append(CONTRACTIONS.get(key)).append(" ");
@@ -194,7 +195,7 @@ public class PortugueseContractionUtility {
}
String leftLower = StringUtil.toLowerCase(parts[parts.length - 1]);
- key = leftLower + "+" + right;
+ key = leftLower + SYMBOL_PLUS + right;
if (CONTRACTIONS.containsKey(key)) {
String r = CONTRACTIONS.get(key);
String firstChar = r.substring(0, 1);
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java
index b1c7703f..1b669d7b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java
@@ -30,6 +30,13 @@ import java.util.Map;
import opennlp.tools.tokenize.WhitespaceTokenizer;
+/**
+ * Encapsulates a type to class mapping for entities, relations, events, etc.
+ * <p>
+ * Details on how a annotation configuration file should be structured can be found
+ * in the <a href="https://brat.nlplab.org/configuration.html">brat annotation configuration</a>
+ * section of the official BRAT documentation.
+ */
public class AnnotationConfiguration {
public static final String SPAN_TYPE = "Span";
@@ -38,68 +45,93 @@ public class AnnotationConfiguration {
public static final String ATTRIBUTE_TYPE = "Attribute";
public static final String EVENT_TYPE = "Event";
+ private static final String SYMBOL_HASH = "#";
+ private static final String BRACKET_OPEN = "[";
+ private static final String BRACKET_CLOSE = "]";
+
private final Map<String, String> typeToClassMap;
+ /**
+ * Initializes an {@link AnnotationConfiguration} with the specified {@code typeToClassMap}.
+ * @param typeToClassMap A type to class mapping. Must not be {@code null}.
+ */
public AnnotationConfiguration(Map<String, String> typeToClassMap) {
this.typeToClassMap = Map.copyOf(typeToClassMap);
}
+ /**
+ * @param type The type to get the type class for.
+ * @return Retrieves the class for the specified {@code type}, {@code null} if not found.
+ */
public String getTypeClass(String type) {
return typeToClassMap.get(type);
}
-
+ /**
+ * Parses a given {@link File annConfigFile} into a {@link AnnotationConfiguration}.
+ *
+ * @param in A valid {@link File annConfigFile} from which the config should
+ * be read. Must not be {@code null} and must be in the correct format,
+ * see: <a href="https://brat.nlplab.org/configuration.html">
+ * Brat annotation configuration</a>
+ *
+ * @return A valid {@link AnnotationConfiguration} instance.
+ * @throws IOException Thrown if IO errors occurred during parsing.
+ */
public static AnnotationConfiguration parse(InputStream in) throws IOException {
Map<String, String> typeToClassMap = new HashMap<>();
- BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
-
- // Note: This only supports entities and relations section
- String line;
- String sectionType = null;
-
- while ((line = reader.readLine()) != null) {
- line = line.trim();
-
- if (!line.isEmpty()) {
- if (!line.startsWith("#")) {
- if (line.startsWith("[") && line.endsWith("]")) {
- sectionType = line.substring(line.indexOf('[') + 1, line.indexOf(']'));
- }
- else {
- String typeName = WhitespaceTokenizer.INSTANCE.tokenize(line)[0];
-
- switch (sectionType) {
- case "entities":
- typeToClassMap.put(typeName, AnnotationConfiguration.ENTITY_TYPE);
- break;
-
- case "relations":
- typeToClassMap.put(typeName, AnnotationConfiguration.RELATION_TYPE);
- break;
-
- case "attributes":
- typeToClassMap.put(typeName, AnnotationConfiguration.ATTRIBUTE_TYPE);
- break;
-
- case "events":
- typeToClassMap.put(typeName, AnnotationConfiguration.EVENT_TYPE);
- break;
-
- default:
- break;
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
+ // Note: This only supports entities and relations section
+ String line;
+ String sectionType = null;
+
+ while ((line = reader.readLine()) != null) {
+ line = line.trim();
+
+ if (!line.isEmpty()) {
+ if (!line.startsWith(SYMBOL_HASH)) {
+ if (line.startsWith(BRACKET_OPEN) && line.endsWith(BRACKET_CLOSE)) {
+ sectionType = line.substring(line.indexOf('[') + 1, line.indexOf(']'));
+ } else {
+ String typeName = WhitespaceTokenizer.INSTANCE.tokenize(line)[0];
+
+ switch (sectionType) {
+ case "entities":
+ typeToClassMap.put(typeName, AnnotationConfiguration.ENTITY_TYPE);
+ break;
+ case "relations":
+ typeToClassMap.put(typeName, AnnotationConfiguration.RELATION_TYPE);
+ break;
+ case "attributes":
+ typeToClassMap.put(typeName, AnnotationConfiguration.ATTRIBUTE_TYPE);
+ break;
+ case "events":
+ typeToClassMap.put(typeName, AnnotationConfiguration.EVENT_TYPE);
+ break;
+ default:
+ break;
+ }
}
}
}
}
}
-
return new AnnotationConfiguration(typeToClassMap);
}
+ /**
+ * Parses a given {@link File annConfigFile} into a {@link AnnotationConfiguration}.
+ *
+ * @param annConfigFile A valid {@link File annConfigFile} from which the config should
+ * be read. Must not be {@code null} and must be in the correct format,
+ * see: <a href="https://brat.nlplab.org/configuration.html">
+ * Brat annotation configuration</a>
+ *
+ * @return A valid {@link AnnotationConfiguration} instance.
+ * @throws IOException Thrown if IO errors occurred during parsing.
+ */
public static AnnotationConfiguration parse(File annConfigFile) throws IOException {
- try (InputStream in = new BufferedInputStream(new FileInputStream(annConfigFile))) {
- return parse(in);
- }
+ return parse(new BufferedInputStream(new FileInputStream(annConfigFile)));
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java
index e45797d4..43dc3a25 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java
@@ -199,8 +199,7 @@ public class BratAnnotationStream implements ObjectStream<BratAnnotation> {
@Override
BratAnnotation parse(Span[] tokens, CharSequence line) throws IOException {
-
- Span noteSpan = new Span( tokens[START_VALUE_OFFSET].getStart(),
+ Span noteSpan = new Span( tokens[START_VALUE_OFFSET].getStart(),
tokens[tokens.length - 1].getEnd() );
return new AnnotatorNoteAnnotation(tokens[ID_OFFSET].getCoveredText(line).toString(),
@@ -208,6 +207,7 @@ public class BratAnnotationStream implements ObjectStream<BratAnnotation> {
noteSpan.getCoveredText(line).toString());
}
}
+
private final AnnotationConfiguration config;
private final BufferedReader reader;
private final String id;
@@ -219,10 +219,9 @@ public class BratAnnotationStream implements ObjectStream<BratAnnotation> {
reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
}
+ @Override
public BratAnnotation read() throws IOException {
-
String line = reader.readLine();
-
if (line != null) {
Span[] tokens = WhitespaceTokenizer.INSTANCE.tokenizePos(line);
@@ -259,7 +258,7 @@ public class BratAnnotationStream implements ObjectStream<BratAnnotation> {
}
break;
default:
- // Skip it, do that for everything unsupported (e.g. "*" id)
+ // Skip it, do that for everything unsupported (e.g. "*" id)
return read();
}
@@ -275,10 +274,12 @@ public class BratAnnotationStream implements ObjectStream<BratAnnotation> {
return null;
}
+ @Override
public void reset() throws IOException, UnsupportedOperationException {
reader.reset();
}
+ @Override
public void close() throws IOException {
reader.close();
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratDocument.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratDocument.java
index 8f786749..8f14d54b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratDocument.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratDocument.java
@@ -100,25 +100,22 @@ public class BratDocument {
public static BratDocument parseDocument(AnnotationConfiguration config, String id,
InputStream txtIn, InputStream annIn) throws IOException {
- Reader txtReader = new InputStreamReader(txtIn, StandardCharsets.UTF_8);
-
- StringBuilder text = new StringBuilder();
-
- char[] cbuf = new char[1024];
-
- int len;
- while ((len = txtReader.read(cbuf)) > 0) {
- text.append(cbuf, 0, len);
- }
-
- Collection<BratAnnotation> annotations = new ArrayList<>();
- ObjectStream<BratAnnotation> annStream = new BratAnnotationStream(config, id, annIn);
- BratAnnotation ann;
- while ((ann = annStream.read()) != null) {
- annotations.add(ann);
+ try (Reader txtReader = new InputStreamReader(txtIn, StandardCharsets.UTF_8);
+ ObjectStream<BratAnnotation> annStream = new BratAnnotationStream(config, id, annIn)) {
+
+ StringBuilder text = new StringBuilder();
+ char[] cbuf = new char[1024];
+ int len;
+ while ((len = txtReader.read(cbuf)) > 0) {
+ text.append(cbuf, 0, len);
+ }
+ Collection<BratAnnotation> annotations = new ArrayList<>();
+ BratAnnotation ann;
+ while ((ann = annStream.read()) != null) {
+ annotations.add(ann);
+ }
+ return new BratDocument(config, id, text.toString(), annotations);
}
- annStream.close();
-
- return new BratDocument(config, id, text.toString(), annotations);
}
+
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStream.java
index d799b4c4..feb3a4c3 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStream.java
@@ -38,44 +38,41 @@ public class BratNameSampleStream extends SegmenterObjectStream<BratDocument, Na
private final BratDocumentParser parser;
/**
- * Creates a new {@link BratNameSampleStream}.
- * @param sentDetector a {@link SentenceDetector} instance
- * @param tokenizer a {@link Tokenizer} instance
- * @param samples a {@link BratDocument} {@link ObjectStream}
+ * Initializes a new {@link BratNameSampleStream} with the specified parameters
+ * based on a straem of {@link BratDocument documents}.
+ *
+ * @param sentDetector A valid {@link SentenceDetector} instance.
+ * @param tokenizer A valid {@link Tokenizer} instance.
+ * @param samples The {@link BratDocument} {@link ObjectStream} to process.
*/
public BratNameSampleStream(SentenceDetector sentDetector,
Tokenizer tokenizer, ObjectStream<BratDocument> samples) {
- super(samples);
-
- this.parser = new BratDocumentParser(sentDetector, tokenizer, null);
+ this(sentDetector, tokenizer, samples, null);
}
/**
- * Creates a new {@link BratNameSampleStream}.
- * @param sentModel a {@link SentenceModel} model
- * @param tokenModel a {@link TokenizerModel} model
- * @param samples a {@link BratDocument} {@link ObjectStream}
+ * Initializes a new {@link BratNameSampleStream} with the specified (model) parameters.
+ *
+ * @param sentModel A valid {@link SentenceModel sentence detection model}.
+ * @param tokenModel A valid {@link TokenizerModel tokenizer model}.
+ * @param samples The {@link BratDocument} {@link ObjectStream} to process.
*/
public BratNameSampleStream(SentenceModel sentModel, TokenizerModel tokenModel,
ObjectStream<BratDocument> samples) {
- super(samples);
-
- // TODO: We can pass in custom validators here ...
- this.parser = new BratDocumentParser(new SentenceDetectorME(sentModel),
- new TokenizerME(tokenModel), null);
+ this(new SentenceDetectorME(sentModel), new TokenizerME(tokenModel), samples, null);
}
/**
* Creates a new {@link BratNameSampleStream}.
- * @param sentDetector a {@link SentenceDetector} instance
- * @param tokenizer a {@link Tokenizer} instance
- * @param samples a {@link BratDocument} {@link ObjectStream}
- * @param nameTypes the name types to use or null if all name types
+ * @param sentDetector A valid {@link SentenceDetector} instance.
+ * @param tokenizer A valid {@link Tokenizer} instance.
+ * @param samples The {@link BratDocument} {@link ObjectStream} to process.
+ *
+ * @param nameTypes the name types to use or {@code null} if all name types.
*/
public BratNameSampleStream(SentenceDetector sentDetector,
Tokenizer tokenizer, ObjectStream<BratDocument> samples, Set<String> nameTypes) {
super(samples);
-
this.parser = new BratDocumentParser(sentDetector, tokenizer, nameTypes);
}
@@ -84,13 +81,12 @@ public class BratNameSampleStream extends SegmenterObjectStream<BratDocument, Na
* @param sentModel a {@link SentenceModel} model
* @param tokenModel a {@link TokenizerModel} model
* @param samples a {@link BratDocument} {@link ObjectStream}
- * @param nameTypes the name types to use or null if all name types
+ * @param nameTypes the name types to use or {@code null} if all name types
*/
public BratNameSampleStream(SentenceModel sentModel, TokenizerModel tokenModel,
ObjectStream<BratDocument> samples, Set<String> nameTypes) {
super(samples);
-
- // TODO: We can pass in custom validators here ...
+ // Hint: We can pass in custom validators here ...
this.parser = new BratDocumentParser(new SentenceDetectorME(sentModel),
new TokenizerME(tokenModel), nameTypes);
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStreamFactory.java
index d5e8793e..9017c944 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStreamFactory.java
@@ -41,10 +41,15 @@ import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.tokenize.WhitespaceTokenizer;
import opennlp.tools.util.ObjectStream;
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see BratNameSampleStream
+ */
public class BratNameSampleStreamFactory
extends AbstractSampleStreamFactory<NameSample, BratNameSampleStreamFactory.Parameters> {
- interface Parameters {
+ public interface Parameters {
@ParameterDescription(valueName = "bratDataDir", description = "location of brat data dir")
File getBratDataDir();
@@ -76,37 +81,40 @@ public class BratNameSampleStreamFactory
super(Parameters.class);
}
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(NameSample.class, "brat",
+ new BratNameSampleStreamFactory());
+ }
+
/**
- * Checks that non of the passed values are null.
+ * Checks that non of the passed values are {@code null}.
*
- * @param objects
- * @return true or false
+ * @param objects The objects to check for {@code null}.
+ * @return {@code true} if at least one object is {@code null}, {@code false} otherwise.
*/
private boolean notNull(Object... objects) {
-
for (Object obj : objects) {
if (obj == null)
return false;
}
-
return true;
}
@Override
public ObjectStream<NameSample> create(String[] args) {
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
Parameters params = ArgumentParser.parse(args, Parameters.class);
if (notNull(params.getRuleBasedTokenizer(), params.getTokenizerModel())) {
throw new TerminateToolException(-1, "Either use rule based or statistical tokenizer!");
}
- // TODO: Provide the file name to the annotation.conf file and implement the parser ...
AnnotationConfiguration annConfig;
try {
annConfig = AnnotationConfiguration.parse(params.getAnnotationConfig());
- }
- catch (IOException e) {
+ } catch (IOException e) {
throw new TerminateToolException(1, "Failed to parse annotation.conf file!");
}
@@ -121,37 +129,30 @@ public class BratNameSampleStreamFactory
}
SentenceDetector sentDetector;
-
if (params.getSentenceDetectorModel() != null) {
try {
sentDetector = new SentenceDetectorME(new SentenceModel(params.getSentenceDetectorModel()));
} catch (IOException e) {
throw new TerminateToolException(-1, "Failed to load sentence detector model!", e);
}
- }
- else {
+ } else {
sentDetector = new NewlineSentenceDetector();
}
Tokenizer tokenizer = WhitespaceTokenizer.INSTANCE;
-
if (params.getTokenizerModel() != null) {
try {
tokenizer = new TokenizerME(new TokenizerModel(params.getTokenizerModel()));
} catch (IOException e) {
throw new TerminateToolException(-1, "Failed to load tokenizer model!", e);
}
- }
- else if (params.getRuleBasedTokenizer() != null) {
+ } else if (params.getRuleBasedTokenizer() != null) {
String tokenizerName = params.getRuleBasedTokenizer();
-
if ("simple".equals(tokenizerName)) {
tokenizer = SimpleTokenizer.INSTANCE;
- }
- else if ("whitespace".equals(tokenizerName)) {
+ } else if ("whitespace".equals(tokenizerName)) {
tokenizer = WhitespaceTokenizer.INSTANCE;
- }
- else {
+ } else {
throw new TerminateToolException(-1, "Unknown tokenizer: " + tokenizerName);
}
}
@@ -167,8 +168,4 @@ public class BratNameSampleStreamFactory
return new BratNameSampleStream(sentDetector, tokenizer, samples, nameTypes);
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(NameSample.class, "brat",
- new BratNameSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/SegmenterObjectStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/SegmenterObjectStream.java
index 82c4e4aa..05ee3e3d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/SegmenterObjectStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/SegmenterObjectStream.java
@@ -35,6 +35,7 @@ public abstract class SegmenterObjectStream<S, T> extends FilterObjectStream<S,
protected abstract List<T> read(S sample) throws IOException;
+ @Override
public final T read() throws IOException {
if (sampleIt.hasNext()) {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactory.java
index c7b1f77b..2dedf86c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactory.java
@@ -32,11 +32,15 @@ import opennlp.tools.util.ObjectStream;
/**
* <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see LemmaSample
+ * @see ConlluLemmaSampleStream
*/
@Internal
-public class ConlluLemmaSampleStreamFactory<P> extends AbstractSampleStreamFactory<LemmaSample, P> {
+public class ConlluLemmaSampleStreamFactory extends
+ AbstractSampleStreamFactory<LemmaSample, ConlluLemmaSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ArgumentParser.ParameterDescription(valueName = "tagset",
description = "u|x u for unified tags and x for language-specific part-of-speech tags")
@ArgumentParser.OptionalParameter(defaultValue = "u")
@@ -46,15 +50,15 @@ public class ConlluLemmaSampleStreamFactory<P> extends AbstractSampleStreamFacto
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(LemmaSample.class,
ConlluPOSSampleStreamFactory.CONLLU_FORMAT,
- new ConlluLemmaSampleStreamFactory<>(Parameters.class));
+ new ConlluLemmaSampleStreamFactory(Parameters.class));
}
- protected ConlluLemmaSampleStreamFactory(Class<P> params) {
+ protected ConlluLemmaSampleStreamFactory(Class<Parameters> params) {
super(params);
}
public ObjectStream<LemmaSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ConlluTagset tagset = switch (params.getTagset()) {
case "u" -> ConlluTagset.U;
@@ -68,7 +72,6 @@ public class ConlluLemmaSampleStreamFactory<P> extends AbstractSampleStreamFacto
try {
return new ConlluLemmaSampleStream(new ConlluStream(inFactory), tagset);
} catch (IOException e) {
- // That will throw an exception
CmdLineUtil.handleCreateObjectStreamError(e);
}
return null;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactory.java
index 6601dbb6..9a40b0a1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactory.java
@@ -32,13 +32,17 @@ import opennlp.tools.util.ObjectStream;
/**
* <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see POSSample
+ * @see ConlluPOSSampleStream
*/
@Internal
-public class ConlluPOSSampleStreamFactory<P> extends AbstractSampleStreamFactory<POSSample, P> {
+public class ConlluPOSSampleStreamFactory extends
+ AbstractSampleStreamFactory<POSSample, ConlluPOSSampleStreamFactory.Parameters> {
public static final String CONLLU_FORMAT = "conllu";
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ArgumentParser.ParameterDescription(valueName = "tagset",
description = "u|x u for unified tags and x for language-specific part-of-speech tags")
@ArgumentParser.OptionalParameter(defaultValue = "u")
@@ -47,15 +51,16 @@ public class ConlluPOSSampleStreamFactory<P> extends AbstractSampleStreamFactory
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(POSSample.class,
- CONLLU_FORMAT, new ConlluPOSSampleStreamFactory<>(Parameters.class));
+ CONLLU_FORMAT, new ConlluPOSSampleStreamFactory(Parameters.class));
}
- protected ConlluPOSSampleStreamFactory(Class<P> params) {
+ protected ConlluPOSSampleStreamFactory(Class<Parameters> params) {
super(params);
}
+ @Override
public ObjectStream<POSSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ConlluTagset tagset = switch (params.getTagset()) {
case "u" -> ConlluTagset.U;
@@ -69,7 +74,6 @@ public class ConlluPOSSampleStreamFactory<P> extends AbstractSampleStreamFactory
try {
return new ConlluPOSSampleStream(new ConlluStream(inFactory), tagset);
} catch (IOException e) {
- // That will throw an exception
CmdLineUtil.handleCreateObjectStreamError(e);
}
return null;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentence.java b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentence.java
index 695534d1..2150be30 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentence.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentence.java
@@ -44,9 +44,7 @@ public class ConlluSentence {
public ConlluSentence(List<ConlluWordLine> wordLines, String sentenceIdComment, String textComment,
boolean newDocument, String documentId, boolean newParagraph, String paragraphId,
Map<Locale, String> textLang, String translit) {
- this.wordLines = wordLines;
- this.sentenceIdComment = sentenceIdComment;
- this.textComment = textComment;
+ this(wordLines, sentenceIdComment, textComment);
this.newDocument = newDocument;
this.documentId = documentId;
this.newParagraph = newParagraph;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactory.java
index 3b1164ca..6e665759 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactory.java
@@ -30,15 +30,16 @@ import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
/**
- * <b>Note:</b>
- * Do not use this class, internal use only!
+ * <b>Note:</b> Do not use this class, internal use only!
*
+ * @see SentenceSample
* @see ConlluSentenceSampleStream
*/
@Internal
-public class ConlluSentenceSampleStreamFactory<P> extends AbstractSampleStreamFactory<SentenceSample, P> {
+public class ConlluSentenceSampleStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, ConlluSentenceSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ArgumentParser.ParameterDescription(valueName = "sentencesPerSample",
description = "number of sentences per sample")
String getSentencesPerSample();
@@ -47,16 +48,16 @@ public class ConlluSentenceSampleStreamFactory<P> extends AbstractSampleStreamFa
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
ConlluPOSSampleStreamFactory.CONLLU_FORMAT,
- new ConlluSentenceSampleStreamFactory<>(ConlluSentenceSampleStreamFactory.Parameters.class));
+ new ConlluSentenceSampleStreamFactory(ConlluSentenceSampleStreamFactory.Parameters.class));
}
- protected ConlluSentenceSampleStreamFactory(Class<P> params) {
+ protected ConlluSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
InputStreamFactory inFactory =
CmdLineUtil.createInputStreamFactory(params.getData());
@@ -65,7 +66,6 @@ public class ConlluSentenceSampleStreamFactory<P> extends AbstractSampleStreamFa
return new ConlluSentenceSampleStream(new ConlluStream(inFactory),
Integer.parseInt(params.getSentencesPerSample()));
} catch (IOException e) {
- // That will throw an exception
CmdLineUtil.handleCreateObjectStreamError(e);
}
return null;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactory.java
index 5f813a65..ac3b18d4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactory.java
@@ -19,7 +19,6 @@ package opennlp.tools.formats.conllu;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
@@ -30,30 +29,31 @@ import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
/**
- * <b>Note:</b>
- * Do not use this class, internal use only!
+ * <b>Note:</b> Do not use this class, internal use only!
*
+ * @see TokenSample
* @see ConlluTokenSampleStream
*/
@Internal
-public class ConlluTokenSampleStreamFactory<P> extends AbstractSampleStreamFactory<TokenSample, P> {
+public class ConlluTokenSampleStreamFactory extends
+ AbstractSampleStreamFactory<TokenSample, ConlluTokenSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class,
ConlluPOSSampleStreamFactory.CONLLU_FORMAT,
- new ConlluTokenSampleStreamFactory<>(ConlluTokenSampleStreamFactory.Parameters.class));
+ new ConlluTokenSampleStreamFactory(ConlluTokenSampleStreamFactory.Parameters.class));
}
- protected ConlluTokenSampleStreamFactory(Class<P> params) {
+ protected ConlluTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
InputStreamFactory inFactory =
CmdLineUtil.createInputStreamFactory(params.getData());
@@ -61,7 +61,6 @@ public class ConlluTokenSampleStreamFactory<P> extends AbstractSampleStreamFacto
try {
return new ConlluTokenSampleStream(new ConlluStream(inFactory));
} catch (IOException e) {
- // That will throw an exception
CmdLineUtil.handleCreateObjectStreamError(e);
}
return null;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactory.java
index ad5354e4..7297126b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactory.java
@@ -34,23 +34,24 @@ import opennlp.tools.util.ObjectStream;
* @see NameToSentenceSampleStream
*/
@Internal
-public class NameToSentenceSampleStreamFactory<P> extends DetokenizerSampleStreamFactory<SentenceSample, P> {
+public class NameToSentenceSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<SentenceSample, NameToSentenceSampleStreamFactory.Parameters> {
interface Parameters extends NameSampleDataStreamFactory.Parameters, DetokenizerParameter {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
- "namefinder", new NameToSentenceSampleStreamFactory<>(Parameters.class));
+ "namefinder", new NameToSentenceSampleStreamFactory(Parameters.class));
}
- protected NameToSentenceSampleStreamFactory(Class<P> params) {
+ protected NameToSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<NameSample> nameSampleStream = StreamFactoryRegistry.getFactory(
NameSample.class, StreamFactoryRegistry.DEFAULT_FORMAT).create(
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactory.java
index 4d2c10bc..54392331 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactory.java
@@ -34,23 +34,24 @@ import opennlp.tools.util.ObjectStream;
* @see NameToTokenSampleStream
*/
@Internal
-public class NameToTokenSampleStreamFactory<P> extends DetokenizerSampleStreamFactory<TokenSample, P> {
+public class NameToTokenSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<TokenSample, NameToTokenSampleStreamFactory.Parameters> {
- interface Parameters extends NameSampleDataStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends NameSampleDataStreamFactory.Parameters, DetokenizerParameter {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class,
- "namefinder", new NameToTokenSampleStreamFactory<>(Parameters.class));
+ "namefinder", new NameToTokenSampleStreamFactory(Parameters.class));
}
- protected NameToTokenSampleStreamFactory(Class<P> params) {
+ protected NameToTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<NameSample> nameSampleStream = StreamFactoryRegistry.getFactory(
NameSample.class, StreamFactoryRegistry.DEFAULT_FORMAT).create(
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactory.java
index 25162683..8aaf104f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactory.java
@@ -34,23 +34,24 @@ import opennlp.tools.util.ObjectStream;
* @see POSToSentenceSampleStream
*/
@Internal
-public class POSToSentenceSampleStreamFactory<P> extends DetokenizerSampleStreamFactory<SentenceSample, P> {
+public class POSToSentenceSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<SentenceSample, POSToSentenceSampleStreamFactory.Parameters> {
- interface Parameters extends WordTagSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends WordTagSampleStreamFactory.Parameters, DetokenizerParameter {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
- "pos", new POSToSentenceSampleStreamFactory<>(Parameters.class));
+ "pos", new POSToSentenceSampleStreamFactory(Parameters.class));
}
- protected POSToSentenceSampleStreamFactory(Class<P> params) {
+ protected POSToSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<POSSample> posSampleStream = StreamFactoryRegistry.getFactory(POSSample.class,
StreamFactoryRegistry.DEFAULT_FORMAT).create(
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactory.java
index e5750a33..5f7eb51d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactory.java
@@ -34,23 +34,24 @@ import opennlp.tools.util.ObjectStream;
* @see POSToTokenSampleStream
*/
@Internal
-public class POSToTokenSampleStreamFactory<P> extends DetokenizerSampleStreamFactory<TokenSample, P> {
+public class POSToTokenSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<TokenSample, POSToTokenSampleStreamFactory.Parameters> {
- interface Parameters extends WordTagSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends WordTagSampleStreamFactory.Parameters, DetokenizerParameter {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class,
- "pos", new POSToTokenSampleStreamFactory<>(Parameters.class));
+ "pos", new POSToTokenSampleStreamFactory(Parameters.class));
}
- protected POSToTokenSampleStreamFactory(Class<P> params) {
+ protected POSToTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<POSSample> posSampleStream = StreamFactoryRegistry.getFactory(POSSample.class,
StreamFactoryRegistry.DEFAULT_FORMAT).create(
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactory.java
index 73f12a3c..eb9855e9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactory.java
@@ -19,6 +19,7 @@ package opennlp.tools.formats.convert;
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.params.DetokenizerParameter;
import opennlp.tools.commons.Internal;
import opennlp.tools.formats.LanguageSampleStreamFactory;
import opennlp.tools.formats.ParseSampleStreamFactory;
@@ -34,17 +35,25 @@ import opennlp.tools.util.ObjectStream;
*/
@Internal
public class ParseToPOSSampleStreamFactory
- extends LanguageSampleStreamFactory<POSSample, ParseSampleStreamFactory.Parameters> {
+ extends LanguageSampleStreamFactory<POSSample, ParseToPOSSampleStreamFactory.Parameters> {
+ public interface Parameters extends ParseSampleStreamFactory.Parameters, DetokenizerParameter {
+ }
+
private ParseToPOSSampleStreamFactory() {
- super(ParseSampleStreamFactory.Parameters.class);
+ super(ParseToPOSSampleStreamFactory.Parameters.class);
+ }
+
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(POSSample.class,
+ "parse", new ParseToPOSSampleStreamFactory());
}
@Override
public ObjectStream<POSSample> create(String[] args) {
ParseSampleStreamFactory.Parameters params =
- ArgumentParser.parse(args, ParseSampleStreamFactory.Parameters.class);
+ validateBasicFormatParameters(args, ParseSampleStreamFactory.Parameters.class);
ObjectStream<Parse> parseSampleStream = StreamFactoryRegistry.getFactory(Parse.class,
StreamFactoryRegistry.DEFAULT_FORMAT).create(
@@ -53,8 +62,4 @@ public class ParseToPOSSampleStreamFactory
return new ParseToPOSSampleStream(parseSampleStream);
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(POSSample.class,
- "parse", new ParseToPOSSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactory.java
index cfa253b7..15648a90 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactory.java
@@ -32,19 +32,24 @@ import opennlp.tools.util.ObjectStream;
* Do not use this class, internal use only!
*/
@Internal
-public class ParseToSentenceSampleStreamFactory extends DetokenizerSampleStreamFactory
- <SentenceSample, ParseToSentenceSampleStreamFactory.Parameters> {
+public class ParseToSentenceSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<SentenceSample, ParseToSentenceSampleStreamFactory.Parameters> {
- interface Parameters extends ParseSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends ParseSampleStreamFactory.Parameters, DetokenizerParameter {
}
private ParseToSentenceSampleStreamFactory() {
super(Parameters.class);
}
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(SentenceSample.class,
+ "parse", new ParseToSentenceSampleStreamFactory());
+ }
+
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<Parse> parseSampleStream = StreamFactoryRegistry.getFactory(Parse.class,
StreamFactoryRegistry.DEFAULT_FORMAT).create(
@@ -54,8 +59,5 @@ public class ParseToSentenceSampleStreamFactory extends DetokenizerSampleStreamF
new ParseToPOSSampleStream(parseSampleStream), 30);
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(SentenceSample.class,
- "parse", new ParseToSentenceSampleStreamFactory());
- }
+
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java
index 1a39c2c1..4d0d0728 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java
@@ -33,19 +33,24 @@ import opennlp.tools.util.ObjectStream;
* Do not use this class, internal use only!
*/
@Internal
-public class ParseToTokenSampleStreamFactory extends DetokenizerSampleStreamFactory<TokenSample,
- ParseToTokenSampleStreamFactory.Parameters> {
+public class ParseToTokenSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<TokenSample, ParseToTokenSampleStreamFactory.Parameters> {
- interface Parameters extends ParseSampleStreamFactory.Parameters, DetokenizerParameter {
+ public interface Parameters extends ParseSampleStreamFactory.Parameters, DetokenizerParameter {
}
private ParseToTokenSampleStreamFactory() {
super(Parameters.class);
}
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(TokenSample.class,
+ "parse", new ParseToTokenSampleStreamFactory());
+ }
+
@Override
public ObjectStream<TokenSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
ObjectStream<Parse> parseSampleStream = StreamFactoryRegistry.getFactory(Parse.class,
StreamFactoryRegistry.DEFAULT_FORMAT).create(
@@ -55,8 +60,4 @@ public class ParseToTokenSampleStreamFactory extends DetokenizerSampleStreamFact
new ParseToPOSSampleStream(parseSampleStream));
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(TokenSample.class,
- "parse", new ParseToTokenSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java
index a724d77b..f2f57f94 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java
@@ -34,7 +34,6 @@ class ConstitDocumentHandler extends DefaultHandler {
private static final String SENT_ELEMENT_NAME = "SENT";
private static final String WORD_ELEMENT_NAME = "w";
-
private static final String SENT_TYPE_NAME = "S";
private final List<Parse> parses;
@@ -139,7 +138,7 @@ class ConstitDocumentHandler extends DefaultHandler {
if (WORD_ELEMENT_NAME.equals(qName)) {
String token = tokenBuffer.toString().trim();
- if (token.length() > 0) {
+ if (!token.isEmpty()) {
cons.add(new Constituent(AbstractBottomUpParser.TOK_NODE,
new Span(offset, offset + token.length())));
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactory.java
index 11cd0108..d68b9be6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactory.java
@@ -28,9 +28,9 @@ import opennlp.tools.parser.Parse;
import opennlp.tools.util.ObjectStream;
/**
- * <b>Note:</b>
- * Do not use this class, internal use only!
+ * <b>Note:</b> Do not use this class, internal use only!
*
+ * @see Parse
* @see ConstitParseSampleStream
*/
@Internal
@@ -45,18 +45,19 @@ public class ConstitParseSampleStreamFactory
super(Parameters.class);
}
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(Parse.class, "frenchtreebank",
+ new ConstitParseSampleStreamFactory());
+ }
+
@Override
public ObjectStream<Parse> create(String[] args) {
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
Parameters params = ArgumentParser.parse(args, Parameters.class);
-
-
return new ConstitParseSampleStream(new FileToByteArraySampleStream(
new DirectorySampleStream(params.getData(), null, false)));
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(Parse.class, "frenchtreebank",
- new ConstitParseSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactory.java
index 3decd8e0..5c1036f1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactory.java
@@ -19,39 +19,40 @@ package opennlp.tools.formats.irishsentencebank;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.util.ObjectStream;
/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see SentenceSample
* @see IrishSentenceBankSentenceStream
*/
-public class IrishSentenceBankSentenceStreamFactory<P>
- extends AbstractSampleStreamFactory<SentenceSample, P> {
+@Internal
+public class IrishSentenceBankSentenceStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, IrishSentenceBankSentenceStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
- "irishsentencebank", new IrishSentenceBankSentenceStreamFactory<>(
+ "irishsentencebank", new IrishSentenceBankSentenceStreamFactory(
IrishSentenceBankSentenceStreamFactory.Parameters.class));
}
- protected IrishSentenceBankSentenceStreamFactory(Class<P> params) {
+ protected IrishSentenceBankSentenceStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
IrishSentenceBankDocument isbDoc = null;
try {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactory.java
index 213c0b8a..b41d5741 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactory.java
@@ -19,39 +19,40 @@ package opennlp.tools.formats.irishsentencebank;
import java.io.IOException;
-import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.DetokenizerSampleStreamFactory;
import opennlp.tools.tokenize.TokenSample;
import opennlp.tools.util.ObjectStream;
/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see TokenSample
* @see IrishSentenceBankTokenSampleStream
*/
-public class IrishSentenceBankTokenSampleStreamFactory<P>
- extends DetokenizerSampleStreamFactory<TokenSample, P> {
+@Internal
+public class IrishSentenceBankTokenSampleStreamFactory extends
+ DetokenizerSampleStreamFactory<TokenSample, IrishSentenceBankTokenSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class,
- "irishsentencebank", new IrishSentenceBankTokenSampleStreamFactory<>(
+ "irishsentencebank", new IrishSentenceBankTokenSampleStreamFactory(
IrishSentenceBankTokenSampleStreamFactory.Parameters.class));
}
- protected IrishSentenceBankTokenSampleStreamFactory(Class<P> params) {
+ protected IrishSentenceBankTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<TokenSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
IrishSentenceBankDocument isbDoc = null;
try {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactory.java
index 4b3b769d..789904f0 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactory.java
@@ -38,49 +38,49 @@ import opennlp.tools.util.ObjectStream;
* @see LeipzigLanguageSampleStream
*/
@Internal
-public class LeipzigLanguageSampleStreamFactory<P>
- extends AbstractSampleStreamFactory<LanguageSample, P> {
+public class LeipzigLanguageSampleStreamFactory extends
+ AbstractSampleStreamFactory<LanguageSample, LeipzigLanguageSampleStreamFactory.Parameters> {
- interface Parameters extends EncodingParameter {
+ public interface Parameters extends EncodingParameter {
@ParameterDescription(valueName = "sentencesDir",
description = "dir with Leipzig sentences to be used")
File getSentencesDir();
@ParameterDescription(valueName = "sentencesPerSample",
description = "number of sentences per sample")
- String getSentencesPerSample();
+ Integer getSentencesPerSample();
@ParameterDescription(valueName = "samplesPerLanguage",
description = "number of samples per language")
- String getSamplesPerLanguage();
+ Integer getSamplesPerLanguage();
@ParameterDescription(valueName = "samplesToSkip",
description = "number of samples to skip before returning")
@OptionalParameter(defaultValue = "0")
- String getSamplesToSkip();
+ Integer getSamplesToSkip();
}
- protected LeipzigLanguageSampleStreamFactory(Class<P> params) {
+ protected LeipzigLanguageSampleStreamFactory(Class<Parameters> params) {
super(params);
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(LanguageSample.class,
- "leipzig", new LeipzigLanguageSampleStreamFactory<>(Parameters.class));
+ "leipzig", new LeipzigLanguageSampleStreamFactory(Parameters.class));
}
@Override
public ObjectStream<LanguageSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
- File sentencesFileDir = params.getSentencesDir();
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ Parameters p = ArgumentParser.parse(args, Parameters.class);
+ File sentencesFileDir = p.getSentencesDir();
try {
- return new SampleSkipStream<>(new SampleShuffleStream<>(
- new LeipzigLanguageSampleStream(sentencesFileDir,
- Integer.parseInt(params.getSentencesPerSample()),
- Integer.parseInt(params.getSamplesPerLanguage()) + Integer.parseInt(params.getSamplesToSkip()))),
- Integer.parseInt(params.getSamplesToSkip()));
+ return new SampleSkipStream<>(new SampleShuffleStream<>(new LeipzigLanguageSampleStream(
+ sentencesFileDir, p.getSentencesPerSample(),
+ p.getSamplesPerLanguage() + p.getSamplesToSkip())), p.getSamplesToSkip());
} catch (IOException e) {
throw new TerminateToolException(-1, "IO error while opening sample data.", e);
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleShuffleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleShuffleStream.java
index 417d202e..1e3d5e69 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleShuffleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleShuffleStream.java
@@ -24,33 +24,40 @@ import java.util.Iterator;
import java.util.List;
import java.util.Random;
+import opennlp.tools.commons.Sample;
import opennlp.tools.util.ObjectStream;
+/**
+ * A specialization of {@link ObjectStream} that shuffles samples.
+ * @param <T> The template parameter which represents
+ * the {@link Sample} type.
+ */
class SampleShuffleStream<T> implements ObjectStream<T> {
private final List<T> bufferedSamples = new ArrayList<>();
private Iterator<T> sampleIt;
+ /**
+ * Initializes a {@link SampleShuffleStream} with the specified parameters.
+ *
+ * @param samples The {@link ObjectStream} to process.
+ * @throws IOException Thrown if IO errors occurred during skip operation.
+ */
SampleShuffleStream(ObjectStream<T> samples) throws IOException {
-
T sample;
while ((sample = samples.read()) != null) {
bufferedSamples.add(sample);
}
-
Collections.shuffle(bufferedSamples, new Random(23));
-
reset();
}
@Override
public T read() throws IOException {
-
if (sampleIt.hasNext()) {
return sampleIt.next();
}
-
return null;
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleSkipStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleSkipStream.java
index 2bfff5b8..e5f72e03 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleSkipStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/leipzig/SampleSkipStream.java
@@ -19,13 +19,26 @@ package opennlp.tools.formats.leipzig;
import java.io.IOException;
+import opennlp.tools.commons.Sample;
import opennlp.tools.util.ObjectStream;
+/**
+ * A specialization of {@link ObjectStream} that skips a number of samples.
+ * @param <T> The template parameter which represents
+ * the {@link Sample} type.
+ */
class SampleSkipStream<T> implements ObjectStream<T> {
private final ObjectStream<T> samples;
private final int samplesToSkip;
+ /**
+ * Initializes a {@link SampleSkipStream} with the specified parameters.
+ *
+ * @param samples The {@link ObjectStream} to process.
+ * @param samplesToSkip The number of samples to skip. Must be greater than {@code 0}.
+ * @throws IOException Thrown if IO errors occurred during skip operation.
+ */
SampleSkipStream(ObjectStream<T> samples, int samplesToSkip) throws IOException {
this.samples = samples;
this.samplesToSkip = samplesToSkip;
@@ -46,7 +59,6 @@ class SampleSkipStream<T> implements ObjectStream<T> {
private void skipSamples() throws IOException {
int i = 0;
-
while (i < samplesToSkip && (samples.read()) != null) {
i++;
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactory.java
index e3474257..1077ab37 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactory.java
@@ -25,6 +25,7 @@ import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.tokenize.DetokenizationDictionary;
@@ -33,11 +34,16 @@ import opennlp.tools.tokenize.DictionaryDetokenizer;
import opennlp.tools.util.ObjectStream;
/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see SentenceSample
* @see LetsmtSentenceStream
*/
-public class LetsmtSentenceStreamFactory<P> extends AbstractSampleStreamFactory<SentenceSample, P> {
+@Internal
+public class LetsmtSentenceStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, LetsmtSentenceStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ArgumentParser.ParameterDescription(valueName = "dictionary",
description = "specifies the file with detokenizer dictionary.")
@ArgumentParser.OptionalParameter
@@ -45,21 +51,17 @@ public class LetsmtSentenceStreamFactory<P> extends AbstractSampleStreamFactory<
}
public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(SentenceSample.class,
- "letsmt", new LetsmtSentenceStreamFactory<>(
- LetsmtSentenceStreamFactory.Parameters.class));
+ StreamFactoryRegistry.registerFactory(SentenceSample.class, "letsmt",
+ new LetsmtSentenceStreamFactory(LetsmtSentenceStreamFactory.Parameters.class));
}
- protected LetsmtSentenceStreamFactory(Class<P> params) {
+ protected LetsmtSentenceStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
LetsmtDocument letsmtDoc = null;
try {
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/Masc.java
similarity index 71%
copy from opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java
copy to opennlp-tools/src/main/java/opennlp/tools/formats/masc/Masc.java
index c154f21d..8b79b83d 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/Masc.java
@@ -15,14 +15,13 @@
* limitations under the License.
*/
-package opennlp.tools.formats;
+package opennlp.tools.formats.masc;
-import opennlp.tools.util.InputStreamFactory;
-
-
-public abstract class AbstractSampleStreamTest extends AbstractFormatTest {
+/**
+ * A simple marker interface for classes that support or refer to
+ * the {@link #MASC_FORMAT}.
+ */
+public interface Masc {
- protected InputStreamFactory getFactory(String resource) {
- return new ResourceAsStreamFactory(AbstractSampleStreamTest.class, FORMATS_BASE_DIR + resource);
- }
+ String MASC_FORMAT = "masc";
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascDocument.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascDocument.java
index e4e1177c..7af48a76 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascDocument.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascDocument.java
@@ -40,7 +40,6 @@ import org.xml.sax.SAXException;
import opennlp.tools.util.Span;
import opennlp.tools.util.XmlUtil;
-
public class MascDocument {
private static final Logger logger = LoggerFactory.getLogger(MascDocument.class);
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactory.java
index 1adf3535..ab85f88d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactory.java
@@ -24,29 +24,49 @@ import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.namefind.NameSample;
import opennlp.tools.util.ObjectStream;
-public class MascNamedEntitySampleStreamFactory<P> extends AbstractSampleStreamFactory<NameSample, P> {
- public static final String MASC_FORMAT = "masc";
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see NameSample
+ * @see MascPOSSampleStream
+ */
+@Internal
+public class MascNamedEntitySampleStreamFactory extends
+ AbstractSampleStreamFactory<NameSample, MascNamedEntitySampleStreamFactory.Parameters>
+ implements Masc {
+
+ public interface Parameters extends BasicFormatParams {
- protected MascNamedEntitySampleStreamFactory(Class<P> params) {
+ @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
+ description = "search through files recursively")
+ Boolean getRecurrentSearch();
+
+ @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
+ description = "only include files which contain a given string in their name")
+ String getFileFilter();
+
+ }
+
+ protected MascNamedEntitySampleStreamFactory(Class<Parameters> params) {
super(params);
}
public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(NameSample.class,
- MASC_FORMAT,
- new MascNamedEntitySampleStreamFactory<>(
- MascNamedEntitySampleStreamFactory.Parameters.class));
+ StreamFactoryRegistry.registerFactory(NameSample.class, MASC_FORMAT,
+ new MascNamedEntitySampleStreamFactory(MascNamedEntitySampleStreamFactory.Parameters.class));
}
@Override
public ObjectStream<NameSample> create(String[] args) {
- MascNamedEntitySampleStreamFactory.Parameters params =
- ArgumentParser.parse(args, MascNamedEntitySampleStreamFactory.Parameters.class);
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ Parameters params = ArgumentParser.parse(args, Parameters.class);
try {
FileFilter fileFilter = pathname -> pathname.getName().contains(params.getFileFilter());
@@ -57,17 +77,4 @@ public class MascNamedEntitySampleStreamFactory<P> extends AbstractSampleStreamF
}
return null;
}
-
- interface Parameters extends BasicFormatParams {
-
- @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
- description = "search through files recursively")
- boolean getRecurrentSearch();
-
- @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
- description = "only include files which contain a given string in their name")
- String getFileFilter();
-
- }
-
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactory.java
index 4e910c68..b3bcedde 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactory.java
@@ -24,29 +24,49 @@ import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.postag.POSSample;
import opennlp.tools.util.ObjectStream;
-public class MascPOSSampleStreamFactory<P> extends AbstractSampleStreamFactory<POSSample, P> {
- public static final String MASC_FORMAT = "masc";
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see POSSample
+ * @see MascPOSSampleStream
+ */
+@Internal
+public class MascPOSSampleStreamFactory extends
+ AbstractSampleStreamFactory<POSSample, MascPOSSampleStreamFactory.Parameters>
+ implements Masc {
+
+ public interface Parameters extends BasicFormatParams {
+
+ @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
+ description = "search through files recursively")
+ Boolean getRecurrentSearch();
+
+ @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
+ description = "only include files which contain a given string in their name")
+ String getFileFilter();
+
+ }
- protected MascPOSSampleStreamFactory(Class<P> params) {
+ protected MascPOSSampleStreamFactory(Class<Parameters> params) {
super(params);
}
public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(POSSample.class,
- MASC_FORMAT,
- new MascPOSSampleStreamFactory<>(
- MascPOSSampleStreamFactory.Parameters.class));
+ StreamFactoryRegistry.registerFactory(POSSample.class, MASC_FORMAT,
+ new MascPOSSampleStreamFactory(Parameters.class));
}
@Override
public ObjectStream<POSSample> create(String[] args) {
- MascPOSSampleStreamFactory.Parameters params =
- ArgumentParser.parse(args, MascPOSSampleStreamFactory.Parameters.class);
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ Parameters params = ArgumentParser.parse(args, Parameters.class);
try {
FileFilter fileFilter = pathname -> pathname.getName().contains(params.getFileFilter());
@@ -59,17 +79,5 @@ public class MascPOSSampleStreamFactory<P> extends AbstractSampleStreamFactory<P
return null;
}
- interface Parameters extends BasicFormatParams {
-
- @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
- description = "search through files recursively")
- boolean getRecurrentSearch();
-
- @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
- description = "only include files which contain a given string in their name")
- String getFileFilter();
-
- }
-
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentence.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentence.java
index 05d661e8..808bb65c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentence.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentence.java
@@ -18,6 +18,7 @@
package opennlp.tools.formats.masc;
import java.io.IOException;
+import java.io.Serial;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -34,8 +35,9 @@ import opennlp.tools.util.Span;
public class MascSentence extends Span {
- private static final Logger logger = LoggerFactory.getLogger(MascSentence.class);
+ @Serial
private static final long serialVersionUID = 6295507533472650848L;
+ private static final Logger logger = LoggerFactory.getLogger(MascSentence.class);
/**
* A helper class to extract the extract a quark from the corpus file even if it is beyond the
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactory.java
index d44bf780..9b6eeb43 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactory.java
@@ -24,28 +24,52 @@ import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.util.ObjectStream;
-public class MascSentenceSampleStreamFactory<P> extends AbstractSampleStreamFactory<SentenceSample, P> {
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see SentenceSample
+ * @see MascSentenceSampleStream
+ */
+@Internal
+public class MascSentenceSampleStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, MascSentenceSampleStreamFactory.Parameters>
+ implements Masc {
+
+ public interface Parameters extends BasicFormatParams {
+ @ArgumentParser.ParameterDescription(valueName = "sentencesPerSample",
+ description = "number of sentences per sample")
+ String getSentencesPerSample();
+
+ @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
+ description = "search through files recursively")
+ Boolean getRecurrentSearch();
+
+ @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
+ description = "only include files which contain a given string in their name")
+ String getFileFilter();
- public static final String MASC_FORMAT = "masc";
+ }
- protected MascSentenceSampleStreamFactory(Class<P> params) {
+ protected MascSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class, MASC_FORMAT,
- new MascSentenceSampleStreamFactory<>(MascSentenceSampleStreamFactory.Parameters.class));
+ new MascSentenceSampleStreamFactory(Parameters.class));
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- MascSentenceSampleStreamFactory.Parameters params =
- ArgumentParser.parse(args, MascSentenceSampleStreamFactory.Parameters.class);
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ Parameters params = ArgumentParser.parse(args, Parameters.class);
try {
FileFilter fileFilter = pathname -> pathname.getName().contains(params.getFileFilter());
@@ -59,20 +83,5 @@ public class MascSentenceSampleStreamFactory<P> extends AbstractSampleStreamFact
return null;
}
- interface Parameters extends BasicFormatParams {
- @ArgumentParser.ParameterDescription(valueName = "sentencesPerSample",
- description = "number of sentences per sample")
- String getSentencesPerSample();
-
- @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
- description = "search through files recursively")
- boolean getRecurrentSearch();
-
- @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
- description = "only include files which contain a given string in their name")
- String getFileFilter();
-
- }
-
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascToken.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascToken.java
index 5fa96eeb..aef52dc0 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascToken.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascToken.java
@@ -17,6 +17,8 @@
package opennlp.tools.formats.masc;
+import java.io.Serial;
+
import opennlp.tools.util.Span;
/**
@@ -24,6 +26,7 @@ import opennlp.tools.util.Span;
*/
public class MascToken extends Span {
+ @Serial
private static final long serialVersionUID = -780646706788037041L;
private final String pos;
private final String base;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactory.java
index c58eb133..8a70c0b7 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactory.java
@@ -24,29 +24,52 @@ import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.tokenize.TokenSample;
import opennlp.tools.util.ObjectStream;
-public class MascTokenSampleStreamFactory<P> extends AbstractSampleStreamFactory<TokenSample, P> {
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see TokenSample
+ * @see MascTokenSampleStream
+ */
+@Internal
+public class MascTokenSampleStreamFactory extends
+ AbstractSampleStreamFactory<TokenSample, MascTokenSampleStreamFactory.Parameters>
+ implements Masc {
+
+ public interface Parameters extends BasicFormatParams {
+ @ArgumentParser.ParameterDescription(valueName = "sentencesPerSample",
+ description = "number of sentences per sample")
+ String getSentencesPerSample();
+
+ @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
+ description = "search through files recursively")
+ Boolean getRecurrentSearch();
+
+ @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
+ description = "only include files which contain a given string in their name")
+ String getFileFilter();
- public static final String MASC_FORMAT = "masc";
+ }
- protected MascTokenSampleStreamFactory(Class<P> params) {
+ protected MascTokenSampleStreamFactory(Class<Parameters> params) {
super(params);
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(TokenSample.class, MASC_FORMAT,
- new MascTokenSampleStreamFactory<>(MascTokenSampleStreamFactory.Parameters.class));
+ new MascTokenSampleStreamFactory(Parameters.class));
}
-
@Override
public ObjectStream<TokenSample> create(String[] args) {
- MascTokenSampleStreamFactory.Parameters params =
- ArgumentParser.parse(args, MascTokenSampleStreamFactory.Parameters.class);
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
+ Parameters params = ArgumentParser.parse(args, Parameters.class);
try {
FileFilter fileFilter = pathname -> pathname.getName().contains(params.getFileFilter());
@@ -59,20 +82,5 @@ public class MascTokenSampleStreamFactory<P> extends AbstractSampleStreamFactory
return null;
}
- interface Parameters extends BasicFormatParams {
- @ArgumentParser.ParameterDescription(valueName = "sentencesPerSample",
- description = "number of sentences per sample")
- String getSentencesPerSample();
-
- @ArgumentParser.ParameterDescription(valueName = "recurrentSearch",
- description = "search through files recursively")
- boolean getRecurrentSearch();
-
- @ArgumentParser.ParameterDescription(valueName = "fileFilterString",
- description = "only include files which contain a given string in their name")
- String getFileFilter();
-
- }
-
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascWord.java b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascWord.java
index 916f6d2d..f351927b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascWord.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/masc/MascWord.java
@@ -17,10 +17,13 @@
package opennlp.tools.formats.masc;
+import java.io.Serial;
+
import opennlp.tools.util.Span;
public class MascWord extends Span {
+ @Serial
private static final long serialVersionUID = 2133473549058189775L;
private final int id;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStream.java
index cd7b10f8..e0f8bc2f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStream.java
@@ -27,6 +27,13 @@ import opennlp.tools.util.FilterObjectStream;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.Span;
+/**
+ * Moses is a statistical machine translation system that allows you
+ * to automatically train translation models for any language pair.
+ * <p>
+ * Details are found on the <a href="https://www2.statmt.org/moses/">
+ * official website</a>.
+ */
public class MosesSentenceSampleStream extends FilterObjectStream<String, SentenceSample> {
/**
@@ -58,9 +65,8 @@ public class MosesSentenceSampleStream extends FilterObjectStream<String, Senten
sentencesString.append(' ');
}
- if (sentenceSpans.size() > 0) {
- return new SentenceSample(sentencesString.toString(),
- sentenceSpans.toArray(new Span[0]));
+ if (!sentenceSpans.isEmpty()) {
+ return new SentenceSample(sentencesString.toString(), sentenceSpans.toArray(new Span[0]));
}
return null;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactory.java
index 6e942c93..529f2179 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactory.java
@@ -17,49 +17,40 @@
package opennlp.tools.formats.moses;
-import java.io.IOException;
-
-import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.sentdetect.SentenceSample;
-import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
/**
* Factory producing OpenNLP {@link MosesSentenceSampleStream} objects.
+ * <p>
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see SentenceSample
+ * @see MosesSentenceSampleStream
*/
-public class MosesSentenceSampleStreamFactory<P> extends AbstractSampleStreamFactory<SentenceSample, P> {
+@Internal
+public class MosesSentenceSampleStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, MosesSentenceSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
}
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class,
- "moses", new MosesSentenceSampleStreamFactory<>(Parameters.class));
+ "moses", new MosesSentenceSampleStreamFactory(Parameters.class));
}
- protected MosesSentenceSampleStreamFactory(Class<P> params) {
+ protected MosesSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
- InputStreamFactory sampleDataIn = CmdLineUtil.createInputStreamFactory(params.getData());
-
- ObjectStream<String> lineStream = null;
- try {
- lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
- } catch (IOException ex) {
- CmdLineUtil.handleCreateObjectStreamError(ex);
- }
-
+ ObjectStream<String> lineStream = readData(args, Parameters.class);
return new MosesSentenceSampleStream(lineStream);
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java
index 28030beb..4acb31d1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java
@@ -23,21 +23,33 @@ import java.nio.charset.StandardCharsets;
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.cmdline.params.BasicFormatParams;
import opennlp.tools.cmdline.tokenizer.TokenizerModelLoader;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.formats.DirectorySampleStream;
import opennlp.tools.formats.convert.FileToStringSampleStream;
import opennlp.tools.namefind.NameSample;
+import opennlp.tools.tokenize.ThreadSafeTokenizerME;
import opennlp.tools.tokenize.Tokenizer;
-import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.StringUtil;
+/**
+ * @see NameSample
+ * @see MucNameSampleStream
+ */
+@Internal
public class Muc6NameSampleStreamFactory
extends AbstractSampleStreamFactory<NameSample, Muc6NameSampleStreamFactory.Parameters> {
+ public interface Parameters extends BasicFormatParams {
+ @ParameterDescription(valueName = "modelFile")
+ File getTokenizerModel();
+ }
+
protected Muc6NameSampleStreamFactory() {
super(Parameters.class);
}
@@ -49,11 +61,16 @@ public class Muc6NameSampleStreamFactory
@Override
public ObjectStream<NameSample> create(String[] args) {
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
Parameters params = ArgumentParser.parse(args, Parameters.class);
+ if (!params.getData().isDirectory() || !params.getData().exists()) {
+ throw new TerminateToolException(-1, "The specified data directory is not valid!");
+ }
TokenizerModel tokenizerModel = new TokenizerModelLoader().load(params.getTokenizerModel());
- Tokenizer tokenizer = new TokenizerME(tokenizerModel);
+ Tokenizer tokenizer = new ThreadSafeTokenizerME(tokenizerModel);
ObjectStream<String> mucDocStream = new FileToStringSampleStream(
new DirectorySampleStream(params.getData(),
@@ -62,9 +79,4 @@ public class Muc6NameSampleStreamFactory
return new MucNameSampleStream(tokenizer, mucDocStream);
}
-
- interface Parameters extends BasicFormatParams {
- @ParameterDescription(valueName = "modelFile")
- File getTokenizerModel();
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java
index 23eb09f2..d3ac523d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java
@@ -31,10 +31,8 @@ class MucElementNames {
static {
CONTENT_ELEMENTS = Set.of(
- MucElementNames.HEADLINE_ELEMENT,
- MucElementNames.DATELINE_ELEMENT,
- MucElementNames.DD_ELEMENT,
- MucElementNames.SENTENCE_ELEMENT);
+ MucElementNames.HEADLINE_ELEMENT, MucElementNames.DATELINE_ELEMENT,
+ MucElementNames.DD_ELEMENT, MucElementNames.SENTENCE_ELEMENT);
}
private MucElementNames() {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java
index f9a8bbc0..caf63a33 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java
@@ -64,7 +64,7 @@ public class MucNameSampleStream extends FilterObjectStream<String, NameSample>
}
}
- if (storedSamples.size() > 0) {
+ if (!storedSamples.isEmpty()) {
return storedSamples.remove(0);
}
else {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSegmentationDocument.java b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSegmentationDocument.java
index 08f615b1..3cbadd82 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSegmentationDocument.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSegmentationDocument.java
@@ -32,6 +32,7 @@ import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.w3c.dom.Document;
+import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
@@ -41,6 +42,13 @@ import opennlp.tools.util.XmlUtil;
public class NKJPSegmentationDocument {
+ private static final String XML_ID = "xml:id";
+ private static final String SEG = "seg";
+ private static final String CHOICE = "choice";
+ private static final String NKJP_PAREN = "nkjp:paren";
+ private static final String NKJP_REJECTED = "nkjp:rejected";
+ private static final String NKJP_NPS = "nkjp:nps";
+
public static class Pointer {
final String doc;
final String id;
@@ -62,8 +70,7 @@ public class NKJPSegmentationDocument {
@Override
public String toString() {
- return doc + "#string-range(" + id + "," + offset
- + "," + length + ")";
+ return doc + "#string-range(" + id + "," + offset + "," + length + ")";
}
}
@@ -103,8 +110,8 @@ public class NKJPSegmentationDocument {
Node sentnode = nl.item(i);
String sentid = null;
- if (sentnode.getAttributes().getNamedItem("xml:id") != null) {
- sentid = sentnode.getAttributes().getNamedItem("xml:id").getTextContent();
+ if (sentnode.getAttributes().getNamedItem(XML_ID) != null) {
+ sentid = sentnode.getAttributes().getNamedItem(XML_ID).getTextContent();
}
Map<String, Pointer> segments = new LinkedHashMap<>();
@@ -112,18 +119,17 @@ public class NKJPSegmentationDocument {
for (int j = 0; j < segnl.getLength(); j++) {
Node n = segnl.item(j);
- if (n.getNodeName().equals("seg")) {
+ if (n.getNodeName().equals(SEG)) {
String segid = xmlID(n);
Pointer pointer = fromSeg(n);
segments.put(segid, pointer);
- } else if (n.getNodeName().equals("choice")) {
-
+ } else if (n.getNodeName().equals(CHOICE)) {
NodeList choices = n.getChildNodes();
-
for (int k = 0; k < choices.getLength(); k++) {
- if (choices.item(k).getNodeName().equals("nkjp:paren")) {
- if (!checkRejectedParen(choices.item(k))) {
- NodeList paren_segs = (NodeList) SEG_NODES_ONLY.evaluate(choices.item(k),
+ Node choice = choices.item(k);
+ if (choice.getNodeName().equals(NKJP_PAREN)) {
+ if (!checkRejectedParen(choice)) {
+ NodeList paren_segs = (NodeList) SEG_NODES_ONLY.evaluate(choice,
XPathConstants.NODESET);
for (int l = 0; l < paren_segs.getLength(); l++) {
@@ -132,17 +138,16 @@ public class NKJPSegmentationDocument {
segments.put(segid, pointer);
}
}
- } else if (choices.item(k).getNodeName().equals("seg")) {
- if (!checkRejected(choices.item(k))) {
- String segid = xmlID(choices.item(k));
- Pointer pointer = fromSeg(choices.item(k));
+ } else if (choice.getNodeName().equals(SEG)) {
+ if (!checkRejected(choice)) {
+ String segid = xmlID(choice);
+ Pointer pointer = fromSeg(choice);
segments.put(segid, pointer);
}
}
}
}
}
-
sentences.put(sentid, segments);
}
@@ -154,14 +159,14 @@ public class NKJPSegmentationDocument {
}
static boolean checkRejected(Node n) {
- if (n.getAttributes() == null) {
+ NamedNodeMap attrs = n.getAttributes();
+ if (attrs == null) {
return false;
}
- if (n.getAttributes().getNamedItem("nkjp:rejected") == null) {
+ if (attrs.getNamedItem(NKJP_REJECTED) == null) {
return false;
}
- String rejected = n.getAttributes().getNamedItem("nkjp:rejected").getTextContent();
- return rejected.equals("true");
+ return attrs.getNamedItem(NKJP_REJECTED).getTextContent().equals("true");
}
static boolean checkRejectedParen(Node n) {
@@ -170,7 +175,7 @@ public class NKJPSegmentationDocument {
} else {
for (int i = 0; i < n.getChildNodes().getLength(); i++) {
Node m = n.getChildNodes().item(i);
- if (m.getNodeName().equals("seg")) {
+ if (m.getNodeName().equals(SEG)) {
if (!checkRejected(m)) {
return false;
}
@@ -181,13 +186,14 @@ public class NKJPSegmentationDocument {
}
static String xmlID(Node n) throws IOException {
- if (n.getAttributes() == null || n.getAttributes().getLength() < 1) {
+ NamedNodeMap attr = n.getAttributes();
+ if (attr == null || attr.getLength() < 1) {
throw new IOException("Missing required attributes");
}
String id = null;
- if (n.getAttributes().getNamedItem("xml:id") != null) {
- id = n.getAttributes().getNamedItem("xml:id").getTextContent();
+ if (attr.getNamedItem(XML_ID) != null) {
+ id = attr.getNamedItem(XML_ID).getTextContent();
}
if (id == null) {
@@ -207,8 +213,8 @@ public class NKJPSegmentationDocument {
ptr = n.getAttributes().getNamedItem("corresp").getTextContent();
}
String spacing = "";
- if (n.getAttributes().getNamedItem("nkjp:nps") != null) {
- spacing = n.getAttributes().getNamedItem("nkjp:nps").getTextContent();
+ if (n.getAttributes().getNamedItem(NKJP_NPS) != null) {
+ spacing = n.getAttributes().getNamedItem(NKJP_NPS).getTextContent();
}
if (ptr == null) {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStream.java
index 03ed8348..b71a4c50 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStream.java
@@ -28,8 +28,8 @@ import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.Span;
public class NKJPSentenceSampleStream implements ObjectStream<SentenceSample> {
- private final NKJPSegmentationDocument segments;
+ private final NKJPSegmentationDocument segments;
private final NKJPTextDocument text;
private Iterator<Map.Entry<String, Map<String, NKJPSegmentationDocument.Pointer>>> segmentIt;
@@ -89,7 +89,7 @@ public class NKJPSentenceSampleStream implements ObjectStream<SentenceSample> {
}
// end of stream is reached, indicate that with null return value
- if (sentenceSpans.size() == 0) {
+ if (sentenceSpans.isEmpty()) {
return null;
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactory.java
index d126d21e..da9e5989 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactory.java
@@ -24,13 +24,22 @@ import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.cmdline.params.BasicFormatParams;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.util.ObjectStream;
-public class NKJPSentenceSampleStreamFactory<P> extends AbstractSampleStreamFactory<SentenceSample, P> {
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see SentenceSample
+ * @see NKJPSentenceSampleStream
+ */
+@Internal
+public class NKJPSentenceSampleStreamFactory extends
+ AbstractSampleStreamFactory<SentenceSample, NKJPSentenceSampleStreamFactory.Parameters> {
- interface Parameters extends BasicFormatParams {
+ public interface Parameters extends BasicFormatParams {
@ArgumentParser.ParameterDescription(valueName = "text",
description = "file containing NKJP text")
File getTextFile();
@@ -38,20 +47,16 @@ public class NKJPSentenceSampleStreamFactory<P> extends AbstractSampleStreamFact
public static void registerFactory() {
StreamFactoryRegistry.registerFactory(SentenceSample.class, "nkjp",
- new NKJPSentenceSampleStreamFactory<>(NKJPSentenceSampleStreamFactory.Parameters.class));
+ new NKJPSentenceSampleStreamFactory(NKJPSentenceSampleStreamFactory.Parameters.class));
}
- protected NKJPSentenceSampleStreamFactory(Class<P> params) {
+ protected NKJPSentenceSampleStreamFactory(Class<Parameters> params) {
super(params);
}
@Override
public ObjectStream<SentenceSample> create(String[] args) {
-
- Parameters params = ArgumentParser.parse(args, Parameters.class);
-
- CmdLineUtil.checkInputFile("Data", params.getData());
-
+ Parameters params = validateBasicFormatParameters(args, Parameters.class);
CmdLineUtil.checkInputFile("Text", params.getTextFile());
NKJPSegmentationDocument segDoc = null;
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPTextDocument.java b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPTextDocument.java
index 3c20e704..205914eb 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPTextDocument.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/nkjp/NKJPTextDocument.java
@@ -43,14 +43,14 @@ import opennlp.tools.util.XmlUtil;
* The National corpus of Polish (NKJP) format.
* <p>
* Information about the format are found on this
- * <a href="http://nkjp.pl/index.php?page=0%26lang=1">web site</a>.
+ * <a href="https://nkjp.pl/index.php?page=0%26lang=1">web site</a>.
* <p>
* A 1-million word corpus can be found on this
- * <a href="http://nkjp.pl/index.php?page=14%26lang=1">
+ * <a href="https://nkjp.pl/index.php?page=14%26lang=1">
* web site</a>.
* <p>
* The NKJP schema can be found
- * <a href="http://nlp.ipipan.waw.pl/TEI4NKJP/">here</a>.
+ * <a href="https://nlp.ipipan.waw.pl/TEI4NKJP/">here</a>.
*/
public class NKJPTextDocument {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java
index 8dc09af8..130e7386 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java
@@ -35,10 +35,18 @@ import opennlp.tools.util.Span;
import opennlp.tools.util.StringUtil;
/**
- * Name Sample Stream parser for the OntoNotes 4.0 corpus.
+ * Name Sample Stream parser for the OntoNotes 4.0 named entity files.
*/
public class OntoNotesNameSampleStream extends FilterObjectStream<String, NameSample> {
+ private static final String TAG_DOC_OPEN = "<DOC";
+ private static final String TAG_DOC_CLOSE = "</DOC>";
+ private static final String TAG_ENAMEX_OPEN = "<ENAMEX";
+ private static final String TAG_ENAMEX_CLOSE = "</ENAMEX>";
+ private static final String TYPE = "TYPE=\"";
+ private static final String SYMBOL_CLOSE = ">";
+ private static final String SYMBOL_OPEN = "<";
+
private final Map<String, String> tokenConversionMap;
private final List<NameSample> nameSamples = new LinkedList<>();
@@ -69,21 +77,18 @@ public class OntoNotesNameSampleStream extends FilterObjectStream<String, NameSa
StringBuilder convertedToken = new StringBuilder(token);
- int startTagEndIndex = convertedToken.indexOf(">");
-
+ int startTagEndIndex = convertedToken.indexOf(SYMBOL_CLOSE);
if (token.contains("=\"") && startTagEndIndex != -1) {
convertedToken.delete(0, startTagEndIndex + 1);
}
- int endTagBeginIndex = convertedToken.indexOf("<");
- int endTagEndIndex = convertedToken.indexOf(">");
-
+ int endTagBeginIndex = convertedToken.indexOf(SYMBOL_OPEN);
+ int endTagEndIndex = convertedToken.indexOf(SYMBOL_CLOSE);
if (endTagBeginIndex != -1 && endTagEndIndex != -1) {
convertedToken.delete(endTagBeginIndex, endTagEndIndex + 1);
}
String cleanedToken = convertedToken.toString();
-
if (tokenConversionMap.get(cleanedToken) != null) {
cleanedToken = tokenConversionMap.get(cleanedToken);
}
@@ -96,74 +101,64 @@ public class OntoNotesNameSampleStream extends FilterObjectStream<String, NameSa
if (nameSamples.isEmpty()) {
String doc = samples.read();
-
if (doc != null) {
- BufferedReader docIn = new BufferedReader(new StringReader(doc));
-
boolean clearAdaptiveData = true;
-
String line;
- while ((line = docIn.readLine()) != null) {
-
- if (line.startsWith("<DOC")) {
- continue;
- }
-
- if (line.equals("</DOC>")) {
- break;
- }
-
- String[] tokens = WhitespaceTokenizer.INSTANCE.tokenize(line);
+ try (BufferedReader docIn = new BufferedReader(new StringReader(doc))) {
+ while ((line = docIn.readLine()) != null) {
- List<Span> entities = new LinkedList<>();
- List<String> cleanedTokens = new ArrayList<>(tokens.length);
-
- int tokenIndex = 0;
- int entityBeginIndex = -1;
- String entityType = null;
- boolean insideStartEnmaxTag = false;
- for (String token : tokens) {
-
- // Split here, next part of tag is in new token
- if (token.startsWith("<ENAMEX")) {
- insideStartEnmaxTag = true;
+ if (line.startsWith(TAG_DOC_OPEN)) {
continue;
}
+ if (line.equals(TAG_DOC_CLOSE)) {
+ break;
+ }
- if (insideStartEnmaxTag) {
-
- String typeBegin = "TYPE=\"";
+ String[] tokens = WhitespaceTokenizer.INSTANCE.tokenize(line);
+ List<Span> entities = new LinkedList<>();
+ List<String> cleanedTokens = new ArrayList<>(tokens.length);
- if (token.startsWith(typeBegin)) {
+ int tokenIndex = 0;
+ int entityBeginIndex = -1;
+ String entityType = null;
+ boolean insideStartEnmaxTag = false;
+ for (String token : tokens) {
- int typeEnd = token.indexOf("\"", typeBegin.length());
+ // Split here, next part of tag is in new token
+ if (token.startsWith(TAG_ENAMEX_OPEN)) {
+ insideStartEnmaxTag = true;
+ continue;
+ }
- entityType = StringUtil.toLowerCase(token.substring(typeBegin.length(), typeEnd));
+ if (insideStartEnmaxTag) {
+ String typeBegin = TYPE;
+ if (token.startsWith(typeBegin)) {
+ int typeEnd = token.indexOf("\"", typeBegin.length());
+ entityType = StringUtil.toLowerCase(token.substring(typeBegin.length(), typeEnd));
+ }
+
+ if (token.contains(SYMBOL_CLOSE)) {
+ entityBeginIndex = tokenIndex;
+ insideStartEnmaxTag = false;
+ } else {
+ continue;
+ }
}
- if (token.contains(">")) {
- entityBeginIndex = tokenIndex;
- insideStartEnmaxTag = false;
- } else {
- continue;
+ if (token.endsWith(TAG_ENAMEX_CLOSE)) {
+ entities.add(new Span(entityBeginIndex, tokenIndex + 1, entityType));
+ entityBeginIndex = -1;
}
- }
- if (token.endsWith("</ENAMEX>")) {
- entities.add(new Span(entityBeginIndex, tokenIndex + 1,
- entityType));
- entityBeginIndex = -1;
+ cleanedTokens.add(convertToken(token));
+ tokenIndex++;
}
- cleanedTokens.add(convertToken(token));
- tokenIndex++;
- }
+ nameSamples.add(new NameSample(cleanedTokens.toArray(new String[0]),
+ entities.toArray(new Span[0]), clearAdaptiveData));
- nameSamples.add(new NameSample(cleanedTokens
- .toArray(new String[0]), entities
- .toArray(new Span[0]), clearAdaptiveData));
-
- clearAdaptiveData = false;
+ clearAdaptiveData = false;
+ }
}
}
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java
index f71e3042..9bb154fa 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java
@@ -22,23 +22,42 @@ import java.nio.charset.StandardCharsets;
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.formats.DirectorySampleStream;
import opennlp.tools.formats.convert.FileToStringSampleStream;
import opennlp.tools.namefind.NameSample;
import opennlp.tools.util.ObjectStream;
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see OntoNotesNameSampleStream
+ */
+@Internal
public class OntoNotesNameSampleStreamFactory extends
AbstractSampleStreamFactory<NameSample, OntoNotesFormatParameters> {
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(NameSample.class,
+ "ontonotes", new OntoNotesNameSampleStreamFactory());
+ }
+
public OntoNotesNameSampleStreamFactory() {
super(OntoNotesFormatParameters.class);
}
@Override
public ObjectStream<NameSample> create(String[] args) {
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
OntoNotesFormatParameters params = ArgumentParser.parse(args, OntoNotesFormatParameters.class);
+ final File ontoDir = new File(params.getOntoNotesDir());
+ if (!ontoDir.isDirectory() || !ontoDir.exists()) {
+ throw new TerminateToolException(-1, "The specified OntoNotes directory is not valid!");
+ }
ObjectStream<File> documentStream = new DirectorySampleStream(new File(
params.getOntoNotesDir()),
@@ -54,8 +73,4 @@ public class OntoNotesNameSampleStreamFactory extends
new FileToStringSampleStream(documentStream, StandardCharsets.UTF_8));
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(NameSample.class,
- "ontonotes", new OntoNotesNameSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesPOSSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesPOSSampleStreamFactory.java
index a82c0d97..493823f4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesPOSSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesPOSSampleStreamFactory.java
@@ -18,12 +18,19 @@
package opennlp.tools.formats.ontonotes;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.formats.convert.ParseToPOSSampleStream;
import opennlp.tools.parser.Parse;
import opennlp.tools.postag.POSSample;
import opennlp.tools.util.ObjectStream;
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see ParseToPOSSampleStream
+ */
+@Internal
public class OntoNotesPOSSampleStreamFactory
extends AbstractSampleStreamFactory<POSSample, OntoNotesFormatParameters> {
@@ -34,14 +41,18 @@ public class OntoNotesPOSSampleStreamFactory
super(OntoNotesFormatParameters.class);
}
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(POSSample.class, "ontonotes",
+ new OntoNotesPOSSampleStreamFactory());
+ }
+
@Override
public ObjectStream<POSSample> create(String[] args) {
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
ObjectStream<Parse> parseSampleStream = parseSampleStreamFactory.create(args);
return new ParseToPOSSampleStream(parseSampleStream);
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(POSSample.class, "ontonotes",
- new OntoNotesPOSSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java
index 84fd1a5a..a18d5558 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java
@@ -52,7 +52,7 @@ public class OntoNotesParseSampleStream extends FilterObjectStream<String, Parse
}
if (parse == null || parse.isEmpty()) {
- if (parseString.length() > 0) {
+ if (!parseString.isEmpty()) {
return Parse.parseParse(parseString.toString());
}
else {
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java
index 5b9fa0b5..35bbe33e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java
@@ -22,12 +22,20 @@ import java.nio.charset.StandardCharsets;
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.commons.Internal;
import opennlp.tools.formats.AbstractSampleStreamFactory;
import opennlp.tools.formats.DirectorySampleStream;
import opennlp.tools.formats.convert.FileToStringSampleStream;
import opennlp.tools.parser.Parse;
import opennlp.tools.util.ObjectStream;
+/**
+ * <b>Note:</b> Do not use this class, internal use only!
+ *
+ * @see OntoNotesParseSampleStream
+ */
+@Internal
public class OntoNotesParseSampleStreamFactory
extends AbstractSampleStreamFactory<Parse, OntoNotesFormatParameters> {
@@ -35,10 +43,21 @@ public class OntoNotesParseSampleStreamFactory
super(OntoNotesFormatParameters.class);
}
+ public static void registerFactory() {
+ StreamFactoryRegistry.registerFactory(Parse.class, "ontonotes",
+ new OntoNotesParseSampleStreamFactory());
+ }
+
@Override
public ObjectStream<Parse> create(String[] args) {
-
+ if (args == null) {
+ throw new IllegalArgumentException("Passed args must not be null!");
+ }
OntoNotesFormatParameters params = ArgumentParser.parse(args, OntoNotesFormatParameters.class);
+ final File ontoDir = new File(params.getOntoNotesDir());
+ if (!ontoDir.isDirectory() || !ontoDir.exists()) {
+ throw new TerminateToolException(-1, "The specified OntoNotes directory is not valid!");
+ }
ObjectStream<File> documentStream = new DirectorySampleStream(new File(
params.getOntoNotesDir()),
@@ -53,13 +72,8 @@ public class OntoNotesParseSampleStreamFactory
// We need file to line here ... and that is probably best doen with the plain text stream
// lets copy it over here, refactor it, and then at some point we replace the current version
// with the refactored version
-
- return new OntoNotesParseSampleStream(new DocumentToLineStream(new FileToStringSampleStream(
- documentStream, StandardCharsets.UTF_8)));
+ return new OntoNotesParseSampleStream(new DocumentToLineStream(
+ new FileToStringSampleStream(documentStream, StandardCharsets.UTF_8)));
}
- public static void registerFactory() {
- StreamFactoryRegistry.registerFactory(Parse.class, "ontonotes",
- new OntoNotesParseSampleStreamFactory());
- }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleDataStream.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleDataStream.java
index acfdbbae..f7e53dab 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleDataStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleDataStream.java
@@ -24,12 +24,12 @@ import opennlp.tools.util.FilterObjectStream;
import opennlp.tools.util.ObjectStream;
/**
- * The {@link NameSampleDataStream} class converts tagged {@link String strings}
+ * The {@link NameSampleDataStream} class converts tagged tokens
* provided by a {@link DataStream} to {@link NameSample} objects.
- * It uses text that is one-sentence per line and tokenized
- * with names identified by:
* <p>
- * {@code <START>} and {@code <END>} tags.
+ * It uses text that is one-sentence per line and tokenized
+ * with names identified by: <br/>
+ * <{@code START}> and <{@code END}>tags.
*/
public class NameSampleDataStream extends FilterObjectStream<String, NameSample> {
@@ -55,7 +55,7 @@ public class NameSampleDataStream extends FilterObjectStream<String, NameSample>
// An empty line indicates the start of a new article
// for which the adaptive data in the feature generators
// must be cleared
- while (token != null && token.trim().length() == 0) {
+ while (token != null && token.trim().isEmpty()) {
isClearAdaptiveData = true;
token = samples.read();
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/ParseSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/parser/ParseSampleStream.java
index 82cfe0a7..b75edbf6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/ParseSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/ParseSampleStream.java
@@ -22,6 +22,10 @@ import java.io.IOException;
import opennlp.tools.util.FilterObjectStream;
import opennlp.tools.util.ObjectStream;
+/**
+ * @see Parse
+ * @see FilterObjectStream
+ */
public class ParseSampleStream extends FilterObjectStream<String, Parse> {
/**
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/WordTagSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/postag/WordTagSampleStream.java
index 4aac6869..40653117 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/WordTagSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/WordTagSampleStream.java
@@ -28,7 +28,7 @@ import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.ObjectStream;
/**
- * A stream filter which reads a sentence per line which contains
+ * A {@link FilterObjectStream stream filter} which reads a sentence per line that contains
* words and tags in {@code word_tag} format and outputs a {@link POSSample} objects.
*/
public class WordTagSampleStream extends FilterObjectStream<String, POSSample> {
@@ -45,14 +45,14 @@ public class WordTagSampleStream extends FilterObjectStream<String, POSSample> {
}
/**
- * Parses the next sentence and return the next {@link POSSample} object.
+ * Reads the next tokens and parses it into the next {@link POSSample} object.
* <p>
* If an error occurs an empty {@link POSSample} object is returned
* and a warning message is logged. Usually it does not matter if one
* or many sentences are ignored.
*
* @return A valid {@link POSSample} or {@code null} if the
- * {@link ObjectStream sentence stream} is exhausted.
+ * {@link ObjectStream stream} is exhausted.
*
* @throws IOException Thrown if IO errors occurred during read.
*/
@@ -66,17 +66,13 @@ public class WordTagSampleStream extends FilterObjectStream<String, POSSample> {
try {
sample = POSSample.parse(sentence);
} catch (InvalidFormatException e) {
- // TODO: An exception in error case should be thrown.
logger.warn("Error during parsing, ignoring sentence: {}", sentence, e);
-
sample = new POSSample(new String[]{}, new String[]{});
}
-
return sample;
}
else {
- // sentences stream is exhausted
- return null;
+ return null; // sentences stream is exhausted
}
}
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamFactoryTest.java
new file mode 100644
index 00000000..bcd2e2ec
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamFactoryTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public abstract class AbstractSampleStreamFactoryTest<S, P> extends AbstractFormatTest {
+
+ protected static final String FORMAT_SAMPLE_DIR = "opennlp/tools/formats/";
+
+ protected abstract AbstractSampleStreamFactory<S, P> getFactory();
+ protected abstract String getDataFilePath();
+
+ @Test
+ void testCreateWithNullParameter() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ try (ObjectStream<S> stream = getFactory().create(null)) {
+ stream.read();
+ }
+ });
+ }
+
+ @Test
+ void testCreateWithInvalidParameter() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ try (ObjectStream<S> stream = getFactory().create(new String[]{"X"})) {
+ stream.read();
+ }
+ });
+ }
+
+ /*
+ * Note:
+ * This test case must be overridden for non-simple cases where
+ * more than the '-data' param is required.
+ */
+ @Test
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<S> stream = getFactory().create(
+ new String[]{"-data", getDataFilePath() + "xyz"})) {
+ S sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java
index c154f21d..f56b7711 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/AbstractSampleStreamTest.java
@@ -19,8 +19,7 @@ package opennlp.tools.formats;
import opennlp.tools.util.InputStreamFactory;
-
-public abstract class AbstractSampleStreamTest extends AbstractFormatTest {
+abstract class AbstractSampleStreamTest extends AbstractFormatTest {
protected InputStreamFactory getFactory(String resource) {
return new ResourceAsStreamFactory(AbstractSampleStreamTest.class, FORMATS_BASE_DIR + resource);
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..25c19fa2
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactoryTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class BioNLP2004NameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, BioNLP2004NameSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "bionlp2004-01.sample";
+
+ // SUT
+ private BioNLP2004NameSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory
+ <NameSample, BioNLP2004NameSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ BioNLP2004NameSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, BioNLP2004NameSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "bionlp2004");
+ assertInstanceOf(BioNLP2004NameSampleStreamFactory.class, f);
+ factory = (BioNLP2004NameSampleStreamFactory) f;
+ assertEquals(BioNLP2004NameSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-types", "DNA,protein,cell_type,cell_line,RNA", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ /* some extra checks to make sure the 'types' parameter is handled correctly */
+ assertEquals(5, sample.getNames().length);
+ assertEquals("protein", sample.getNames()[0].getType());
+ assertEquals("protein", sample.getNames()[1].getType());
+ assertEquals("protein", sample.getNames()[2].getType());
+ assertEquals("protein", sample.getNames()[3].getType());
+ assertEquals("cell_type", sample.getNames()[4].getType());
+ }
+ }
+
+ @Test
+ void testCreateWithUnsupportedTypes() throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-types", "xyz", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-types", "DNA,protein,cell_type,cell_line,RNA", "-data", sampleFileFullPath + "xyz"})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ChunkerSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ChunkerSampleStreamFactoryTest.java
new file mode 100644
index 00000000..ad5a4e1c
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ChunkerSampleStreamFactoryTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.chunker.ChunkSample;
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class ChunkerSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<ChunkSample, ChunkerSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "chunker-01.sample";
+
+ // SUT
+ private ChunkerSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<ChunkSample, ChunkerSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ChunkerSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<ChunkSample, ChunkerSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(ChunkSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(ChunkerSampleStreamFactory.class, f);
+ factory = (ChunkerSampleStreamFactory) f;
+ assertEquals(ChunkerSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<ChunkSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ ChunkSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/Conll02NameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/Conll02NameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..dec1fa9a
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/Conll02NameSampleStreamFactoryTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class Conll02NameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, Conll02NameSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "conll2002-es.sample";
+ private static final String SAMPLE_02 = "conll2002-nl.sample";
+
+ // SUT
+ private Conll02NameSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, Conll02NameSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ Conll02NameSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, Conll02NameSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "conll02");
+ assertInstanceOf(Conll02NameSampleStreamFactory.class, f);
+ factory = ((Conll02NameSampleStreamFactory) f);
+ assertEquals(Conll02NameSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"spa", "es", "nld", "nl"})
+ void testCreateWithValidParameter(String lang) throws IOException {
+ // prepare depending on language
+ if ("spa".equals(lang) || "es".equals(lang)) {
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ } else {
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_02).getPath();
+ }
+
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", lang, "-types", "per,loc,org,misc", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", "per", "loc", "org", "misc", "per,loc,org,misc"})
+ void testCreateWithDifferentTypes(String types) throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", "spa", "-types", types, "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @Test
+ void testCreateWithInvalidLanguage() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", "xyz", "-types", "per,loc,org,misc", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-lang", "spa", "-types", "per,loc,org,misc", "-data", sampleFileFullPath + "xyz"})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/Conll03NameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/Conll03NameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..999bdf5c
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/Conll03NameSampleStreamFactoryTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class Conll03NameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, Conll03NameSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "conll2003-de.sample";
+ private static final String SAMPLE_02 = "conll2003-en.sample";
+
+ // SUT
+ private Conll03NameSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, Conll03NameSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ Conll03NameSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, Conll03NameSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "conll03");
+ assertInstanceOf(Conll03NameSampleStreamFactory.class, f);
+ factory = ((Conll03NameSampleStreamFactory) f);
+ assertEquals(Conll03NameSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"deu", "eng"})
+ void testCreateWithValidParameter(String lang) throws IOException {
+ // prepare depending on language
+ if ("deu".equals(lang)) {
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ } else {
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_02).getPath();
+ }
+
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", lang, "-types", "per,loc,org,misc", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", "per", "loc", "org", "misc", "per,loc,org,misc"})
+ void testCreateWithDifferentTypes(String types) throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", "deu", "-types", types, "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @Test
+ void testCreateWithInvalidLanguage() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", "xyz", "-types", "per,loc,org,misc", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-lang", "deu", "-types", "per,loc,org,misc", "-data", sampleFileFullPath + "xyz"})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXPOSSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXPOSSampleStreamFactoryTest.java
new file mode 100644
index 00000000..095a5c00
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXPOSSampleStreamFactoryTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class ConllXPOSSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<POSSample, ConllXPOSSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "conllx.sample";
+
+ // SUT
+ private ConllXPOSSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<POSSample, ConllXPOSSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConllXPOSSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<POSSample, ConllXPOSSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(POSSample.class, ConllXPOSSampleStreamFactory.CONLLX_FORMAT);
+ assertInstanceOf(ConllXPOSSampleStreamFactory.class, f);
+ factory = (ConllXPOSSampleStreamFactory) f;
+ assertEquals(ConllXPOSSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<POSSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..3b00d1c5
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ConllXSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, ConllXSentenceSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "conllx.sample";
+
+ // SUT
+ private ConllXSentenceSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, ConllXSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConllXSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, ConllXSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, ConllXPOSSampleStreamFactory.CONLLX_FORMAT);
+ assertInstanceOf(ConllXSentenceSampleStreamFactory.class, f);
+ factory = (ConllXSentenceSampleStreamFactory) f;
+ assertEquals(ConllXSentenceSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(
+ new String[]{"-encoding", "UTF-8", "-detokenizer", detokFileFullPath, "-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-encoding", "UTF-8", "-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..1c607ced
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ConllXTokenSampleStreamFactoryTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ConllXTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, ConllXTokenSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "conllx.sample";
+
+ // SUT
+ private ConllXTokenSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, ConllXTokenSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConllXTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, ConllXTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, ConllXPOSSampleStreamFactory.CONLLX_FORMAT);
+ assertInstanceOf(ConllXTokenSampleStreamFactory.class, f);
+ factory = (ConllXTokenSampleStreamFactory) f;
+ assertEquals(ConllXTokenSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-encoding", "UTF-8", "-detokenizer", detokFileFullPath, "-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-encoding", "UTF-8", "-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..eb6eebf8
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamFactoryTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class EvalitaNameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, EvalitaNameSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "evalita-ner-it-01.sample";
+
+ // SUT
+ private EvalitaNameSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, EvalitaNameSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ EvalitaNameSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, EvalitaNameSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "evalita");
+ assertInstanceOf(EvalitaNameSampleStreamFactory.class, f);
+ factory = ((EvalitaNameSampleStreamFactory) f);
+ assertEquals(EvalitaNameSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", "it", "-types", "per,loc,org,gpe", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @Test
+ void testCreateWithInvalidLanguage() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-lang", "xy", "-types", "per,loc,org,gpe", "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-lang", "it", "-types", "per,loc,org,gpe", "-data", sampleFileFullPath + "xyz"})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamTest.java
index 37929e3e..9bfcc57a 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/EvalitaNameSampleStreamTest.java
@@ -18,54 +18,102 @@
package opennlp.tools.formats;
import java.io.IOException;
+import java.util.stream.Stream;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
import opennlp.tools.formats.EvalitaNameSampleStream.LANGUAGE;
import opennlp.tools.namefind.NameSample;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.Span;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
/**
* Note:
* Sample training data must be UTF-8 encoded and uncompressed!
*/
public class EvalitaNameSampleStreamTest extends AbstractSampleStreamTest {
- @Test
- void testParsingItalianSample() throws IOException {
+ private static final String SAMPLE_01 = "evalita-ner-it-01.sample";
+ private static final String SAMPLE_02 = "evalita-ner-it-02.sample";
+ private static final String SAMPLE_03 = "evalita-ner-it-03.sample";
+ private static final String SAMPLE_BROKEN = "evalita-ner-it-broken.sample";
+ private static final String SAMPLE_INCORRECT = "evalita-ner-it-incorrect.sample";
- try (ObjectStream<NameSample> sampleStream = openData()) {
- NameSample personName = sampleStream.read();
- Assertions.assertNotNull(personName);
+ @ParameterizedTest
+ @MethodSource(value = "provideData")
+ void testReadItalianDifferentEntityTypes(String file, int nerType, int expectedSentLength,
+ int expectedStart, int expectedEnd) throws IOException {
- Assertions.assertEquals(11, personName.getSentence().length);
- Assertions.assertEquals(1, personName.getNames().length);
- Assertions.assertTrue(personName.isClearAdaptiveDataSet());
+ try (ObjectStream<NameSample> sampleStream = openData(file, nerType)) {
+ NameSample ne = sampleStream.read();
+ assertNotNull(ne);
- Span nameSpan = personName.getNames()[0];
- Assertions.assertEquals(8, nameSpan.getStart());
- Assertions.assertEquals(10, nameSpan.getEnd());
- Assertions.assertTrue(personName.isClearAdaptiveDataSet());
+ assertEquals(expectedSentLength, ne.getSentence().length);
+ assertEquals(1, ne.getNames().length);
+ assertTrue(ne.isClearAdaptiveDataSet());
- Assertions.assertEquals(0, sampleStream.read().getNames().length);
+ Span nameSpan = ne.getNames()[0];
+ assertEquals(expectedStart, nameSpan.getStart());
+ assertEquals(expectedEnd, nameSpan.getEnd());
+ assertTrue(ne.isClearAdaptiveDataSet());
- Assertions.assertNull(sampleStream.read());
+ if (SAMPLE_01.equals(file)) { // this file has an extra sentence
+ assertEquals(0, sampleStream.read().getNames().length);
+ }
+ assertNull(sampleStream.read());
}
}
+ @Test
+ void testReadWithIncorrectInput() {
+ assertThrows(IOException.class, () -> {
+ try (ObjectStream<NameSample> sampleStream = openData(
+ SAMPLE_INCORRECT, EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES)) {
+ sampleStream.read();
+ }
+ });
+ }
+
+ @Test
+ void testReadWithBrokenDocument() {
+ assertThrows(IOException.class, () -> {
+ try (ObjectStream<NameSample> sampleStream = openData(
+ SAMPLE_BROKEN, EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES)) {
+ sampleStream.read();
+ }
+ });
+ }
+
@Test
void testReset() throws IOException {
- try (ObjectStream<NameSample> sampleStream = openData()) {
+ try (ObjectStream<NameSample> sampleStream = openData(SAMPLE_01,
+ EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES)) {
NameSample sample = sampleStream.read();
sampleStream.reset();
- Assertions.assertEquals(sample, sampleStream.read());
+ assertEquals(sample, sampleStream.read());
}
}
- private ObjectStream<NameSample> openData() throws IOException {
- return new EvalitaNameSampleStream(LANGUAGE.IT, getFactory("evalita-ner-it.sample"),
- EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES);
+ // Note: This needs to be public as JUnit 5 requires it like this.
+ public static Stream<Arguments> provideData() {
+ return Stream.of(
+ Arguments.of(SAMPLE_01, EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES, 11, 8, 10),
+ Arguments.of(SAMPLE_02, EvalitaNameSampleStream.GENERATE_PERSON_ENTITIES, 27, 11, 13),
+ Arguments.of(SAMPLE_02, EvalitaNameSampleStream.GENERATE_ORGANIZATION_ENTITIES, 27, 10, 11),
+ Arguments.of(SAMPLE_03, EvalitaNameSampleStream.GENERATE_GPE_ENTITIES, 20, 18, 19)
+ );
+ }
+
+ private ObjectStream<NameSample> openData(String fileName, int nerType) throws IOException {
+ return new EvalitaNameSampleStream(LANGUAGE.IT, getFactory(fileName), nerType);
}
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactoryTest.java
new file mode 100644
index 00000000..ad50ec82
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/LanguageDetectorSampleStreamFactoryTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.langdetect.LanguageSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class LanguageDetectorSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<LanguageSample, LanguageDetectorSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "lang-detect-01.sample";
+
+ // SUT
+ private LanguageDetectorSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory
+ <LanguageSample, LanguageDetectorSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ LanguageDetectorSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<LanguageSample, LanguageDetectorSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(LanguageSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(LanguageDetectorSampleStreamFactory.class, f);
+ factory = (LanguageDetectorSampleStreamFactory) f;
+ assertEquals(LanguageDetectorSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<LanguageSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ LanguageSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/LemmatizerSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/LemmatizerSampleStreamFactoryTest.java
new file mode 100644
index 00000000..03a4af0d
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/LemmatizerSampleStreamFactoryTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.lemmatizer.LemmaSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class LemmatizerSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<LemmaSample, LemmatizerSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "lemma-01.sample";
+
+ // SUT
+ private LemmatizerSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<LemmaSample, LemmatizerSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ LemmatizerSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<LemmaSample, LemmatizerSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(LemmaSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(LemmatizerSampleStreamFactory.class, f);
+ factory = (LemmatizerSampleStreamFactory) f;
+ assertEquals(LemmatizerSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<LemmaSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ LemmaSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/NameSampleDataStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/NameSampleDataStreamFactoryTest.java
new file mode 100644
index 00000000..b1f6f8f6
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/NameSampleDataStreamFactoryTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.Span;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class NameSampleDataStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, NameSampleDataStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "name-data-01.sample";
+
+ // SUT
+ private NameSampleDataStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, NameSampleDataStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ NameSampleDataStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, NameSampleDataStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(NameSampleDataStreamFactory.class, f);
+ factory = (NameSampleDataStreamFactory) f;
+ assertEquals(NameSampleDataStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ Span[] hits = sample.getNames();
+ assertNotNull(hits);
+ assertEquals(1, hits.length);
+ assertEquals(5, hits[0].getStart());
+ assertEquals(6, hits[0].getEnd());
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ParseSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ParseSampleStreamFactoryTest.java
new file mode 100644
index 00000000..6bb24b7c
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ParseSampleStreamFactoryTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.parser.Parse;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class ParseSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<Parse, ParseSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "parse-01.sample";
+
+ // SUT
+ private ParseSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<Parse, ParseSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ParseSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<Parse, ParseSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(Parse.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(ParseSampleStreamFactory.class, f);
+ factory = (ParseSampleStreamFactory) f;
+ assertEquals(ParseSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<Parse> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ Parse sample = stream.read();
+ assertNotNull(sample);
+ assertEquals("The test shall come today . ", sample.getText());
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/SentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/SentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..60c084b5
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/SentenceSampleStreamFactoryTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class SentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, SentenceSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "sentences-01.sample";
+
+ // SUT
+ private SentenceSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, SentenceSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ SentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, SentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(SentenceSampleStreamFactory.class, f);
+ factory = (SentenceSampleStreamFactory) f;
+ assertEquals(SentenceSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/TokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/TokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..44e478fa
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/TokenSampleStreamFactoryTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class TokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, TokenSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "tokens-01.sample";
+
+ // SUT
+ private TokenSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, TokenSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ TokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, TokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(TokenSampleStreamFactory.class, f);
+ factory = (TokenSampleStreamFactory) f;
+ assertEquals(TokenSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactoryTest.java
new file mode 100644
index 00000000..d07f1ece
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/TwentyNewsgroupSampleStreamFactoryTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EmptySource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.doccat.DocumentSample;
+import opennlp.tools.tokenize.TokenizerModel;
+import opennlp.tools.util.DownloadUtil;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.fail;
+
+public class TwentyNewsgroupSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<DocumentSample, TwentyNewsgroupSampleStreamFactory.Parameters> {
+
+ private static final Path OPENNLP_DIR = Paths.get(System.getProperty("OPENNLP_DOWNLOAD_HOME",
+ System.getProperty("user.home"))).resolve(".opennlp");
+ private static final String TOKENIZER_MODEL_NAME = "opennlp-en-ud-ewt-tokens-1.2-2.5.0.bin";
+
+ // SUT
+ private TwentyNewsgroupSampleStreamFactory factory;
+
+ private static String tokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<DocumentSample, TwentyNewsgroupSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ TwentyNewsgroupSampleStreamFactory.registerFactory();
+ try {
+ // ensure, the model is available locally for later test purposes
+ DownloadUtil.downloadModel("en", DownloadUtil.ModelType.TOKENIZER, TokenizerModel.class);
+ } catch (IOException e) {
+ fail(e.getLocalizedMessage());
+ }
+ tokFileFullPath = new File(OPENNLP_DIR + File.separator + TOKENIZER_MODEL_NAME).getPath();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<DocumentSample, TwentyNewsgroupSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(DocumentSample.class, "20newsgroup");
+ assertInstanceOf(TwentyNewsgroupSampleStreamFactory.class, f);
+ factory = (TwentyNewsgroupSampleStreamFactory) f;
+ assertEquals(TwentyNewsgroupSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "20newsgroup").getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"simple", "whitespace"})
+ void testCreateWithValidParameter(String tokenizerType) throws IOException {
+ try (ObjectStream<DocumentSample> stream = factory.create(new String[]
+ {"-ruleBasedTokenizer", tokenizerType, "-dataDir", sampleFileFullPath})) {
+ DocumentSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @EmptySource
+ @ValueSource(strings = {" ", "unknown"})
+ void testCreateWithInvalidRBTokenizer(String tokenizerType) {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<DocumentSample> stream = factory.create(new String[]
+ {"-ruleBasedTokenizer", tokenizerType, "-dataDir", sampleFileFullPath})) {
+ DocumentSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ @Test
+ void testCreateWithMETokenizer() throws IOException {
+ try (ObjectStream<DocumentSample> stream = factory.create(new String[]
+ {"-tokenizerModel", tokFileFullPath, "-dataDir", sampleFileFullPath})) {
+ DocumentSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @EmptySource
+ @ValueSource(strings = {" ", "unknown"})
+ void testCreateWithInvalidMETokenizer(String path) {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<DocumentSample> stream = factory.create(new String[]
+ {"-tokenizerModel", path, "-dataDir", sampleFileFullPath})) {
+ DocumentSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<DocumentSample> stream = factory.create(new String[]
+ {"-ruleBasedTokenizer", "whitespace", "-dataDir", sampleFileFullPath + "xyz"})) {
+ DocumentSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/WordTagSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/WordTagSampleStreamFactoryTest.java
new file mode 100644
index 00000000..c7b7cc96
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/WordTagSampleStreamFactoryTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class WordTagSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<POSSample, WordTagSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "word-tags-01.sample";
+
+ // SUT
+ private WordTagSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<POSSample, WordTagSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ WordTagSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<POSSample, WordTagSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(POSSample.class, StreamFactoryRegistry.DEFAULT_FORMAT);
+ assertInstanceOf(WordTagSampleStreamFactory.class, f);
+ factory = (WordTagSampleStreamFactory) f;
+ assertEquals(WordTagSampleStreamFactory.Parameters.class, factory.params);
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<POSSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ assertNotNull(sample.getTags());
+ assertEquals(6, sample.getTags().length);
+ assertNotNull(sample.getSentence());
+ assertEquals(6, sample.getSentence().length);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADChunkSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADChunkSampleStreamFactoryTest.java
new file mode 100644
index 00000000..052a988a
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADChunkSampleStreamFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.ad;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.chunker.ChunkSample;
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ADChunkSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<ChunkSample, ADChunkSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "ad.sample";
+
+ // SUT
+ private ADChunkSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<ChunkSample, ADChunkSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ADChunkSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<ChunkSample, ADChunkSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(ChunkSample.class, "ad");
+ assertInstanceOf(ADChunkSampleStreamFactory.class, f);
+ factory = (ADChunkSampleStreamFactory) f;
+ assertEquals(ADChunkSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(
+ FORMAT_SAMPLE_DIR + "ad/" + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<ChunkSample> stream = factory.create(new String[]
+ {"-start", "0", "-end", "1", "-lang", "por", "-encoding", "UTF-8",
+ "-data", sampleFileFullPath})) {
+ ChunkSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<ChunkSample> stream = factory.create(new String[]
+ {"-start", "0", "-end", "1", "-lang", "por", "-encoding", "UTF-8",
+ "-data", sampleFileFullPath + "xyz"})) {
+ ChunkSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamFactoryTest.java
new file mode 100644
index 00000000..3ba1413b
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamFactoryTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.ad;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ADPOSSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<POSSample, ADPOSSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "ad.sample";
+
+ // SUT
+ private ADPOSSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<POSSample, ADPOSSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ADPOSSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<POSSample, ADPOSSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(POSSample.class, "ad");
+ assertInstanceOf(ADPOSSampleStreamFactory.class, f);
+ factory = (ADPOSSampleStreamFactory) f;
+ assertEquals(ADPOSSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(
+ FORMAT_SAMPLE_DIR + "ad/" + SAMPLE_01).getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"true", "false"})
+ void testCreateWithValidParameter(String expandME) throws IOException {
+ try (ObjectStream<POSSample> stream = factory.create(new String[]
+ {"-expandME", expandME, "-lang", "por", "-encoding", "UTF-8",
+ "-data", sampleFileFullPath})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<POSSample> stream = factory.create(new String[]
+ {"-expandME", "false", "-lang", "por", "-encoding", "UTF-8",
+ "-data", sampleFileFullPath + "xyz"})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADParagraphStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADParagraphStreamTest.java
index 1f55a494..486b51f9 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADParagraphStreamTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADParagraphStreamTest.java
@@ -43,7 +43,7 @@ public class ADParagraphStreamTest extends AbstractADSampleStreamTest<SentenceSa
int count = 0;
ADSentenceStream.Sentence paragraph = stream.read();
- paragraph.getRoot();
+ paragraph.root();
while (paragraph != null) {
count++;
paragraph = stream.read();
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..75d9b977
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.ad;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ADSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, ADSentenceSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "ad.sample";
+
+ // SUT
+ private ADSentenceSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, ADSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ADSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, ADSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "ad");
+ assertInstanceOf(ADSentenceSampleStreamFactory.class, f);
+ factory = (ADSentenceSampleStreamFactory) f;
+ assertEquals(ADSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(
+ FORMAT_SAMPLE_DIR + "ad/" + SAMPLE_01).getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"true", "false"})
+ void testCreateWithValidParameter(String includeTitles) throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-includeTitles", includeTitles, "-lang", "por", "-encoding", "UTF-8",
+ "-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-includeTitles", "false", "-lang", "por", "-encoding", "UTF-8",
+ "-data", sampleFileFullPath + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..2e7abf02
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactoryTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.ad;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ADTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, ADTokenSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "ad.sample";
+
+ // SUT
+ private ADTokenSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, ADTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ADTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, ADTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, "ad");
+ assertInstanceOf(ADTokenSampleStreamFactory.class, f);
+ factory = (ADTokenSampleStreamFactory) f;
+ assertEquals(ADTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ sampleFileFullPath = getResourceWithoutPrefix(
+ FORMAT_SAMPLE_DIR + "ad/" + SAMPLE_01).getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"true", "false"})
+ void testCreateWithValidParameter(String includeTitles) throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-splitHyphenatedTokens", includeTitles, "-lang", "por", "-encoding", "UTF-8",
+ "-detokenizer", detokFileFullPath, "-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-splitHyphenatedTokens", "false", "-lang", "por", "-encoding", "UTF-8",
+ "-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADTokenSampleStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADTokenSampleStreamTest.java
index 14039e48..2cd2b34e 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADTokenSampleStreamTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/ADTokenSampleStreamTest.java
@@ -43,10 +43,10 @@ public class ADTokenSampleStreamTest extends AbstractADSampleStreamTest<TokenSam
void setup() throws IOException {
super.setup();
- ADTokenSampleStreamFactory<ADTokenSampleStreamFactory.Parameters> factory =
- new ADTokenSampleStreamFactory<>(ADTokenSampleStreamFactory.Parameters.class);
+ ADTokenSampleStreamFactory factory =
+ new ADTokenSampleStreamFactory(ADTokenSampleStreamFactory.Parameters.class);
- File data = new File(getResource("ad.sample").getFile());
+ File data = new File(getResourceWithoutPrefix("opennlp/tools/formats/ad/ad.sample").getFile());
Assertions.assertNotNull(data);
File dict = new File(getResourceWithoutPrefix("opennlp/tools/tokenize/latin-detokenizer.xml").getFile());
Assertions.assertNotNull(dict);
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ad/AbstractADSampleStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/AbstractADSampleStreamTest.java
index 025915a4..b176696a 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/formats/ad/AbstractADSampleStreamTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ad/AbstractADSampleStreamTest.java
@@ -17,6 +17,7 @@
package opennlp.tools.formats.ad;
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -37,6 +38,7 @@ public abstract class AbstractADSampleStreamTest<T extends Sample> extends Abstr
@BeforeEach
void setup() throws IOException {
- in = new ResourceAsStreamFactory(AbstractADSampleStreamTest.class, FORMATS_BASE_DIR + "ad.sample");
+ in = new ResourceAsStreamFactory(AbstractADSampleStreamTest.class,
+ FORMATS_BASE_DIR + "ad" + File.separator + "ad.sample");
}
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratAnnotationStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratAnnotationStreamTest.java
index f039dd58..1602c908 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratAnnotationStreamTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratAnnotationStreamTest.java
@@ -21,12 +21,14 @@ import java.io.IOException;
import java.util.LinkedHashSet;
import java.util.Set;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import opennlp.tools.util.ObjectStream;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.fail;
public class BratAnnotationStreamTest extends AbstractBratTest {
@@ -49,7 +51,7 @@ public class BratAnnotationStreamTest extends AbstractBratTest {
AnnotationConfiguration annConfig = new AnnotationConfiguration(typeToClassMap);
ObjectStream<BratAnnotation> annStream = creatBratAnnotationStream(annConfig,
"brat/voa-with-entities.ann");
- Assertions.assertNotNull(annStream);
+ assertNotNull(annStream);
validateBratAnnotationStream(annStream, 5, 10, 3, 0, 2, 0);
}
@@ -62,7 +64,7 @@ public class BratAnnotationStreamTest extends AbstractBratTest {
AnnotationConfiguration annConfig = new AnnotationConfiguration(typeToClassMap);
ObjectStream<BratAnnotation> annStream = creatBratAnnotationStream(annConfig,
"brat/voa-with-relations.ann");
- Assertions.assertNotNull(annStream);
+ assertNotNull(annStream);
validateBratAnnotationStream(annStream, 5, 10, 3, 0, 0, 7);
}
@@ -87,16 +89,16 @@ public class BratAnnotationStreamTest extends AbstractBratTest {
BratAnnotation ann;
while ((ann = annStream.read()) != null) {
- Assertions.assertNotNull(ann);
+ assertNotNull(ann);
String type = ann.getType();
- Assertions.assertNotNull(type);
+ assertNotNull(type);
String coveredText = null;
RelationAnnotation rAnnotation = null;
AnnotatorNoteAnnotation aAnnotation = null;
if (ann instanceof SpanAnnotation sAnnotation) {
coveredText = sAnnotation.getCoveredText();
- Assertions.assertNotNull(coveredText);
+ assertNotNull(coveredText);
} else if (ann instanceof RelationAnnotation) {
rAnnotation = (RelationAnnotation) ann;
} else if (ann instanceof AnnotatorNoteAnnotation) {
@@ -122,26 +124,27 @@ public class BratAnnotationStreamTest extends AbstractBratTest {
break;
} case BRAT_TYPE_RELATED: {
relations++;
- Assertions.assertNotNull(rAnnotation);
+ assertNotNull(rAnnotation);
break;
} case BRAT_TYPE_ANNOTATION: {
annotations++;
- Assertions.assertNotNull(aAnnotation);
+ assertNotNull(aAnnotation);
break;
} default: {
fail("Found an unsupported BRAT type!");
}
}
}
- Assertions.assertEquals(expectDates, dates);
- Assertions.assertEquals(expectPersons, persons);
- Assertions.assertEquals(expectLocations, locations);
- Assertions.assertEquals(expectAnnotations, annotations);
- Assertions.assertEquals(expectOrganizations, organizations);
- Assertions.assertEquals(expectRelations, relations);
-
- Assertions.assertArrayEquals(VOA_DATES, annotatedDates.toArray());
- Assertions.assertArrayEquals(VOA_PERSONS, annotatedPersons.toArray());
- Assertions.assertArrayEquals(VOA_LOCATIONS, annotatedLocations.toArray());
+ assertEquals(expectDates, dates);
+ assertEquals(expectPersons, persons);
+ assertEquals(expectLocations, locations);
+ assertEquals(expectAnnotations, annotations);
+ assertEquals(expectOrganizations, organizations);
+ assertEquals(expectOrganizations, organizations);
+ assertEquals(expectRelations, relations);
+
+ assertArrayEquals(VOA_DATES, annotatedDates.toArray());
+ assertArrayEquals(VOA_PERSONS, annotatedPersons.toArray());
+ assertArrayEquals(VOA_LOCATIONS, annotatedLocations.toArray());
}
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratNameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratNameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..edfbd844
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratNameSampleStreamFactoryTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.brat;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EmptySource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.tokenize.TokenizerModel;
+import opennlp.tools.util.DownloadUtil;
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.Span;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.fail;
+
+public class BratNameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, BratNameSampleStreamFactory.Parameters> {
+
+ private static final Path OPENNLP_DIR = Paths.get(System.getProperty("OPENNLP_DOWNLOAD_HOME",
+ System.getProperty("user.home"))).resolve(".opennlp");
+ private static final String TOKENIZER_MODEL_NAME = "opennlp-en-ud-ewt-tokens-1.2-2.5.0.bin";
+
+ // SUT
+ private BratNameSampleStreamFactory factory;
+
+ private static String tokFileFullPath;
+ private String bratFullPath;
+ private String configPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, BratNameSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return bratFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ BratNameSampleStreamFactory.registerFactory();
+ try {
+ // ensure, the model is available locally for later test purposes
+ DownloadUtil.downloadModel("en", DownloadUtil.ModelType.TOKENIZER, TokenizerModel.class);
+ } catch (IOException e) {
+ fail(e.getLocalizedMessage());
+ }
+ tokFileFullPath = new File(OPENNLP_DIR + File.separator + TOKENIZER_MODEL_NAME).getPath();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, BratNameSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "brat");
+ assertInstanceOf(BratNameSampleStreamFactory.class, f);
+ factory = (BratNameSampleStreamFactory) f;
+ assertEquals(BratNameSampleStreamFactory.Parameters.class, factory.getParameters());
+ bratFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "brat/").getPath();
+ configPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "brat/" + "brat-ann.conf").getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"simple", "whitespace"})
+ void testCreateWithValidParameter(String tokType) throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]{"-ruleBasedTokenizer", tokType,
+ "-annotationConfig", configPath, "-bratDataDir", bratFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ Span[] names = sample.getNames();
+ assertNotNull(names);
+ assertEquals(1, names.length);
+ assertEquals("Name", names[0].getType());
+ assertEquals(0, names[0].getStart());
+ if ("whitespace".equals(tokType)) {
+ assertEquals(2, names[0].getEnd());
+ } else {
+ assertEquals(6, names[0].getEnd());
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @EmptySource
+ @ValueSource(strings = {" ", "unknown"})
+ void testCreateWithInvalidRBTokenizer(String tokType) {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]{"-ruleBasedTokenizer", tokType,
+ "-annotationConfig", configPath, "-bratDataDir", bratFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ @Test
+ void testCreateWithMETokenizer() throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]{"-tokenizerModel", tokFileFullPath,
+ "-annotationConfig", configPath, "-bratDataDir", bratFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @EmptySource
+ @ValueSource(strings = {" ", "unknown"})
+ void testCreateWithInvalidMETokenizer(String path) {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]{"-tokenizerModel", path,
+ "-annotationConfig", configPath, "-bratDataDir", bratFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]{"-ruleBasedTokenizer", "whitespace",
+ "-annotationConfig", configPath, "-bratDataDir", bratFullPath + "xyz"})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratNameSampleStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratNameSampleStreamTest.java
index 9ddae805..416edd7d 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratNameSampleStreamTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/brat/BratNameSampleStreamTest.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.Collections;
import java.util.Set;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -31,6 +30,10 @@ import opennlp.tools.sentdetect.NewlineSentenceDetector;
import opennlp.tools.tokenize.WhitespaceTokenizer;
import opennlp.tools.util.ObjectStream;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
public class BratNameSampleStreamTest extends AbstractBratTest {
@BeforeEach
@@ -49,19 +52,19 @@ public class BratNameSampleStreamTest extends AbstractBratTest {
sample = stream.read();
}
- Assertions.assertEquals(8, count);
+ assertEquals(8, count);
}
@Test
void readOverlapFail() {
- Assertions.assertThrows(RuntimeException.class, () -> {
- BratNameSampleStream stream = createNameSampleWith("overlapping",
+ assertThrows(RuntimeException.class, () -> {
+ BratNameSampleStream stream = createNameSampleWith("-overlapping",
null);
NameSample sample = stream.read();
while (sample != null) {
sample = stream.read();
- Assertions.assertNotNull(sample);
+ assertNotNull(sample);
}
});
@@ -69,7 +72,7 @@ public class BratNameSampleStreamTest extends AbstractBratTest {
@Test
void emptySample() {
- Assertions.assertThrows(IllegalArgumentException.class, () -> createNameSampleWith("overlapping",
+ assertThrows(IllegalArgumentException.class, () -> createNameSampleWith("overlapping",
Collections.emptySet()));
}
@@ -84,7 +87,7 @@ public class BratNameSampleStreamTest extends AbstractBratTest {
sample = stream.read();
}
- Assertions.assertEquals(8, count);
+ assertEquals(8, count);
}
private BratNameSampleStream createNameSampleWith(String nameContainsFilter,
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactoryTest.java
new file mode 100644
index 00000000..0e7f499e
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluLemmaSampleStreamFactoryTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.conllu;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.lemmatizer.LemmaSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ConlluLemmaSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<LemmaSample, ConlluLemmaSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "es-ud-sample.conllu";
+
+ // SUT
+ private ConlluLemmaSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<LemmaSample, ConlluLemmaSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConlluLemmaSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<LemmaSample, ConlluLemmaSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(LemmaSample.class, ConlluPOSSampleStreamFactory.CONLLU_FORMAT);
+ assertInstanceOf(ConlluLemmaSampleStreamFactory.class, f);
+ factory = (ConlluLemmaSampleStreamFactory) f;
+ assertEquals(ConlluLemmaSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "conllu/" + SAMPLE_01).getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"u", "x"})
+ void testCreateWithValidParameter(String tagset) throws IOException {
+ try (ObjectStream<LemmaSample> stream = factory.create(
+ new String[]{"-tagset", tagset, "-data", sampleFileFullPath})) {
+ LemmaSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {" ", "y"})
+ void testCreateWithUnknownTagset(String tagset) {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<LemmaSample> stream = factory.create(
+ new String[]{"-tagset", tagset, "-data", sampleFileFullPath})) {
+ assertNotNull(stream.read());
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<LemmaSample> stream = factory.create(new String[]
+ {"-tagset", "u", "-data", sampleFileFullPath + "xyz"})) {
+ LemmaSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactoryTest.java
new file mode 100644
index 00000000..40208bbc
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluPOSSampleStreamFactoryTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.conllu;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ConlluPOSSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<POSSample, ConlluPOSSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "es-ud-sample.conllu";
+
+ // SUT
+ private ConlluPOSSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<POSSample, ConlluPOSSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConlluPOSSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<POSSample, ConlluPOSSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(POSSample.class, ConlluPOSSampleStreamFactory.CONLLU_FORMAT);
+ assertInstanceOf(ConlluPOSSampleStreamFactory.class, f);
+ factory = (ConlluPOSSampleStreamFactory) f;
+ assertEquals(ConlluPOSSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "conllu/" + SAMPLE_01).getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"u", "x"})
+ void testCreateWithValidParameter(String tagset) throws IOException {
+ try (ObjectStream<POSSample> stream = factory.create(
+ new String[]{"-tagset", tagset, "-data", sampleFileFullPath})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {" ", "y"})
+ void testCreateWithUnknownTagset(String tagset) {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<POSSample> stream = factory.create(
+ new String[]{"-tagset", tagset, "-data", sampleFileFullPath})) {
+ assertNotNull(stream.read());
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<POSSample> stream = factory.create(new String[]
+ {"-tagset", "u", "-data", sampleFileFullPath + "xyz"})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..05cb3efa
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.conllu;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ConlluSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, ConlluSentenceSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "es-ud-sample.conllu";
+
+ // SUT
+ private ConlluSentenceSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, ConlluSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConlluSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, ConlluSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, ConlluPOSSampleStreamFactory.CONLLU_FORMAT);
+ assertInstanceOf(ConlluSentenceSampleStreamFactory.class, f);
+ factory = (ConlluSentenceSampleStreamFactory) f;
+ assertEquals(ConlluSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "conllu/" + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(
+ new String[]{"-sentencesPerSample", "5", "-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-sentencesPerSample", "5", "-data", sampleFileFullPath + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..ea6f606c
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/conllu/ConlluTokenSampleStreamFactoryTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.conllu;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class ConlluTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, ConlluTokenSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "es-ud-sample.conllu";
+
+ // SUT
+ private ConlluTokenSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, ConlluTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConlluTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, ConlluTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, ConlluPOSSampleStreamFactory.CONLLU_FORMAT);
+ assertInstanceOf(ConlluTokenSampleStreamFactory.class, f);
+ factory = (ConlluTokenSampleStreamFactory) f;
+ assertEquals(ConlluTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "conllu/" + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..a42cde19
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/NameToSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class NameToSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, NameToSentenceSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "name-data-01.sample";
+
+ // SUT
+ private NameToSentenceSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, NameToSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ NameToSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, NameToSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "namefinder");
+ assertInstanceOf(NameToSentenceSampleStreamFactory.class, f);
+ factory = (NameToSentenceSampleStreamFactory) f;
+ assertEquals(NameToSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath,"-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..7c8ae451
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/NameToTokenSampleStreamFactoryTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class NameToTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, NameToTokenSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "name-data-01.sample";
+
+ // SUT
+ private NameToTokenSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, NameToTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ NameToTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, NameToTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, "namefinder");
+ assertInstanceOf(NameToTokenSampleStreamFactory.class, f);
+ factory = (NameToTokenSampleStreamFactory) f;
+ assertEquals(NameToTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath,"-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..6e92de9d
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/POSToSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class POSToSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, POSToSentenceSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "word-tags-01.sample";
+
+ // SUT
+ private POSToSentenceSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, POSToSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ POSToSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, POSToSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "pos");
+ assertInstanceOf(POSToSentenceSampleStreamFactory.class, f);
+ factory = (POSToSentenceSampleStreamFactory) f;
+ assertEquals(POSToSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath,"-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..2501e8c2
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/POSToTokenSampleStreamFactoryTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class POSToTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, POSToTokenSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "word-tags-01.sample";
+
+ // SUT
+ private POSToTokenSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, POSToTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ POSToTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, POSToTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, "pos");
+ assertInstanceOf(POSToTokenSampleStreamFactory.class, f);
+ factory = (POSToTokenSampleStreamFactory) f;
+ assertEquals(POSToTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath,"-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactoryTest.java
new file mode 100644
index 00000000..e1ca0452
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToPOSSampleStreamFactoryTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class ParseToPOSSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<POSSample, ParseToPOSSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "parse-01.sample";
+
+ // SUT
+ private ParseToPOSSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<POSSample, ParseToPOSSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ParseToPOSSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<POSSample, ParseToPOSSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(POSSample.class, "parse");
+ assertInstanceOf(ParseToPOSSampleStreamFactory.class, f);
+ factory = (ParseToPOSSampleStreamFactory) f;
+ assertEquals(ParseToPOSSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<POSSample> stream = factory.create(new String[]
+ {"-data", sampleFileFullPath})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..a11ce4f5
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ParseToSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, ParseToSentenceSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "parse-01.sample";
+
+ // SUT
+ private ParseToSentenceSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, ParseToSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ParseToSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, ParseToSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "parse");
+ assertInstanceOf(ParseToSentenceSampleStreamFactory.class, f);
+ factory = (ParseToSentenceSampleStreamFactory) f;
+ assertEquals(ParseToSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath,"-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..08ae5211
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactoryTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.convert;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ParseToTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, ParseToTokenSampleStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "parse-01.sample";
+
+ // SUT
+ private ParseToTokenSampleStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, ParseToTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ParseToTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, ParseToTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, "parse");
+ assertInstanceOf(ParseToTokenSampleStreamFactory.class, f);
+ factory = (ParseToTokenSampleStreamFactory) f;
+ assertEquals(ParseToTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath,"-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as different exception is expected!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath + "xyz"})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactoryTest.java
new file mode 100644
index 00000000..14a26733
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStreamFactoryTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.frenchtreebank;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.parser.Parse;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class ConstitParseSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<Parse, ConstitParseSampleStreamFactory.Parameters> {
+
+ // SUT
+ private ConstitParseSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<Parse, ConstitParseSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ ConstitParseSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<Parse, ConstitParseSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(Parse.class, "frenchtreebank");
+ assertInstanceOf(ConstitParseSampleStreamFactory.class, f);
+ factory = (ConstitParseSampleStreamFactory) f;
+ assertEquals(ConstitParseSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "frenchtreebank/").getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<Parse> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ Parse sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @Test
+ @Override // due to different exception
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ try (ObjectStream<Parse> stream = getFactory().create(
+ new String[]{"-data", getDataFilePath() + "xyz"})) {
+ Parse sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactoryTest.java
new file mode 100644
index 00000000..26a1c20f
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankSentenceStreamFactoryTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.irishsentencebank;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class IrishSentenceBankSentenceStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, IrishSentenceBankSentenceStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "irishsentencebank-sample.xml";
+
+ // SUT
+ private IrishSentenceBankSentenceStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, IrishSentenceBankSentenceStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ IrishSentenceBankSentenceStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, IrishSentenceBankSentenceStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "irishsentencebank");
+ assertInstanceOf(IrishSentenceBankSentenceStreamFactory.class, f);
+ factory = (IrishSentenceBankSentenceStreamFactory) f;
+ assertEquals(IrishSentenceBankSentenceStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(
+ FORMAT_SAMPLE_DIR + "irishsentencebank/" + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..ffea30b4
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/irishsentencebank/IrishSentenceBankTokenSampleStreamFactoryTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.irishsentencebank;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class IrishSentenceBankTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, IrishSentenceBankTokenSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "irishsentencebank-sample.xml";
+
+ // SUT
+ private IrishSentenceBankTokenSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, IrishSentenceBankTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ IrishSentenceBankTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, IrishSentenceBankTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, "irishsentencebank");
+ assertInstanceOf(IrishSentenceBankTokenSampleStreamFactory.class, f);
+ factory = (IrishSentenceBankTokenSampleStreamFactory) f;
+ assertEquals(IrishSentenceBankTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(
+ FORMAT_SAMPLE_DIR + "irishsentencebank/" + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactoryTest.java
new file mode 100644
index 00000000..36593b8c
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamFactoryTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.leipzig;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.langdetect.LanguageSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class LeipzigLanguageSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<LanguageSample, LeipzigLanguageSampleStreamFactory.Parameters> {
+
+ // SUT
+ private LeipzigLanguageSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<LanguageSample, LeipzigLanguageSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ LeipzigLanguageSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<LanguageSample, LeipzigLanguageSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(LanguageSample.class, "leipzig");
+ assertInstanceOf(LeipzigLanguageSampleStreamFactory.class, f);
+ factory = (LeipzigLanguageSampleStreamFactory) f;
+ assertEquals(LeipzigLanguageSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "leipzig/samples").getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<LanguageSample> stream = factory.create(new String[]
+ {"-sentencesPerSample", "1","-samplesPerLanguage", "1", "-samplesToSkip", "1",
+ "-sentencesDir", sampleFileFullPath})) {
+ LanguageSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ @Test // Given the sample Leipzig are super small, these parameters should not work!
+ void testCreateWithValidParametersTooLarge() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<LanguageSample> stream = factory.create(new String[]
+ {"-sentencesPerSample", "2","-samplesPerLanguage", "2", "-samplesToSkip", "1",
+ "-sentencesDir", sampleFileFullPath})) {
+ LanguageSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<LanguageSample> stream = factory.create(new String[]
+ {"-sentencesPerSample", "2","-samplesPerLanguage", "2", "-samplesToSkip", "1",
+ "-sentencesDir", sampleFileFullPath + "xyz"})) {
+ LanguageSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactoryTest.java
new file mode 100644
index 00000000..ca94a529
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/letsmt/LetsmtSentenceStreamFactoryTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.letsmt;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class LetsmtSentenceStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, LetsmtSentenceStreamFactory.Parameters> {
+
+ private static final String DETOKENIZER_FILE = "opennlp/tools/tokenize/latin-detokenizer.xml";
+ private static final String SAMPLE_01 = "letsmt-with-words.xml";
+
+ // SUT
+ private LetsmtSentenceStreamFactory factory;
+
+ private String detokFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, LetsmtSentenceStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ LetsmtSentenceStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, LetsmtSentenceStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "letsmt");
+ assertInstanceOf(LetsmtSentenceStreamFactory.class, f);
+ factory = (LetsmtSentenceStreamFactory) f;
+ assertEquals(LetsmtSentenceStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "letsmt/" + SAMPLE_01).getPath();
+ detokFileFullPath = getResourceWithoutPrefix(DETOKENIZER_FILE).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(
+ new String[]{"-detokenizer", detokFileFullPath, "-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactoryTest.java
new file mode 100644
index 00000000..27029cc2
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascNamedEntitySampleStreamFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.masc;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class MascNamedEntitySampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, MascNamedEntitySampleStreamFactory.Parameters> {
+
+ // SUT
+ private MascNamedEntitySampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, MascNamedEntitySampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ MascNamedEntitySampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, MascNamedEntitySampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, Masc.MASC_FORMAT);
+ assertInstanceOf(MascNamedEntitySampleStreamFactory.class, f);
+ factory = (MascNamedEntitySampleStreamFactory) f;
+ assertEquals(MascNamedEntitySampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "masc/").getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"True", "False"})
+ void testCreateWithValidParameter(String recurrentSearch) throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-fileFilter", "fakeMASC", "-recurrentSearch", recurrentSearch,
+ "-data", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note:
+ * Overridden more parameters than the '-data' param is required.
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = getFactory().create(new String[]
+ {"-fileFilter", "fakeMASC", "-recurrentSearch", "True",
+ "-data", getDataFilePath() + "xyz"})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactoryTest.java
new file mode 100644
index 00000000..112d0dbb
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascPOSSampleStreamFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.masc;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class MascPOSSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<POSSample, MascPOSSampleStreamFactory.Parameters> {
+
+ // SUT
+ private MascPOSSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<POSSample, MascPOSSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ MascPOSSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<POSSample, MascPOSSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(POSSample.class, Masc.MASC_FORMAT);
+ assertInstanceOf(MascPOSSampleStreamFactory.class, f);
+ factory = (MascPOSSampleStreamFactory) f;
+ assertEquals(MascPOSSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "masc/").getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"True", "False"})
+ void testCreateWithValidParameter(String recurrentSearch) throws IOException {
+ try (ObjectStream<POSSample> stream = factory.create(new String[]
+ {"-fileFilter", "fakeMASC", "-recurrentSearch", recurrentSearch,
+ "-data", sampleFileFullPath})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note:
+ * Overridden more parameters than the '-data' param is required.
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<POSSample> stream = getFactory().create(new String[]
+ {"-fileFilter", "fakeMASC", "-recurrentSearch", "True",
+ "-data", getDataFilePath() + "xyz"})) {
+ POSSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..c67388dc
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.masc;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class MascSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, MascSentenceSampleStreamFactory.Parameters> {
+
+ // SUT
+ private MascSentenceSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, MascSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ MascSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, MascSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, Masc.MASC_FORMAT);
+ assertInstanceOf(MascSentenceSampleStreamFactory.class, f);
+ factory = (MascSentenceSampleStreamFactory) f;
+ assertEquals(MascSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "masc/").getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"True", "False"})
+ void testCreateWithValidParameter(String recurrentSearch) throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-fileFilter", "fakeMASC", "-sentencesPerSample", "5", "-recurrentSearch", recurrentSearch,
+ "-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note:
+ * Overridden more parameters than the '-data' param is required.
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = getFactory().create(new String[]
+ {"-fileFilter", "fakeMASC", "-sentencesPerSample", "5", "-recurrentSearch", "True",
+ "-data", getDataFilePath() + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactoryTest.java
new file mode 100644
index 00000000..04ea91f2
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/masc/MascTokenSampleStreamFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.masc;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class MascTokenSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<TokenSample, MascTokenSampleStreamFactory.Parameters> {
+
+ // SUT
+ private MascTokenSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<TokenSample, MascTokenSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ MascTokenSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<TokenSample, MascTokenSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(TokenSample.class, Masc.MASC_FORMAT);
+ assertInstanceOf(MascTokenSampleStreamFactory.class, f);
+ factory = (MascTokenSampleStreamFactory) f;
+ assertEquals(MascTokenSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "masc/").getPath();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"True", "False"})
+ void testCreateWithValidParameter(String recurrentSearch) throws IOException {
+ try (ObjectStream<TokenSample> stream = factory.create(new String[]
+ {"-fileFilter", "fakeMASC", "-sentencesPerSample", "5", "-recurrentSearch", recurrentSearch,
+ "-data", sampleFileFullPath})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note:
+ * Overridden more parameters than the '-data' param is required.
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<TokenSample> stream = getFactory().create(new String[]
+ {"-fileFilter", "fakeMASC", "-sentencesPerSample", "5", "-recurrentSearch", "True",
+ "-data", getDataFilePath() + "xyz"})) {
+ TokenSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..1b9b618f
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/moses/MosesSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.moses;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class MosesSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, MosesSentenceSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "moses-tiny.sample";
+
+ // SUT
+ private MosesSentenceSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, MosesSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ MosesSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, MosesSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "moses");
+ assertInstanceOf(MosesSentenceSampleStreamFactory.class, f);
+ factory = (MosesSentenceSampleStreamFactory) f;
+ assertEquals(MosesSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "moses/" + SAMPLE_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(
+ new String[]{"-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..ffe1c0a6
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactoryTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.muc;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.tokenize.TokenizerModel;
+import opennlp.tools.util.DownloadUtil;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.fail;
+
+public class Muc6NameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, Muc6NameSampleStreamFactory.Parameters> {
+
+ private static final Path OPENNLP_DIR = Paths.get(System.getProperty("OPENNLP_DOWNLOAD_HOME",
+ System.getProperty("user.home"))).resolve(".opennlp");
+ private static final String TOKENIZER_MODEL_NAME = "opennlp-en-ud-ewt-tokens-1.2-2.5.0.bin";
+
+ // SUT
+ private Muc6NameSampleStreamFactory factory;
+
+ private static String tokenizerFileFullPath;
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, Muc6NameSampleStreamFactory.Parameters> getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ Muc6NameSampleStreamFactory.registerFactory();
+ try {
+ // ensure, the model is available locally for later test purposes
+ DownloadUtil.downloadModel("en", DownloadUtil.ModelType.TOKENIZER, TokenizerModel.class);
+ } catch (IOException e) {
+ fail(e.getLocalizedMessage());
+ }
+ tokenizerFileFullPath = new File(OPENNLP_DIR + File.separator + TOKENIZER_MODEL_NAME).getPath();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, Muc6NameSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "muc6");
+ assertInstanceOf(Muc6NameSampleStreamFactory.class, f);
+ factory = (Muc6NameSampleStreamFactory) f;
+ assertEquals(Muc6NameSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "muc").getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-tokenizerModel", tokenizerFileFullPath, "-data", sampleFileFullPath + File.separator})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note: Overriding this test case, as more params are required!
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-tokenizerModel", tokenizerFileFullPath, "-data", sampleFileFullPath + "xyz" + File.separator})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactoryTest.java
new file mode 100644
index 00000000..c628caad
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/nkjp/NKJPSentenceSampleStreamFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.nkjp;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class NKJPSentenceSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<SentenceSample, NKJPSentenceSampleStreamFactory.Parameters> {
+
+ private static final String SAMPLE_01 = "ann_segmentation.xml";
+ private static final String TEXT_01 = "text_structure.xml";
+
+ // SUT
+ private NKJPSentenceSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+ private String textFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<SentenceSample, NKJPSentenceSampleStreamFactory.Parameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ NKJPSentenceSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<SentenceSample, NKJPSentenceSampleStreamFactory.Parameters> f =
+ StreamFactoryRegistry.getFactory(SentenceSample.class, "nkjp");
+ assertInstanceOf(NKJPSentenceSampleStreamFactory.class, f);
+ factory = (NKJPSentenceSampleStreamFactory) f;
+ assertEquals(NKJPSentenceSampleStreamFactory.Parameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "nkjp/" + SAMPLE_01).getPath();
+ textFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "nkjp/" + TEXT_01).getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<SentenceSample> stream = factory.create(new String[]
+ {"-textFile", textFileFullPath, "-data", sampleFileFullPath})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
+
+ /*
+ * Note:
+ * Overridden more parameters than the '-data' param is required.
+ */
+ @Test
+ @Override
+ protected void testCreateWithInvalidDataFilePath() {
+ assertThrows(TerminateToolException.class, () -> {
+ try (ObjectStream<SentenceSample> stream = getFactory().create(new String[]
+ {"-textFile", textFileFullPath, "-data", getDataFilePath() + "xyz"})) {
+ SentenceSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ });
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactoryTest.java
new file mode 100644
index 00000000..59194ad8
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactoryTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.ontonotes;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.cmdline.ObjectStreamFactory;
+import opennlp.tools.cmdline.StreamFactoryRegistry;
+import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.formats.AbstractSampleStreamFactory;
+import opennlp.tools.formats.AbstractSampleStreamFactoryTest;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.ObjectStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class OntoNotesNameSampleStreamFactoryTest extends
+ AbstractSampleStreamFactoryTest<NameSample, OntoNotesFormatParameters> {
+
+ // SUT
+ private OntoNotesNameSampleStreamFactory factory;
+
+ private String sampleFileFullPath;
+
+ @Override
+ protected AbstractSampleStreamFactory<NameSample, OntoNotesFormatParameters>
+ getFactory() {
+ return factory;
+ }
+
+ @Override
+ protected String getDataFilePath() {
+ return sampleFileFullPath;
+ }
+
+ @BeforeAll
+ static void initEnv() {
+ OntoNotesNameSampleStreamFactory.registerFactory();
+ }
+
+ @BeforeEach
+ void setUp() {
+ ObjectStreamFactory<NameSample, OntoNotesFormatParameters> f =
+ StreamFactoryRegistry.getFactory(NameSample.class, "ontonotes");
+ assertInstanceOf(OntoNotesNameSampleStreamFactory.class, f);
+ factory = (OntoNotesNameSampleStreamFactory) f;
+ assertEquals(OntoNotesFormatParameters.class, factory.getParameters());
+ sampleFileFullPath = getResourceWithoutPrefix(FORMAT_SAMPLE_DIR + "ontonotes").getPath();
+ }
+
+ @Test
+ void testCreateWithValidParameter() throws IOException {
+ try (ObjectStream<NameSample> stream = factory.create(new String[]
+ {"-ontoNotesDir", sampleFileFullPath})) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ }
+ }
... 653 lines suppressed ...