You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2024/12/21 09:01:30 UTC
(opennlp) branch OPENNLP-1677-Extend-JavaDoc-of-POSTaggerME created (now 35413cfa)
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a change to branch OPENNLP-1677-Extend-JavaDoc-of-POSTaggerME
in repository https://gitbox.apache.org/repos/asf/opennlp.git
at 35413cfa OPENNLP-1677 Extend JavaDoc of POSTaggerME
This branch includes the following new commits:
new 35413cfa OPENNLP-1677 Extend JavaDoc of POSTaggerME
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opennlp) 01/01: OPENNLP-1677 Extend JavaDoc of POSTaggerME
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch OPENNLP-1677-Extend-JavaDoc-of-POSTaggerME
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 35413cfa47a1229702e11a0c497011bd526aaa31
Author: Martin Wiesner <ma...@hs-heilbronn.de>
AuthorDate: Sat Dec 21 10:00:44 2024 +0100
OPENNLP-1677 Extend JavaDoc of POSTaggerME
---
.../java/opennlp/tools/postag/POSTaggerME.java | 74 +++++++++++++++-------
1 file changed, 52 insertions(+), 22 deletions(-)
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
index cd419e67..a85ceb55 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
@@ -50,15 +50,22 @@ import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.featuregen.StringPattern;
/**
- * A {@link POSTagger part-of-speech tagger} that uses maximum entropy.
+ * A {@link POSTagger part-of-speech tagger} implementation that uses maximum entropy.
* <p>
- * Tries to predict whether words are nouns, verbs, or any of 70 other POS tags
+ * Tries to predict whether words are nouns, verbs, or any other {@link POSTagFormat POS tags}
* depending on their surrounding context.
+ *
+ * @see POSModel
+ * @see POSTagFormat
+ * @see POSTagger
*/
public class POSTaggerME implements POSTagger {
private static final Logger logger = LoggerFactory.getLogger(POSTaggerME.class);
+ /**
+ * The default beam size value is 3.
+ */
public static final int DEFAULT_BEAM_SIZE = 3;
private final POSModel modelPackage;
@@ -66,7 +73,7 @@ public class POSTaggerME implements POSTagger {
/**
* The {@link POSContextGenerator feature context generator}.
*/
- protected final POSContextGenerator contextGen;
+ protected final POSContextGenerator cg;
/**
* {@link TagDictionary} used for restricting words to a fixed set of tags.
@@ -140,7 +147,7 @@ public class POSTaggerME implements POSTagger {
modelPackage = model;
- contextGen = factory.getPOSContextGenerator(beamSize);
+ cg = factory.getPOSContextGenerator(beamSize);
tagDictionary = factory.getTagDictionary();
size = beamSize;
@@ -165,14 +172,20 @@ public class POSTaggerME implements POSTagger {
return model.getOutcomes();
}
+ /**
+ * {@inheritDoc}
+ */
@Override
public String[] tag(String[] sentence) {
return this.tag(sentence, null);
}
+ /**
+ * {@inheritDoc}
+ */
@Override
public String[] tag(String[] sentence, Object[] additionalContext) {
- bestSequence = model.bestSequence(sentence, additionalContext, contextGen, sequenceValidator);
+ bestSequence = model.bestSequence(sentence, additionalContext, cg, sequenceValidator);
final List<String> t = bestSequence.getOutcomes();
return convertTags(t);
}
@@ -186,7 +199,7 @@ public class POSTaggerME implements POSTagger {
*/
public String[][] tag(int numTaggings, String[] sentence) {
Sequence[] bestSequences = model.bestSequences(numTaggings, sentence, null,
- contextGen, sequenceValidator);
+ cg, sequenceValidator);
String[][] tags = new String[bestSequences.length][];
for (int si = 0; si < tags.length; si++) {
List<String> t = bestSequences[si].getOutcomes();
@@ -204,18 +217,25 @@ public class POSTaggerME implements POSTagger {
}
}
+ /**
+ * {@inheritDoc}
+ */
@Override
public Sequence[] topKSequences(String[] sentence) {
return this.topKSequences(sentence, null);
}
+ /**
+ * {@inheritDoc}
+ */
@Override
public Sequence[] topKSequences(String[] sentence, Object[] additionalContext) {
- return model.bestSequences(size, sentence, additionalContext, contextGen, sequenceValidator);
+ return model.bestSequences(size, sentence, additionalContext, cg, sequenceValidator);
}
/**
- * Populates the specified array with the probabilities for each tag of the last tagged sentence.
+ * Populates the specified {@code probs} array with the probabilities
+ * for each tag of the last tagged sentence.
*
* @param probs An array to put the probabilities into.
*/
@@ -239,7 +259,7 @@ public class POSTaggerME implements POSTagger {
MaxentModel posModel = modelPackage.getArtifact(POSModel.POS_MODEL_ENTRY_NAME);
if (posModel != null) {
- double[] probs = posModel.eval(contextGen.getContext(index, words.toArray(new String[0]),
+ double[] probs = posModel.eval(cg.getContext(index, words.toArray(new String[0]),
tags.toArray(new String[0]), null));
String[] orderedTags = new String[probs.length];
@@ -263,34 +283,44 @@ public class POSTaggerME implements POSTagger {
}
}
- public static POSModel train(String languageCode,
- ObjectStream<POSSample> samples, TrainingParameters trainParams,
- POSTaggerFactory posFactory) throws IOException {
-
- int beamSize = trainParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, POSTaggerME.DEFAULT_BEAM_SIZE);
-
- POSContextGenerator contextGenerator = posFactory.getPOSContextGenerator();
+ /**
+ * Starts a training of a {@link POSModel} with the given parameters.
+ *
+ * @param languageCode The ISO language code to train the model. Must not be {@code null}.
+ * @param samples The {@link ObjectStream} of {@link POSSample} used as input for training.
+ * @param mlParams The {@link TrainingParameters} for the context of the training process.
+ * @param posFactory The {@link POSTaggerFactory} for creating related objects as defined
+ * via {@code mlParams}.
+ *
+ * @return A valid, trained {@link POSModel} instance.
+ * @throws IOException Thrown if IO errors occurred.
+ */
+ public static POSModel train(String languageCode, ObjectStream<POSSample> samples,
+ TrainingParameters mlParams, POSTaggerFactory posFactory)
+ throws IOException {
- Map<String, String> manifestInfoEntries = new HashMap<>();
+ final int beamSize = mlParams.getIntParameter(
+ BeamSearch.BEAM_SIZE_PARAMETER, POSTaggerME.DEFAULT_BEAM_SIZE);
- TrainerType trainerType = TrainerFactory.getTrainerType(trainParams);
+ final POSContextGenerator contextGenerator = posFactory.getPOSContextGenerator();
+ final TrainerType trainerType = TrainerFactory.getTrainerType(mlParams);
+ final Map<String, String> manifestInfoEntries = new HashMap<>();
MaxentModel posModel = null;
SequenceClassificationModel seqPosModel = null;
if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) {
ObjectStream<Event> es = new POSSampleEventStream(samples, contextGenerator);
- EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams,
- manifestInfoEntries);
+ EventTrainer trainer = TrainerFactory.getEventTrainer(mlParams, manifestInfoEntries);
posModel = trainer.train(es);
} else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) {
POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator);
EventModelSequenceTrainer<POSSample> trainer =
- TrainerFactory.getEventModelSequenceTrainer(trainParams, manifestInfoEntries);
+ TrainerFactory.getEventModelSequenceTrainer(mlParams, manifestInfoEntries);
posModel = trainer.train(ss);
} else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) {
SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer(
- trainParams, manifestInfoEntries);
+ mlParams, manifestInfoEntries);
// TODO: This will probably cause issue, since the feature generator uses the outcomes array