You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by co...@apache.org on 2012/02/23 04:43:29 UTC
svn commit: r1292637 - in
/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools:
cmdline/sentdetect/ sentdetect/
Author: colen
Date: Thu Feb 23 03:43:28 2012
New Revision: 1292637
URL: http://svn.apache.org/viewvc?rev=1292637&view=rev
Log:
OPENNLP-434: Changed cross validator and command line tools to use the SentenceDetectorFactory
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java?rev=1292637&r1=1292636&r2=1292637&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java Thu Feb 23 03:43:28 2012
@@ -27,6 +27,7 @@ import opennlp.tools.cmdline.sentdetect.
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.sentdetect.SDCrossValidator;
import opennlp.tools.sentdetect.SentenceDetectorEvaluationMonitor;
+import opennlp.tools.sentdetect.SentenceDetectorFactory;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.util.eval.FMeasure;
import opennlp.tools.util.model.ModelUtil;
@@ -66,8 +67,10 @@ public final class SentenceDetectorCross
try {
Dictionary abbreviations = SentenceDetectorTrainerTool.loadDict(params.getAbbDict());
- validator = new SDCrossValidator(factory.getLang(), mlParams,
- abbreviations, eos, errorListener);
+ SentenceDetectorFactory sdFactory = SentenceDetectorFactory.create(
+ params.getFactory(), factory.getLang(), true, abbreviations, eos);
+ validator = new SDCrossValidator(factory.getLang(), mlParams, sdFactory,
+ errorListener);
validator.evaluate(sampleStream, params.getFolds());
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java?rev=1292637&r1=1292636&r2=1292637&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java Thu Feb 23 03:43:28 2012
@@ -28,6 +28,7 @@ import opennlp.tools.cmdline.TerminateTo
import opennlp.tools.cmdline.params.TrainingToolParams;
import opennlp.tools.cmdline.sentdetect.SentenceDetectorTrainerTool.TrainerToolParams;
import opennlp.tools.dictionary.Dictionary;
+import opennlp.tools.sentdetect.SentenceDetectorFactory;
import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.sentdetect.SentenceSample;
@@ -82,8 +83,10 @@ public final class SentenceDetectorTrain
try {
Dictionary dict = loadDict(params.getAbbDict());
- model = SentenceDetectorME.train(factory.getLang(), sampleStream, true,
- dict, eos, mlParams);
+ SentenceDetectorFactory sdFactory = SentenceDetectorFactory.create(
+ params.getFactory(), factory.getLang(), true, dict, eos);
+ model = SentenceDetectorME.train(factory.getLang(), sampleStream,
+ sdFactory, mlParams);
} catch (IOException e) {
throw new TerminateToolException(-1, "IO error while reading training data or indexing data: " + e.getMessage());
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java?rev=1292637&r1=1292636&r2=1292637&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java Thu Feb 23 03:43:28 2012
@@ -37,4 +37,8 @@ interface TrainingParams extends BasicTr
@ParameterDescription(valueName = "string", description = "EOS characters.")
@OptionalParameter
String getEosChars();
+
+ @ParameterDescription(valueName = "factoryName", description = "A sub-class of SentenceDetectorFactory where to get implementation and resources.")
+ @OptionalParameter
+ String getFactory();
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java?rev=1292637&r1=1292636&r2=1292637&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java Thu Feb 23 03:43:28 2012
@@ -33,35 +33,40 @@ public class SDCrossValidator {
private final String languageCode;
- private final Dictionary abbreviations;
-
private final TrainingParameters params;
private FMeasure fmeasure = new FMeasure();
private SentenceDetectorEvaluationMonitor[] listeners;
-
- private char[] eosCharacters;
+
+ private SentenceDetectorFactory sdFactory;
public SDCrossValidator(String languageCode, TrainingParameters params,
- Dictionary abbreviations, char[] eosCharacters, SentenceDetectorEvaluationMonitor... listeners) {
+ SentenceDetectorFactory sdFactory,
+ SentenceDetectorEvaluationMonitor... listeners) {
this.languageCode = languageCode;
this.params = params;
- this.abbreviations = abbreviations;
- this.eosCharacters = eosCharacters;
this.listeners = listeners;
+ this.sdFactory = sdFactory;
}
/**
- * @deprecated use {@link #SDCrossValidator(String, TrainingParameters)}
- * instead and pass in a TrainingParameters object.
+ * @deprecated Use
+ * {@link #SDCrossValidator(String, TrainingParameters, SentenceDetectorFactory, SentenceDetectorEvaluationMonitor...)}
+ * and pass in a {@link SentenceDetectorFactory}.
*/
public SDCrossValidator(String languageCode, int cutoff, int iterations) {
this(languageCode, ModelUtil.createTrainingParameters(cutoff, iterations));
}
+ /**
+ * @deprecated Use
+ * {@link #SDCrossValidator(String, TrainingParameters, SentenceDetectorFactory, SentenceDetectorEvaluationMonitor...)}
+ * and pass in a {@link SentenceDetectorFactory}.
+ */
public SDCrossValidator(String languageCode, TrainingParameters params) {
- this(languageCode, params, (Dictionary) null, null);
+ this(languageCode, params, new SentenceDetectorFactory(languageCode, true,
+ null, null));
}
/**
@@ -71,14 +76,24 @@ public class SDCrossValidator {
@Deprecated
public SDCrossValidator(String languageCode, int cutoff, int iterations, Dictionary abbreviations) {
this(languageCode, ModelUtil.createTrainingParameters(cutoff, iterations),
- abbreviations, null);
+ new SentenceDetectorFactory(languageCode, true, abbreviations, null));
}
+ /**
+ * @deprecated use
+ * {@link #SDCrossValidator(String, TrainingParameters, Dictionary, SentenceDetectorEvaluationMonitor...)}
+ * instead and pass in a TrainingParameters object.
+ */
public SDCrossValidator(String languageCode, TrainingParameters params,
SentenceDetectorEvaluationMonitor... listeners) {
- this(languageCode, params, null, null, listeners);
+ this(languageCode, params, new SentenceDetectorFactory(languageCode, true,
+ null, null), listeners);
}
+ /**
+ * @deprecated use {@link #SDCrossValidator(String, TrainingParameters, Dictionary, SentenceDetectorEvaluationMonitor...)}
+ * instead and pass in a TrainingParameters object.
+ */
public SDCrossValidator(String languageCode) {
this(languageCode, 5, 100);
}
@@ -106,7 +121,7 @@ public class SDCrossValidator {
SentenceModel model;
model = SentenceDetectorME.train(languageCode, trainingSampleStream,
- true, abbreviations, eosCharacters, params);
+ sdFactory, params);
// do testing
SentenceDetectorEvaluator evaluator = new SentenceDetectorEvaluator(