You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by co...@apache.org on 2011/09/01 22:12:58 UTC
svn commit: r1164240 - in
/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline:
sentdetect/ tokenizer/
Author: colen
Date: Thu Sep 1 20:12:57 2011
New Revision: 1164240
URL: http://svn.apache.org/viewvc?rev=1164240&view=rev
Log:
OPENNLP-292 Removed case sensitivity command line argument
Modified:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerTrainerTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TrainingParams.java
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java?rev=1164240&r1=1164239&r2=1164240&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java Thu Sep 1 20:12:57 2011
@@ -90,8 +90,8 @@ public final class SentenceDetectorCross
}
try {
- Dictionary abbreviations = SentenceDetectorTrainerTool.loadDict(
- params.getAbbDict(), params.getIsAbbDictCS());
+ Dictionary abbreviations = SentenceDetectorTrainerTool.loadDict(params
+ .getAbbDict());
validator = new SDCrossValidator(params.getLang(), mlParams,
abbreviations, errorListener);
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java?rev=1164240&r1=1164239&r2=1164240&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java Thu Sep 1 20:12:57 2011
@@ -68,11 +68,11 @@ public final class SentenceDetectorTrain
return new SentenceSampleStream(lineStream);
}
- static Dictionary loadDict(File f, boolean caseSensitive) throws IOException {
+ static Dictionary loadDict(File f) throws IOException {
Dictionary dict = null;
if (f != null) {
CmdLineUtil.checkInputFile("abb dict", f);
- dict = new Dictionary(new FileInputStream(f), caseSensitive);
+ dict = new Dictionary(new FileInputStream(f));
}
return dict;
}
@@ -106,7 +106,7 @@ public final class SentenceDetectorTrain
SentenceModel model;
try {
- Dictionary dict = loadDict(params.getAbbDict(), params.getIsAbbDictCS());
+ Dictionary dict = loadDict(params.getAbbDict());
if (mlParams == null) {
model = SentenceDetectorME.train(params.getLang(), sampleStream, true, dict,
params.getCutoff(), params.getIterations());
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java?rev=1164240&r1=1164239&r2=1164240&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java Thu Sep 1 20:12:57 2011
@@ -34,8 +34,4 @@ interface TrainingParams extends BasicTr
@OptionalParameter
File getAbbDict();
- @ParameterDescription(valueName = "true|false", description = "True if the abbreviation dictionary is case sensitive. Default is true.")
- @OptionalParameter(defaultValue = "true")
- Boolean getIsAbbDictCS();
-
}
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java?rev=1164240&r1=1164239&r2=1164240&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java Thu Sep 1 20:12:57 2011
@@ -32,7 +32,6 @@ import opennlp.tools.tokenize.TokenSampl
import opennlp.tools.tokenize.TokenizerCrossValidator;
import opennlp.tools.tokenize.TokenizerEvaluationMonitor;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.eval.FMeasure;
public final class TokenizerCrossValidatorTool implements CmdLineTool {
@@ -87,7 +86,7 @@ public final class TokenizerCrossValidat
}
try {
- Dictionary dict = TokenizerTrainerTool.loadDict(params.getAbbDict(), params.getIsAbbDictCS());
+ Dictionary dict = TokenizerTrainerTool.loadDict(params.getAbbDict());
validator = new opennlp.tools.tokenize.TokenizerCrossValidator(
params.getLang(), dict, params.getAlphaNumOpt(), mlParams, listener);
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerTrainerTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerTrainerTool.java?rev=1164240&r1=1164239&r2=1164240&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerTrainerTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerTrainerTool.java Thu Sep 1 20:12:57 2011
@@ -68,11 +68,11 @@ public final class TokenizerTrainerTool
return new TokenSampleStream(lineStream);
}
- static Dictionary loadDict(File f, boolean caseSensitive) throws IOException {
+ static Dictionary loadDict(File f) throws IOException {
Dictionary dict = null;
if (f != null) {
CmdLineUtil.checkInputFile("abb dict", f);
- dict = new Dictionary(new FileInputStream(f), caseSensitive);
+ dict = new Dictionary(new FileInputStream(f));
}
return dict;
}
@@ -113,7 +113,7 @@ public final class TokenizerTrainerTool
TokenizerModel model;
try {
- Dictionary dict = loadDict(params.getAbbDict(), params.getIsAbbDictCS());
+ Dictionary dict = loadDict(params.getAbbDict());
model = opennlp.tools.tokenize.TokenizerME.train(params.getLang(),
sampleStream, dict, params.getAlphaNumOpt(), mlParams);
} catch (IOException e) {
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TrainingParams.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TrainingParams.java?rev=1164240&r1=1164239&r2=1164240&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TrainingParams.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TrainingParams.java Thu Sep 1 20:12:57 2011
@@ -36,8 +36,4 @@ interface TrainingParams extends BasicTr
@ParameterDescription(valueName = "path", description = "The abbreviation dictionary in XML format.")
@OptionalParameter
File getAbbDict();
-
- @ParameterDescription(valueName = "true|false", description = "True if the abbreviation dictionary is case sensitive. Default is true.")
- @OptionalParameter(defaultValue = "true")
- Boolean getIsAbbDictCS();
}