You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by co...@apache.org on 2011/07/21 18:04:03 UTC
svn commit: r1149243 - in /incubator/opennlp/trunk/opennlp-tools/src:
main/java/opennlp/tools/cmdline/sentdetect/
main/java/opennlp/tools/sentdetect/ main/java/opennlp/tools/sentdetect/lang/
main/java/opennlp/tools/util/model/ test/java/opennlp/tools/s...
Author: colen
Date: Thu Jul 21 16:03:57 2011
New Revision: 1149243
URL: http://svn.apache.org/viewvc?rev=1149243&view=rev
Log:
OPENNLP-225 Undo changes: will remove the AbbreviationDictionary.
Removed:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/AbbreviationDictionarySerializer.java
Modified:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/DefaultSDContextGenerator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceModel.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/lang/Factory.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMETest.java
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java?rev=1149243&r1=1149242&r2=1149243&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java Thu Jul 21 16:03:57 2011
@@ -27,7 +27,6 @@ import opennlp.tools.cmdline.CVParams;
import opennlp.tools.cmdline.CmdLineTool;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.TerminateToolException;
-import opennlp.tools.dictionary.AbbreviationDictionary;
import opennlp.tools.sentdetect.SDCrossValidator;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.util.ObjectStream;
@@ -72,16 +71,15 @@ public final class SentenceDetectorCross
trainingDataInFile, encoding);
SDCrossValidator validator;
+
+ if (mlParams == null) {
+ validator = new SDCrossValidator(params.getLang(), params.getCutoff(), params.getIterations());
+ }
+ else {
+ validator = new SDCrossValidator(params.getLang(), mlParams);
+ }
try {
- AbbreviationDictionary dict = SentenceDetectorTrainerTool.loadDict(params.getAbbDict(), params.getIsAbbdictCS());
-
- if (mlParams == null) {
- validator = new SDCrossValidator(params.getLang(), params.getCutoff(), params.getIterations(), dict);
- }
- else {
- validator = new SDCrossValidator(params.getLang(), mlParams, dict);
- }
validator.evaluate(sampleStream, params.getFolds(), params.getMisclassified());
}
catch (IOException e) {
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java?rev=1149243&r1=1149242&r2=1149243&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorTrainerTool.java Thu Jul 21 16:03:57 2011
@@ -29,7 +29,6 @@ import opennlp.tools.cmdline.CmdLineTool
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.cmdline.TrainingToolParams;
-import opennlp.tools.dictionary.AbbreviationDictionary;
import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.sentdetect.SentenceSample;
@@ -68,16 +67,6 @@ public final class SentenceDetectorTrain
return new SentenceSampleStream(lineStream);
}
- static AbbreviationDictionary loadDict(File f, boolean caseSensitive)
- throws IOException {
- AbbreviationDictionary dict = null;
- if (f != null) {
- CmdLineUtil.checkInputFile("abb dict", f);
- dict = new AbbreviationDictionary(new FileInputStream(f), caseSensitive);
- }
- return dict;
- }
-
public void run(String[] args) {
if (!ArgumentParser.validateArguments(args, TrainerToolParams.class)) {
System.err.println(getHelp());
@@ -107,14 +96,13 @@ public final class SentenceDetectorTrain
SentenceModel model;
try {
- AbbreviationDictionary dict = loadDict(params.getAbbDict(), params.getIsAbbdictCS());
-
if (mlParams == null) {
- model = SentenceDetectorME.train(params.getLang(), sampleStream, true,
- dict, params.getCutoff(), params.getIterations());
- } else {
- model = SentenceDetectorME.train(params.getLang(), sampleStream, true,
- dict, mlParams);
+ model = SentenceDetectorME.train(params.getLang(), sampleStream, true, null,
+ params.getCutoff(), params.getIterations());
+ }
+ else {
+ model = SentenceDetectorME.train(params.getLang(), sampleStream, true, null,
+ mlParams);
}
} catch (IOException e) {
CmdLineUtil.printTrainingIoError(e);
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java?rev=1149243&r1=1149242&r2=1149243&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/TrainingParams.java Thu Jul 21 16:03:57 2011
@@ -17,11 +17,7 @@
package opennlp.tools.cmdline.sentdetect;
-import java.io.File;
-
import opennlp.tools.cmdline.BasicTrainingParams;
-import opennlp.tools.cmdline.ArgumentParser.OptionalParameter;
-import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
/**
* TrainingParams for Sentence Detector.
@@ -29,13 +25,6 @@ import opennlp.tools.cmdline.ArgumentPar
* Note: Do not use this class, internal use only!
*/
interface TrainingParams extends BasicTrainingParams {
-
- @ParameterDescription(valueName = "path", description = "The abbreviation dictionary in XML format.")
- @OptionalParameter
- File getAbbDict();
- @ParameterDescription(valueName = "true|false", description = "True if the abbreviation dictionary is case sensitive. Default is true.")
- @OptionalParameter(defaultValue = "true")
- Boolean getIsAbbdictCS();
}
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/DefaultSDContextGenerator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/DefaultSDContextGenerator.java?rev=1149243&r1=1149242&r2=1149243&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/DefaultSDContextGenerator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/DefaultSDContextGenerator.java Thu Jul 21 16:03:57 2011
@@ -65,11 +65,7 @@ public class DefaultSDContextGenerator i
* @param eosCharacters
*/
public DefaultSDContextGenerator(Set<String> inducedAbbreviations, char[] eosCharacters) {
- if(inducedAbbreviations != null) { // it can be null
- this.inducedAbbreviations = inducedAbbreviations;
- } else {
- this.inducedAbbreviations = Collections.<String>emptySet();
- }
+ this.inducedAbbreviations = inducedAbbreviations;
this.eosCharacters = eosCharacters;
buf = new StringBuffer();
collectFeats = new ArrayList<String>();
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java?rev=1149243&r1=1149242&r2=1149243&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java Thu Jul 21 16:03:57 2011
@@ -19,7 +19,6 @@ package opennlp.tools.sentdetect;
import java.io.IOException;
-import opennlp.tools.dictionary.AbbreviationDictionary;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.eval.CrossValidationPartitioner;
@@ -34,7 +33,6 @@ public class SDCrossValidator {
private final int cutoff;
private final int iterations;
- private final AbbreviationDictionary abbDict;
private final TrainingParameters params;
@@ -42,29 +40,18 @@ public class SDCrossValidator {
public SDCrossValidator(String languageCode, int cutoff, int iterations) {
- this(languageCode, cutoff, iterations, null);
- }
-
- public SDCrossValidator(String languageCode, TrainingParameters params) {
- this(languageCode, params, null);
- }
-
- public SDCrossValidator(String languageCode, int cutoff, int iterations, AbbreviationDictionary dict) {
-
this.languageCode = languageCode;
this.cutoff = cutoff;
this.iterations = iterations;
- this.abbDict = dict;
params = null;
}
- public SDCrossValidator(String languageCode, TrainingParameters params, AbbreviationDictionary dict) {
+ public SDCrossValidator(String languageCode, TrainingParameters params) {
this.languageCode = languageCode;
this.params = params;
cutoff = -1;
iterations = -1;
- this.abbDict = dict;
}
public SDCrossValidator(String languageCode) {
@@ -112,10 +99,10 @@ public class SDCrossValidator {
SentenceModel model;
if (params == null) {
- model = SentenceDetectorME.train(languageCode, trainingSampleStream, true, this.abbDict, cutoff, iterations);
+ model = SentenceDetectorME.train(languageCode, trainingSampleStream, true, null, cutoff, iterations);
}
else {
- model = SentenceDetectorME.train(languageCode, trainingSampleStream, true, this.abbDict, params);
+ model = SentenceDetectorME.train(languageCode, trainingSampleStream, true, null, params);
}
// do testing
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java?rev=1149243&r1=1149242&r2=1149243&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java Thu Jul 21 16:03:57 2011
@@ -28,7 +28,6 @@ import opennlp.model.AbstractModel;
import opennlp.model.EventStream;
import opennlp.model.MaxentModel;
import opennlp.model.TrainUtil;
-import opennlp.tools.dictionary.AbbreviationDictionary;
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.sentdetect.lang.Factory;
import opennlp.tools.util.ObjectStream;
@@ -89,7 +88,7 @@ public class SentenceDetectorME implemen
public SentenceDetectorME(SentenceModel model, Factory factory) {
this.model = model.getMaxentModel();
- cgen = factory.createSentenceContextGenerator(model.getLanguage(), model.getAbbreviationDictionary());
+ cgen = factory.createSentenceContextGenerator(model.getLanguage());
scanner = factory.createEndOfSentenceScanner(model.getLanguage());
useTokenEnd = model.useTokenEnd();
}
@@ -257,8 +256,7 @@ public class SentenceDetectorME implemen
return true;
}
-
- @Deprecated // should use AbbreviationDictionary (deprecated in 1.5.2)
+
public static SentenceModel train(String languageCode, ObjectStream<SentenceSample> samples,
boolean useTokenEnd, Dictionary abbreviations, TrainingParameters mlParams) throws IOException {
@@ -277,7 +275,6 @@ public class SentenceDetectorME implemen
useTokenEnd, abbreviations, manifestInfoEntries);
}
- @Deprecated // should use AbbreviationDictionary
public static SentenceModel train(String languageCode, ObjectStream<SentenceSample> samples,
boolean useTokenEnd, Dictionary abbreviations, int cutoff, int iterations) throws IOException {
@@ -289,50 +286,8 @@ public class SentenceDetectorME implemen
return train(languageCode, samples, useTokenEnd, abbreviations, mlParams);
}
- @Deprecated // should use AbbreviationDictionary
public static SentenceModel train(String languageCode, ObjectStream<SentenceSample> samples,
boolean useTokenEnd, Dictionary abbreviations) throws IOException {
return train(languageCode, samples, useTokenEnd, abbreviations,5,100);
}
-
- public static SentenceModel train(String languageCode,
- ObjectStream<SentenceSample> samples, boolean useTokenEnd,
- AbbreviationDictionary abbreviations, TrainingParameters mlParams)
- throws IOException {
-
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
-
- Factory factory = new Factory();
-
- // TODO: Fix the EventStream to throw exceptions when training goes wrong
- EventStream eventStream = new SDEventStream(samples,
- factory.createSentenceContextGenerator(languageCode, abbreviations),
- factory.createEndOfSentenceScanner(languageCode));
-
- AbstractModel sentModel = TrainUtil.train(eventStream,
- mlParams.getSettings(), manifestInfoEntries);
-
- return new SentenceModel(languageCode, sentModel, useTokenEnd,
- abbreviations, manifestInfoEntries);
- }
-
- public static SentenceModel train(String languageCode,
- ObjectStream<SentenceSample> samples, boolean useTokenEnd,
- AbbreviationDictionary abbreviations, int cutoff, int iterations)
- throws IOException {
-
- TrainingParameters mlParams = new TrainingParameters();
- mlParams.put(TrainingParameters.ALGORITHM_PARAM, "MAXENT");
- mlParams.put(TrainingParameters.ITERATIONS_PARAM,
- Integer.toString(iterations));
- mlParams.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(cutoff));
-
- return train(languageCode, samples, useTokenEnd, abbreviations, mlParams);
- }
-
- public static SentenceModel train(String languageCode,
- ObjectStream<SentenceSample> samples, boolean useTokenEnd,
- AbbreviationDictionary abbreviations) throws IOException {
- return train(languageCode, samples, useTokenEnd, abbreviations, 5, 100);
- }
}
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceModel.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceModel.java?rev=1149243&r1=1149242&r2=1149243&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceModel.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceModel.java Thu Jul 21 16:03:57 2011
@@ -29,7 +29,6 @@ import java.util.Map;
import opennlp.model.AbstractModel;
import opennlp.model.GenericModelReader;
import opennlp.model.MaxentModel;
-import opennlp.tools.dictionary.AbbreviationDictionary;
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.model.BaseModel;
@@ -45,41 +44,11 @@ public class SentenceModel extends BaseM
private static final String COMPONENT_NAME = "SentenceDetectorME";
private static final String MAXENT_MODEL_ENTRY_NAME = "sent.model";
-
- @Deprecated // should use abbdict (deprecated in 1.5.2)
private static final String ABBREVIATIONS_ENTRY_NAME = "abbreviations.dictionary";
-
- public static final String ABBDICT_ENTRY_NAME = "dict.abbdict";
private static final String TOKEN_END_PROPERTY = "useTokenEnd";
public SentenceModel(String languageCode, AbstractModel sentModel,
- boolean useTokenEnd, AbbreviationDictionary abbreviations, Map<String, String> manifestInfoEntries) {
-
- super(COMPONENT_NAME, languageCode, manifestInfoEntries);
-
- if (sentModel == null)
- throw new IllegalArgumentException("sentModel param must not be null!");
-
- if (!isModelCompatible(sentModel))
- throw new IllegalArgumentException("The maxent model is not compatible!");
-
- artifactMap.put(MAXENT_MODEL_ENTRY_NAME, sentModel);
-
- setManifestProperty(TOKEN_END_PROPERTY, Boolean.toString(useTokenEnd));
-
- // Abbreviations are optional
- if (abbreviations != null)
- artifactMap.put(ABBDICT_ENTRY_NAME, abbreviations);
- }
-
- public SentenceModel(String languageCode, AbstractModel sentModel,
- boolean useTokenEnd, AbbreviationDictionary abbreviations) {
- this (languageCode, sentModel, useTokenEnd, abbreviations, null);
- }
-
- @Deprecated //should use AbbreviationDictionary (1.5.2)
- public SentenceModel(String languageCode, AbstractModel sentModel,
boolean useTokenEnd, Dictionary abbreviations, Map<String, String> manifestInfoEntries) {
super(COMPONENT_NAME, languageCode, manifestInfoEntries);
@@ -99,7 +68,6 @@ public class SentenceModel extends BaseM
artifactMap.put(ABBREVIATIONS_ENTRY_NAME, abbreviations);
}
- @Deprecated //should use Abbreviation
public SentenceModel(String languageCode, AbstractModel sentModel,
boolean useTokenEnd, Dictionary abbreviations) {
this (languageCode, sentModel, useTokenEnd, abbreviations, null);
@@ -131,26 +99,15 @@ public class SentenceModel extends BaseM
if (abbreviationsEntry != null && !(abbreviationsEntry instanceof Dictionary)) {
throw new InvalidFormatException("Abbreviations dictionary has wrong type!");
}
-
- Object abbdictEntry = artifactMap.get(ABBDICT_ENTRY_NAME);
-
- if (abbdictEntry != null && !(abbdictEntry instanceof AbbreviationDictionary)) {
- throw new InvalidFormatException("Abbreviations dictionary has wrong type!");
- }
}
public AbstractModel getMaxentModel() {
return (AbstractModel) artifactMap.get(MAXENT_MODEL_ENTRY_NAME);
}
- @Deprecated //should use Abbreviation
public Dictionary getAbbreviations() {
return (Dictionary) artifactMap.get(ABBREVIATIONS_ENTRY_NAME);
}
-
- public AbbreviationDictionary getAbbreviationDictionary() {
- return (AbbreviationDictionary) artifactMap.get(ABBDICT_ENTRY_NAME);
- }
public boolean useTokenEnd() {
return Boolean.parseBoolean(getManifestProperty(TOKEN_END_PROPERTY));
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/lang/Factory.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/lang/Factory.java?rev=1149243&r1=1149242&r2=1149243&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/lang/Factory.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/lang/Factory.java Thu Jul 21 16:03:57 2011
@@ -18,9 +18,6 @@
package opennlp.tools.sentdetect.lang;
-import java.util.Collections;
-
-import opennlp.tools.dictionary.AbbreviationDictionary;
import opennlp.tools.sentdetect.DefaultEndOfSentenceScanner;
import opennlp.tools.sentdetect.DefaultSDContextGenerator;
import opennlp.tools.sentdetect.EndOfSentenceScanner;
@@ -28,27 +25,21 @@ import opennlp.tools.sentdetect.SDContex
import opennlp.tools.sentdetect.lang.th.SentenceContextGenerator;
public class Factory {
-
- public static final char[] defaultEosCharacters = new char[] { '.', '!', '?' };
public EndOfSentenceScanner createEndOfSentenceScanner(String languageCode) {
if ("th".equals(languageCode)) {
return new DefaultEndOfSentenceScanner(new char[]{' ','\n'});
}
- return new DefaultEndOfSentenceScanner(defaultEosCharacters);
+ return new DefaultEndOfSentenceScanner(new char[]{'.', '!', '?'});
}
-
- public SDContextGenerator createSentenceContextGenerator(String languageCode, AbbreviationDictionary dict) {
+
+ public SDContextGenerator createSentenceContextGenerator(String languageCode) {
+
if ("th".equals(languageCode)) {
return new SentenceContextGenerator();
}
- return new DefaultSDContextGenerator(dict, defaultEosCharacters);
- }
-
- @Deprecated // always pass the abb dictionary, null is allowed.
- public SDContextGenerator createSentenceContextGenerator(String languageCode) {
- return new DefaultSDContextGenerator(Collections.<String>emptySet(), defaultEosCharacters);
+ return new DefaultSDContextGenerator(new char[]{'.', '!', '?'});
}
}
\ No newline at end of file
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java?rev=1149243&r1=1149242&r2=1149243&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java Thu Jul 21 16:03:57 2011
@@ -177,7 +177,6 @@ public abstract class BaseModel {
GenericModelSerializer.register(serializers);
PropertiesSerializer.register(serializers);
DictionarySerializer.register(serializers);
- AbbreviationDictionarySerializer.register(serializers);
return serializers;
}
Modified: incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMETest.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMETest.java?rev=1149243&r1=1149242&r2=1149243&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMETest.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorMETest.java Thu Jul 21 16:03:57 2011
@@ -24,7 +24,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import opennlp.tools.dictionary.AbbreviationDictionary;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
@@ -42,7 +41,7 @@ public class SentenceDetectorMETest {
"/opennlp/tools/sentdetect/Sentences.txt");
SentenceModel sentdetectModel = SentenceDetectorME.train(
- "en", new SentenceSampleStream(new PlainTextByLineStream(new InputStreamReader(in))), true, createAbbDict(), 100, 0);
+ "en", new SentenceSampleStream(new PlainTextByLineStream(new InputStreamReader(in))), true, null, 100, 0);
assertEquals("en", sentdetectModel.getLanguage());
@@ -116,9 +115,4 @@ public class SentenceDetectorMETest {
assertEquals(new Span(0, 15), pos[0]);
assertEquals(new Span(16, 56), pos[1]);
}
-
- private AbbreviationDictionary createAbbDict() {
- AbbreviationDictionary reference = new AbbreviationDictionary();
- return reference;
- }
}