You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by co...@apache.org on 2012/03/27 18:54:12 UTC
svn commit: r1305900 -
/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DefaultTokenContextGenerator.java
Author: colen
Date: Tue Mar 27 16:54:12 2012
New Revision: 1305900
URL: http://svn.apache.org/viewvc?rev=1305900&view=rev
Log:
OPENNLP-484: Tokenizer had better results if we remove the pabb feature
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DefaultTokenContextGenerator.java
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DefaultTokenContextGenerator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DefaultTokenContextGenerator.java?rev=1305900&r1=1305899&r2=1305900&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DefaultTokenContextGenerator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DefaultTokenContextGenerator.java Tue Mar 27 16:54:12 2012
@@ -105,10 +105,6 @@ public class DefaultTokenContextGenerato
if(index == sentence.length() - 1 && inducedAbbreviations.contains(sentence)) {
preds.add("pabb");
}
-
- if(inducedAbbreviations.contains(sentence)) {
- preds.add("abb");
- }
return preds;
}