You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by co...@apache.org on 2012/04/04 20:11:23 UTC
svn commit: r1309510 - in /opennlp/trunk/opennlp-tools:
lang/pt/tokenizer/pt-detokenizer.xml
src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
src/main/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactory.java
Author: colen
Date: Wed Apr 4 18:11:22 2012
New Revision: 1309510
URL: http://svn.apache.org/viewvc?rev=1309510&view=rev
Log:
OPENNLP-481: Instead of using a custom detokenizer we should use the new MERGE_BOTH option
Modified:
opennlp/trunk/opennlp-tools/lang/pt/tokenizer/pt-detokenizer.xml
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactory.java
Modified: opennlp/trunk/opennlp-tools/lang/pt/tokenizer/pt-detokenizer.xml
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/lang/pt/tokenizer/pt-detokenizer.xml?rev=1309510&r1=1309509&r2=1309510&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/lang/pt/tokenizer/pt-detokenizer.xml (original)
+++ opennlp/trunk/opennlp-tools/lang/pt/tokenizer/pt-detokenizer.xml Wed Apr 4 18:11:22 2012
@@ -89,4 +89,7 @@ under the License.
<entry operation="MOVE_RIGHT">
<token>#</token>
</entry>
+ <entry operation="MOVE_BOTH">
+ <token>-</token>
+ </entry>
</dictionary>
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java?rev=1309510&r1=1309509&r2=1309510&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java Wed Apr 4 18:11:22 2012
@@ -22,7 +22,6 @@ import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
@@ -71,7 +70,8 @@ public class ADNameSampleStream implemen
private static final Pattern whitespacePattern = Pattern.compile("\\s+");
private static final Pattern underlinePattern = Pattern.compile("[_]+");
- private static final Pattern alphanumericPattern = Pattern.compile("^[\\p{L}\\p{Nd}-]+$");
+ private static final Pattern hyphenPattern = Pattern.compile("((\\p{L}+)-$)|(^-(\\p{L}+)(.*))|((\\p{L}+)-(\\p{L}+)(.*))");
+ private static final Pattern alphanumericPattern = Pattern.compile("^[\\p{L}\\p{Nd}]+$");
/**
* Map to the Arvores Deitadas types to our types. It is read-only.
@@ -350,7 +350,8 @@ public class ADNameSampleStream implemen
return out;
}
- private Collection<? extends String> processTok(String tok) {
+ private List<String> processTok(String tok) {
+ boolean tokAdded = false;
String original = tok;
List<String> out = new ArrayList<String>();
LinkedList<String> suffix = new LinkedList<String>();
@@ -365,15 +366,52 @@ public class ADNameSampleStream implemen
tok = tok.substring(0, tok.length() - 1);
}
- if(!original.equals(tok) && tok.length() > 1 && !alphanumericPattern.matcher(tok).matches()) {
- out.addAll(processTok(tok));
- } else {
- out.add(tok);
+ // lets split all hyphens
+ if (tok.contains("-") && tok.length() > 1) {
+ Matcher matcher = hyphenPattern.matcher(tok);
+
+ String firstTok = null;
+ String hyphen = "-";
+ String secondTok = null;
+ String rest = null;
+
+ if (matcher.matches()) {
+ if (matcher.group(1) != null) {
+ firstTok = matcher.group(2);
+ } else if (matcher.group(3) != null) {
+ secondTok = matcher.group(4);
+ rest = matcher.group(5);
+ } else if (matcher.group(6) != null) {
+ firstTok = matcher.group(7);
+ secondTok = matcher.group(8);
+ rest = matcher.group(9);
+ }
+
+ addIfNotEmpty(firstTok, out);
+ addIfNotEmpty(hyphen, out);
+ addIfNotEmpty(secondTok, out);
+ addIfNotEmpty(rest, out);
+ tokAdded = true;
+ }
+ }
+ if (!tokAdded) {
+ if (!original.equals(tok) && tok.length() > 1
+ && !alphanumericPattern.matcher(tok).matches()) {
+ out.addAll(processTok(tok));
+ } else {
+ out.add(tok);
+ }
}
out.addAll(suffix);
return out;
}
+ private void addIfNotEmpty(String firstTok, List<String> out) {
+ if (firstTok != null && firstTok.length() > 0) {
+ out.addAll(processTok(firstTok));
+ }
+ }
+
/**
* Parse a NER tag in Arvores Deitadas format.
*
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactory.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactory.java?rev=1309510&r1=1309509&r2=1309510&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactory.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADTokenSampleStreamFactory.java Wed Apr 4 18:11:22 2012
@@ -17,21 +17,12 @@
package opennlp.tools.formats.ad;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.regex.Pattern;
-
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.StreamFactoryRegistry;
-import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.cmdline.params.DetokenizerParameter;
import opennlp.tools.formats.DetokenizerSampleStreamFactory;
import opennlp.tools.formats.NameToTokenSampleStream;
import opennlp.tools.namefind.NameSample;
-import opennlp.tools.tokenize.DetokenizationDictionary;
-import opennlp.tools.tokenize.Detokenizer;
-import opennlp.tools.tokenize.DictionaryDetokenizer;
import opennlp.tools.tokenize.TokenSample;
import opennlp.tools.util.ObjectStream;
@@ -65,45 +56,4 @@ public class ADTokenSampleStreamFactory
ADNameSampleStreamFactory.Parameters.class));
return new NameToTokenSampleStream(createDetokenizer(params), samples);
}
-
- protected Detokenizer createDetokenizer(DetokenizerParameter p) {
- try {
- return new ADDictionaryDetokenizer(new DetokenizationDictionary(
- new FileInputStream(new File(p.getDetokenizer()))));
- } catch (IOException e) {
- throw new TerminateToolException(-1,
- "IO error while loading detokenizer dict: " + e.getMessage());
- }
- }
-
- static class ADDictionaryDetokenizer extends DictionaryDetokenizer {
-
- public ADDictionaryDetokenizer(DetokenizationDictionary dict) {
- super(dict);
- }
-
- @Override
- public DetokenizationOperation[] detokenize(String[] tokens) {
- DetokenizationOperation[] operations = super.detokenize(tokens);
- for (int i = 0; i < tokens.length; i++) {
- if (operations[i].equals(DetokenizationOperation.NO_OPERATION)
- && isMergeToRight(tokens[i])) {
- operations[i] = DetokenizationOperation.MERGE_TO_RIGHT;
- }
- }
- return operations;
- }
-
- private static final Pattern hyphenPattern = Pattern
- .compile(".*?[\\p{L}]-$");
-
- private boolean isMergeToRight(String token) {
- if (token != null) {
- if (hyphenPattern.matcher(token).matches()) {
- return true;
- }
- }
- return false;
- }
- }
}