You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2012/04/20 11:42:23 UTC
svn commit: r1328283 - in
/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools:
cmdline/tokenizer/DictionaryDetokenizerTool.java tokenize/Detokenizer.java
tokenize/DictionaryDetokenizer.java
Author: joern
Date: Fri Apr 20 09:42:23 2012
New Revision: 1328283
URL: http://svn.apache.org/viewvc?rev=1328283&view=rev
Log:
OPENNLP-501 Added detokenization method which outputs a detokenized string.
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/DictionaryDetokenizerTool.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DictionaryDetokenizer.java
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/DictionaryDetokenizerTool.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/DictionaryDetokenizerTool.java?rev=1328283&r1=1328282&r2=1328283&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/DictionaryDetokenizerTool.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/DictionaryDetokenizerTool.java Fri Apr 20 09:42:23 2012
@@ -26,7 +26,6 @@ import opennlp.tools.cmdline.CLI;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.PerformanceMonitor;
import opennlp.tools.tokenize.Detokenizer;
-import opennlp.tools.tokenize.Detokenizer.DetokenizationOperation;
import opennlp.tools.tokenize.DictionaryDetokenizer;
import opennlp.tools.tokenize.WhitespaceTokenizer;
import opennlp.tools.util.ObjectStream;
@@ -37,47 +36,6 @@ public final class DictionaryDetokenizer
public String getHelp() {
return "Usage: " + CLI.CMD + " " + getName() + " detokenizerDictionary";
}
-
- public static String detokenize(String tokens[], DetokenizationOperation operations[]) {
-
- if (tokens.length != operations.length)
- throw new IllegalArgumentException("tokens and operations array must have same length!");
-
-
- StringBuilder untokenizedString = new StringBuilder();
-
- for (int i = 0; i < tokens.length; i++) {
-
- // attach token to string buffer
- untokenizedString.append(tokens[i]);
-
- boolean isAppendSpace;
-
- // if this token is the last token do not attach a space
- if (i + 1 == operations.length) {
- isAppendSpace = false;
- }
- // if next token move left, no space after this token,
- // its safe to access next token
- else if (operations[i + 1].equals(DetokenizationOperation.MERGE_TO_LEFT)
- || operations[i + 1].equals(DetokenizationOperation.MERGE_BOTH)) {
- isAppendSpace = false;
- }
- // if this token is move right, no space
- else if (operations[i].equals(DetokenizationOperation.MERGE_TO_RIGHT)
- || operations[i].equals(DetokenizationOperation.MERGE_BOTH)) {
- isAppendSpace = false;
- }
- else {
- isAppendSpace = true;
- }
-
- if (isAppendSpace)
- untokenizedString.append(' ');
- }
-
- return untokenizedString.toString();
- }
public void run(String[] args) {
@@ -102,9 +60,7 @@ public final class DictionaryDetokenizer
// white space tokenize line
String tokens[] = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine);
- DetokenizationOperation operations[] = detokenizer.detokenize(tokens);
-
- System.out.println(detokenize(tokens, operations));
+ System.out.println(detokenizer.detokenize(tokens, null));
perfMon.incrementCounter();
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java?rev=1328283&r1=1328282&r2=1328283&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java Fri Apr 20 09:42:23 2012
@@ -58,4 +58,16 @@ public interface Detokenizer {
* @return the merge operations to detokenize the input tokens.
*/
DetokenizationOperation[] detokenize(String tokens[]);
+
+ /**
+ * Detokenize the input tokens into a String. Tokens which
+ * are connected without a space inbetween can be separated by
+ * a split marker.
+ *
+ * @param tokens
+ * @param splitMarker the split marker or null
+ *
+ * @return
+ */
+ String detokenize(String tokens[], String splitMarker);
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DictionaryDetokenizer.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DictionaryDetokenizer.java?rev=1328283&r1=1328282&r2=1328283&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DictionaryDetokenizer.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DictionaryDetokenizer.java Fri Apr 20 09:42:23 2012
@@ -80,4 +80,57 @@ public class DictionaryDetokenizer imple
return operations;
}
+
+ public String detokenize(String tokens[], String splitMarker) {
+
+ DetokenizationOperation operations[] = detokenize(tokens);
+
+ if (tokens.length != operations.length)
+ throw new IllegalArgumentException("tokens and operations array must have same length!");
+
+
+ StringBuilder untokenizedString = new StringBuilder();
+
+ for (int i = 0; i < tokens.length; i++) {
+
+ // attach token to string buffer
+ untokenizedString.append(tokens[i]);
+
+ boolean isAppendSpace;
+ boolean isAppendSplitMarker;
+
+ // if this token is the last token do not attach a space
+ if (i + 1 == operations.length) {
+ isAppendSpace = false;
+ isAppendSplitMarker = false;
+ }
+ // if next token move left, no space after this token,
+ // its safe to access next token
+ else if (operations[i + 1].equals(DetokenizationOperation.MERGE_TO_LEFT)
+ || operations[i + 1].equals(DetokenizationOperation.MERGE_BOTH)) {
+ isAppendSpace = false;
+ isAppendSplitMarker = true;
+ }
+ // if this token is move right, no space
+ else if (operations[i].equals(DetokenizationOperation.MERGE_TO_RIGHT)
+ || operations[i].equals(DetokenizationOperation.MERGE_BOTH)) {
+ isAppendSpace = false;
+ isAppendSplitMarker = true;
+ }
+ else {
+ isAppendSpace = true;
+ isAppendSplitMarker = false;
+ }
+
+ if (isAppendSpace) {
+ untokenizedString.append(' ');
+ }
+
+ if (isAppendSplitMarker && splitMarker != null) {
+ untokenizedString.append(splitMarker);
+ }
+ }
+
+ return untokenizedString.toString();
+ }
}