You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2012/04/20 11:42:23 UTC

svn commit: r1328283 - in /opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools: cmdline/tokenizer/DictionaryDetokenizerTool.java tokenize/Detokenizer.java tokenize/DictionaryDetokenizer.java

Author: joern
Date: Fri Apr 20 09:42:23 2012
New Revision: 1328283

URL: http://svn.apache.org/viewvc?rev=1328283&view=rev
Log:
OPENNLP-501 Added detokenization method which outputs a detokenized string.

Modified:
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/DictionaryDetokenizerTool.java
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DictionaryDetokenizer.java

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/DictionaryDetokenizerTool.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/DictionaryDetokenizerTool.java?rev=1328283&r1=1328282&r2=1328283&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/DictionaryDetokenizerTool.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/DictionaryDetokenizerTool.java Fri Apr 20 09:42:23 2012
@@ -26,7 +26,6 @@ import opennlp.tools.cmdline.CLI;
 import opennlp.tools.cmdline.CmdLineUtil;
 import opennlp.tools.cmdline.PerformanceMonitor;
 import opennlp.tools.tokenize.Detokenizer;
-import opennlp.tools.tokenize.Detokenizer.DetokenizationOperation;
 import opennlp.tools.tokenize.DictionaryDetokenizer;
 import opennlp.tools.tokenize.WhitespaceTokenizer;
 import opennlp.tools.util.ObjectStream;
@@ -37,47 +36,6 @@ public final class DictionaryDetokenizer
   public String getHelp() {
     return "Usage: " + CLI.CMD + " " + getName() + " detokenizerDictionary";
   }
-
-  public static String detokenize(String tokens[], DetokenizationOperation operations[]) {
- 
-    if (tokens.length != operations.length)
-      throw new IllegalArgumentException("tokens and operations array must have same length!");
-    
-    
-    StringBuilder untokenizedString = new StringBuilder();
-    
-    for (int i = 0; i < tokens.length; i++) {
-      
-      // attach token to string buffer
-      untokenizedString.append(tokens[i]);
-      
-      boolean isAppendSpace;
-      
-      // if this token is the last token do not attach a space
-      if (i + 1 == operations.length) {
-        isAppendSpace = false;
-      }
-      // if next token move left, no space after this token,
-      // its safe to access next token
-      else if (operations[i + 1].equals(DetokenizationOperation.MERGE_TO_LEFT)
-          || operations[i + 1].equals(DetokenizationOperation.MERGE_BOTH)) {
-        isAppendSpace = false;
-      }
-      // if this token is move right, no space 
-      else if (operations[i].equals(DetokenizationOperation.MERGE_TO_RIGHT)
-          || operations[i].equals(DetokenizationOperation.MERGE_BOTH)) {
-        isAppendSpace = false;
-      }
-      else {
-        isAppendSpace = true;
-      }
-      
-      if (isAppendSpace)
-        untokenizedString.append(' ');
-    }
-    
-    return untokenizedString.toString();
-  }
   
   public void run(String[] args) {
     
@@ -102,9 +60,7 @@ public final class DictionaryDetokenizer
           // white space tokenize line
           String tokens[] = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine);
 
-          DetokenizationOperation operations[] = detokenizer.detokenize(tokens);
-
-          System.out.println(detokenize(tokens, operations));
+          System.out.println(detokenizer.detokenize(tokens, null));
 
           perfMon.incrementCounter();
         }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java?rev=1328283&r1=1328282&r2=1328283&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java Fri Apr 20 09:42:23 2012
@@ -58,4 +58,16 @@ public interface Detokenizer {
    * @return the merge operations to detokenize the input tokens.
    */
   DetokenizationOperation[] detokenize(String tokens[]);
+  
+  /**
+   * Detokenize the input tokens into a String. Tokens which
+   * are connected without a space inbetween can be separated by
+   * a split marker.
+   * 
+   * @param tokens
+   * @param splitMarker the split marker or null
+   * 
+   * @return
+   */
+  String detokenize(String tokens[], String splitMarker);
 }

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DictionaryDetokenizer.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DictionaryDetokenizer.java?rev=1328283&r1=1328282&r2=1328283&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DictionaryDetokenizer.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/DictionaryDetokenizer.java Fri Apr 20 09:42:23 2012
@@ -80,4 +80,57 @@ public class DictionaryDetokenizer imple
     
     return operations;
   }
+  
+  public String detokenize(String tokens[], String splitMarker) {
+    
+    DetokenizationOperation operations[] = detokenize(tokens);
+    
+    if (tokens.length != operations.length)
+      throw new IllegalArgumentException("tokens and operations array must have same length!");
+    
+    
+    StringBuilder untokenizedString = new StringBuilder();
+    
+    for (int i = 0; i < tokens.length; i++) {
+      
+      // attach token to string buffer
+      untokenizedString.append(tokens[i]);
+      
+      boolean isAppendSpace;
+      boolean isAppendSplitMarker;
+      
+      // if this token is the last token do not attach a space
+      if (i + 1 == operations.length) {
+        isAppendSpace = false;
+        isAppendSplitMarker = false;
+      }
+      // if next token move left, no space after this token,
+      // its safe to access next token
+      else if (operations[i + 1].equals(DetokenizationOperation.MERGE_TO_LEFT)
+          || operations[i + 1].equals(DetokenizationOperation.MERGE_BOTH)) {
+        isAppendSpace = false;
+        isAppendSplitMarker = true;
+      }
+      // if this token is move right, no space 
+      else if (operations[i].equals(DetokenizationOperation.MERGE_TO_RIGHT)
+          || operations[i].equals(DetokenizationOperation.MERGE_BOTH)) {
+        isAppendSpace = false;
+        isAppendSplitMarker = true;
+      }
+      else {
+        isAppendSpace = true;
+        isAppendSplitMarker = false;
+      }
+      
+      if (isAppendSpace) {
+        untokenizedString.append(' ');
+      }
+      
+      if (isAppendSplitMarker && splitMarker != null) {
+        untokenizedString.append(splitMarker);
+      }
+    }
+    
+    return untokenizedString.toString();
+  }
 }