You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2011/09/07 23:37:51 UTC

svn commit: r1166411 - in /incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize: TokenizerJob.java TokenizerPreferencePage.java TokenizerViewPage.java

Author: joern
Date: Wed Sep  7 21:37:51 2011
New Revision: 1166411

URL: http://svn.apache.org/viewvc?rev=1166411&view=rev
Log:
OPENNLP-235 Added tokenizer job.

Added:
    incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerJob.java   (with props)
Modified:
    incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerPreferencePage.java
    incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerViewPage.java

Added: incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerJob.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerJob.java?rev=1166411&view=auto
==============================================================================
--- incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerJob.java (added)
+++ incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerJob.java Wed Sep  7 21:37:51 2011
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.opennlp.caseditor.tokenize;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import opennlp.tools.tokenize.SimpleTokenizer;
+import opennlp.tools.tokenize.Tokenizer;
+import opennlp.tools.tokenize.TokenizerME;
+import opennlp.tools.tokenize.TokenizerModel;
+import opennlp.tools.tokenize.WhitespaceTokenizer;
+import opennlp.tools.util.Span;
+
+import org.apache.opennlp.caseditor.ModelUtil;
+import org.apache.opennlp.caseditor.OpenNLPPlugin;
+import org.apache.opennlp.caseditor.OpenNLPPreferenceConstants;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.IStatus;
+import org.eclipse.core.runtime.Status;
+import org.eclipse.core.runtime.jobs.Job;
+
+public class TokenizerJob extends Job {
+
+  private String algorithm;
+  
+  private String modelPath;
+  
+  private String text;
+  
+  private Tokenizer tokenizer;
+  
+  private Span[] tokens;
+  
+  public TokenizerJob() {
+    super("Tokenizer Job");
+  }
+
+  void setTokenizerAlgorithm(String algorithm) {
+    this.algorithm = algorithm;
+    tokenizer = null;
+  }
+  
+  void setModelPath(String modelPath) {
+    this.modelPath = modelPath;
+  }
+  
+  void setText(String text) {
+    this.text = text;
+  }
+  
+  @Override
+  protected IStatus run(IProgressMonitor monitor) {
+    
+    if (OpenNLPPreferenceConstants.TOKENIZER_ALGO_WHITESPACE.equals(algorithm)) {
+      tokenizer = WhitespaceTokenizer.INSTANCE;
+    } else if (OpenNLPPreferenceConstants.TOKENIZER_ALGO_SIMPLE.equals(algorithm)) {
+      tokenizer = SimpleTokenizer.INSTANCE;
+    } else if (OpenNLPPreferenceConstants.TOKENIZER_ALGO_STATISTICAL.equals(algorithm)) {
+      if (tokenizer == null) {
+        InputStream modelIn = ModelUtil.openModelIn(modelPath);
+        
+        try {
+          TokenizerModel model = new TokenizerModel(modelIn);
+          tokenizer = new TokenizerME(model);
+        } catch (IOException e) {
+          e.printStackTrace();
+        } finally {
+          if (modelIn != null) {
+            try {
+              modelIn.close();
+            } catch (IOException e) {
+            }
+          }
+        }
+      }
+    }
+    
+    tokens = tokenizer.tokenizePos(text);
+    
+    return new Status(IStatus.OK, OpenNLPPlugin.ID, "OK");
+  }
+  
+  Span[] getTokens() {
+    return tokens;
+  }
+}

Propchange: incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerJob.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerPreferencePage.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerPreferencePage.java?rev=1166411&r1=1166410&r2=1166411&view=diff
==============================================================================
--- incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerPreferencePage.java (original)
+++ incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerPreferencePage.java Wed Sep  7 21:37:51 2011
@@ -19,6 +19,7 @@ package org.apache.opennlp.caseditor.tok
 
 import org.apache.opennlp.caseditor.OpenNLPPlugin;
 import org.apache.opennlp.caseditor.OpenNLPPreferenceConstants;
+import org.eclipse.jface.preference.ComboFieldEditor;
 import org.eclipse.jface.preference.FieldEditorPreferencePage;
 import org.eclipse.jface.preference.StringFieldEditor;
 import org.eclipse.ui.IWorkbench;
@@ -36,8 +37,24 @@ public class TokenizerPreferencePage ext
   public void init(IWorkbench workbench) {
   }
 
+  // TODO: Do it like in the "Build Path" path preference page ... 
+  //       radio combined with text field
   @Override
   protected void createFieldEditors() {
+    
+    // TODO: Need to set a default value
+    
+    String[][] namesAndValues = new String[][]{
+        new String[]{"Statistical", OpenNLPPreferenceConstants.TOKENIZER_ALGO_STATISTICAL},
+        new String[]{"Whitespace", OpenNLPPreferenceConstants.TOKENIZER_ALGO_WHITESPACE},
+        new String[]{"Simple", OpenNLPPreferenceConstants.TOKENIZER_ALGO_SIMPLE}
+    };
+    
+    ComboFieldEditor algorithmCombo = new ComboFieldEditor("TOKENIZER_ALGORITHM", 
+        "Algorithm", namesAndValues, getFieldEditorParent());
+    addField(algorithmCombo);
+
+    // Activate only if statistical is selected .. how to do that?
     StringFieldEditor modelPath = new StringFieldEditor(
         OpenNLPPreferenceConstants.TOKENIZER_MODEL_PATH,
         "Model Path", getFieldEditorParent());

Modified: incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerViewPage.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerViewPage.java?rev=1166411&r1=1166410&r2=1166411&view=diff
==============================================================================
--- incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerViewPage.java (original)
+++ incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerViewPage.java Wed Sep  7 21:37:51 2011
@@ -18,10 +18,13 @@
 package org.apache.opennlp.caseditor.tokenize;
 
 import org.apache.uima.caseditor.editor.ICasEditor;
+import org.eclipse.jface.action.IToolBarManager;
 import org.eclipse.swt.SWT;
 import org.eclipse.swt.widgets.Composite;
 import org.eclipse.swt.widgets.Control;
 import org.eclipse.swt.widgets.Label;
+import org.eclipse.ui.IActionBars;
+import org.eclipse.ui.actions.BaseSelectionListenerAction;
 import org.eclipse.ui.part.Page;
 
 public class TokenizerViewPage extends Page {
@@ -40,6 +43,24 @@ public class TokenizerViewPage extends P
     testLabel.setText("This is a test :)");
   }
 
+  // Add action to trigger detection, just add all tokens to CAS
+  
+  @Override
+  public void setActionBars(IActionBars actionBars) {
+    super.setActionBars(actionBars);
+    
+    IToolBarManager toolBarManager = actionBars.getToolBarManager();
+    
+    BaseSelectionListenerAction detectAction = new BaseSelectionListenerAction("Detect") {
+      @Override
+      public void run() {
+        super.run();
+        
+        System.out.println("Test");
+      }
+    };
+  }
+  
   @Override
   public Control getControl() {
     return testLabel;