You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2011/09/07 23:37:51 UTC
svn commit: r1166411 - in
/incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize:
TokenizerJob.java TokenizerPreferencePage.java TokenizerViewPage.java
Author: joern
Date: Wed Sep 7 21:37:51 2011
New Revision: 1166411
URL: http://svn.apache.org/viewvc?rev=1166411&view=rev
Log:
OPENNLP-235 Added tokenizer job.
Added:
incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerJob.java (with props)
Modified:
incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerPreferencePage.java
incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerViewPage.java
Added: incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerJob.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerJob.java?rev=1166411&view=auto
==============================================================================
--- incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerJob.java (added)
+++ incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerJob.java Wed Sep 7 21:37:51 2011
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.opennlp.caseditor.tokenize;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import opennlp.tools.tokenize.SimpleTokenizer;
+import opennlp.tools.tokenize.Tokenizer;
+import opennlp.tools.tokenize.TokenizerME;
+import opennlp.tools.tokenize.TokenizerModel;
+import opennlp.tools.tokenize.WhitespaceTokenizer;
+import opennlp.tools.util.Span;
+
+import org.apache.opennlp.caseditor.ModelUtil;
+import org.apache.opennlp.caseditor.OpenNLPPlugin;
+import org.apache.opennlp.caseditor.OpenNLPPreferenceConstants;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.IStatus;
+import org.eclipse.core.runtime.Status;
+import org.eclipse.core.runtime.jobs.Job;
+
+public class TokenizerJob extends Job {
+
+ private String algorithm;
+
+ private String modelPath;
+
+ private String text;
+
+ private Tokenizer tokenizer;
+
+ private Span[] tokens;
+
+ public TokenizerJob() {
+ super("Tokenizer Job");
+ }
+
+ void setTokenizerAlgorithm(String algorithm) {
+ this.algorithm = algorithm;
+ tokenizer = null;
+ }
+
+ void setModelPath(String modelPath) {
+ this.modelPath = modelPath;
+ }
+
+ void setText(String text) {
+ this.text = text;
+ }
+
+ @Override
+ protected IStatus run(IProgressMonitor monitor) {
+
+ if (OpenNLPPreferenceConstants.TOKENIZER_ALGO_WHITESPACE.equals(algorithm)) {
+ tokenizer = WhitespaceTokenizer.INSTANCE;
+ } else if (OpenNLPPreferenceConstants.TOKENIZER_ALGO_SIMPLE.equals(algorithm)) {
+ tokenizer = SimpleTokenizer.INSTANCE;
+ } else if (OpenNLPPreferenceConstants.TOKENIZER_ALGO_STATISTICAL.equals(algorithm)) {
+ if (tokenizer == null) {
+ InputStream modelIn = ModelUtil.openModelIn(modelPath);
+
+ try {
+ TokenizerModel model = new TokenizerModel(modelIn);
+ tokenizer = new TokenizerME(model);
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ if (modelIn != null) {
+ try {
+ modelIn.close();
+ } catch (IOException e) {
+ }
+ }
+ }
+ }
+ }
+
+ tokens = tokenizer.tokenizePos(text);
+
+ return new Status(IStatus.OK, OpenNLPPlugin.ID, "OK");
+ }
+
+ Span[] getTokens() {
+ return tokens;
+ }
+}
Propchange: incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerJob.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerPreferencePage.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerPreferencePage.java?rev=1166411&r1=1166410&r2=1166411&view=diff
==============================================================================
--- incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerPreferencePage.java (original)
+++ incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerPreferencePage.java Wed Sep 7 21:37:51 2011
@@ -19,6 +19,7 @@ package org.apache.opennlp.caseditor.tok
import org.apache.opennlp.caseditor.OpenNLPPlugin;
import org.apache.opennlp.caseditor.OpenNLPPreferenceConstants;
+import org.eclipse.jface.preference.ComboFieldEditor;
import org.eclipse.jface.preference.FieldEditorPreferencePage;
import org.eclipse.jface.preference.StringFieldEditor;
import org.eclipse.ui.IWorkbench;
@@ -36,8 +37,24 @@ public class TokenizerPreferencePage ext
public void init(IWorkbench workbench) {
}
+ // TODO: Do it like in the "Build Path" path preference page ...
+ // radio combined with text field
@Override
protected void createFieldEditors() {
+
+ // TODO: Need to set a default value
+
+ String[][] namesAndValues = new String[][]{
+ new String[]{"Statistical", OpenNLPPreferenceConstants.TOKENIZER_ALGO_STATISTICAL},
+ new String[]{"Whitespace", OpenNLPPreferenceConstants.TOKENIZER_ALGO_WHITESPACE},
+ new String[]{"Simple", OpenNLPPreferenceConstants.TOKENIZER_ALGO_SIMPLE}
+ };
+
+ ComboFieldEditor algorithmCombo = new ComboFieldEditor("TOKENIZER_ALGORITHM",
+ "Algorithm", namesAndValues, getFieldEditorParent());
+ addField(algorithmCombo);
+
+ // Activate only if statistical is selected .. how to do that?
StringFieldEditor modelPath = new StringFieldEditor(
OpenNLPPreferenceConstants.TOKENIZER_MODEL_PATH,
"Model Path", getFieldEditorParent());
Modified: incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerViewPage.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerViewPage.java?rev=1166411&r1=1166410&r2=1166411&view=diff
==============================================================================
--- incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerViewPage.java (original)
+++ incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/tokenize/TokenizerViewPage.java Wed Sep 7 21:37:51 2011
@@ -18,10 +18,13 @@
package org.apache.opennlp.caseditor.tokenize;
import org.apache.uima.caseditor.editor.ICasEditor;
+import org.eclipse.jface.action.IToolBarManager;
import org.eclipse.swt.SWT;
import org.eclipse.swt.widgets.Composite;
import org.eclipse.swt.widgets.Control;
import org.eclipse.swt.widgets.Label;
+import org.eclipse.ui.IActionBars;
+import org.eclipse.ui.actions.BaseSelectionListenerAction;
import org.eclipse.ui.part.Page;
public class TokenizerViewPage extends Page {
@@ -40,6 +43,24 @@ public class TokenizerViewPage extends P
testLabel.setText("This is a test :)");
}
+ // Add action to trigger detection, just add all tokens to CAS
+
+ @Override
+ public void setActionBars(IActionBars actionBars) {
+ super.setActionBars(actionBars);
+
+ IToolBarManager toolBarManager = actionBars.getToolBarManager();
+
+ BaseSelectionListenerAction detectAction = new BaseSelectionListenerAction("Detect") {
+ @Override
+ public void run() {
+ super.run();
+
+ System.out.println("Test");
+ }
+ };
+ }
+
@Override
public Control getControl() {
return testLabel;