You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2011/11/07 20:07:50 UTC
svn commit: r1198880 [4/7] - in /incubator/opennlp/trunk: ./ opennlp-distr/
opennlp-distr/src/main/assembly/ opennlp-distr/src/main/readme/
opennlp-docs/ opennlp-docs/src/docbkx/ opennlp-docs/src/docbkx/css/
opennlp-docs/src/main/resources/xsl/ opennlp...
Modified: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/Chunker.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/Chunker.java?rev=1198880&r1=1198879&r2=1198880&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/Chunker.java (original)
+++ incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/Chunker.java Mon Nov 7 19:07:33 2011
@@ -1,237 +1,237 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreemnets. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.uima.chunker;
-
-import java.util.Iterator;
-
-import opennlp.tools.chunker.ChunkerME;
-import opennlp.tools.chunker.ChunkerModel;
-import opennlp.uima.util.AnnotatorUtil;
-import opennlp.uima.util.UimaUtil;
-
-import org.apache.uima.UimaContext;
-import org.apache.uima.analysis_component.CasAnnotator_ImplBase;
-import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
-import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.FSIndex;
-import org.apache.uima.cas.Feature;
-import org.apache.uima.cas.Type;
-import org.apache.uima.cas.TypeSystem;
-import org.apache.uima.cas.text.AnnotationFS;
-import org.apache.uima.resource.ResourceAccessException;
-import org.apache.uima.resource.ResourceInitializationException;
-import org.apache.uima.util.Level;
-import org.apache.uima.util.Logger;
-
-/**
- * OpenNLP Chunker annotator.
- * <p>
- * Mandatory parameters
- * <table border=1>
- * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- * <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.POSFeature</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.ChunkType</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.ChunkTagFeature</td></tr>
- * </table>
- * <p>
- * Optional parameters
- * <table border=1>
- * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- * <tr><td>Integer</td> <td>opennlp.uima.BeamSize</td></tr>
- * </table>
- */
-public final class Chunker extends CasAnnotator_ImplBase {
-
- /**
- * The chunk type parameter.
- */
- public static final String CHUNK_TYPE_PARAMETER = "opennlp.uima.ChunkType";
-
- /**
- * The chunk tag feature parameter
- */
- public static final String CHUNK_TAG_FEATURE_PARAMETER =
- "opennlp.uima.ChunkTagFeature";
-
- private Type mTokenType;
-
- private Type mChunkType;
-
- private Feature mPosFeature;
-
- private ChunkerME mChunker;
-
- private UimaContext context;
-
- private Logger mLogger;
-
- private Feature mChunkFeature;
-
- /**
- * Initializes a new instance.
- *
- * Note: Use {@link #initialize(UimaContext) } to initialize
- * this instance. Not use the constructor.
- */
- public Chunker() {
- // must not be implemented !
- }
-
- /**
- * Initializes the current instance with the given context.
- *
- * Note: Do all initialization in this method, do not use the constructor.
- */
- public void initialize(UimaContext context)
- throws ResourceInitializationException {
-
- super.initialize(context);
-
- this.context = context;
-
- mLogger = context.getLogger();
-
- if (mLogger.isLoggable(Level.INFO)) {
- mLogger.log(Level.INFO, "Initializing the OpenNLP Chunker annotator.");
- }
-
- ChunkerModel model;
-
- try {
- ChunkerModelResource modelResource =
- (ChunkerModelResource) context.getResourceObject(UimaUtil.MODEL_PARAMETER);
-
- model = modelResource.getModel();
- }
- catch (ResourceAccessException e) {
- throw new ResourceInitializationException(e);
- }
-
- mChunker = new ChunkerME(model);
- }
-
- /**
- * Initializes the type system.
- */
- public void typeSystemInit(TypeSystem typeSystem)
- throws AnalysisEngineProcessException {
-
- // chunk type
- mChunkType = AnnotatorUtil.getRequiredTypeParameter(context, typeSystem,
- CHUNK_TYPE_PARAMETER);
-
- // chunk feature
- mChunkFeature = AnnotatorUtil.getRequiredFeatureParameter(context, mChunkType,
- CHUNK_TAG_FEATURE_PARAMETER, CAS.TYPE_NAME_STRING);
-
- // token type
- mTokenType = AnnotatorUtil.getRequiredTypeParameter(context, typeSystem,
- UimaUtil.TOKEN_TYPE_PARAMETER);
-
- // pos feature
- mPosFeature = AnnotatorUtil.getRequiredFeatureParameter(context, mTokenType, UimaUtil.POS_FEATURE_PARAMETER,
- CAS.TYPE_NAME_STRING);
- }
-
- private void addChunkAnnotation(CAS tcas, AnnotationFS tokenAnnotations[],
- String tag, int start, int end) {
- AnnotationFS chunk = tcas.createAnnotation(mChunkType,
- tokenAnnotations[start].getBegin(), tokenAnnotations[end - 1].getEnd());
-
- chunk.setStringValue(mChunkFeature, tag);
-
- tcas.getIndexRepository().addFS(chunk);
- }
-
- /**
- * Performs chunking on the given tcas object.
- */
- public void process(CAS tcas) {
-
- FSIndex<AnnotationFS> tokenAnnotationIndex = tcas.getAnnotationIndex(mTokenType);
-
- String tokens[] = new String[tokenAnnotationIndex.size()];
- String pos[] = new String[tokenAnnotationIndex.size()];
- AnnotationFS tokenAnnotations[] = new AnnotationFS[tokenAnnotationIndex
- .size()];
-
- int index = 0;
-
- for (Iterator<AnnotationFS> tokenAnnotationIterator = tokenAnnotationIndex.iterator();
- tokenAnnotationIterator.hasNext();) {
-
- AnnotationFS tokenAnnotation = tokenAnnotationIterator.next();
-
- tokenAnnotations[index] = tokenAnnotation;
-
- tokens[index] = tokenAnnotation.getCoveredText();
-
- pos[index++] = tokenAnnotation.getFeatureValueAsString(
- mPosFeature);
- }
-
- String result[] = mChunker.chunk(tokens, pos);
-
- int start = -1;
- int end = -1;
- for (int i = 0; i < result.length; i++) {
-
- String chunkTag = result[i];
-
- if (chunkTag.startsWith("B")) {
- if (start != -1) {
- addChunkAnnotation(tcas, tokenAnnotations, result[i - 1].substring(2),
- start, end);
- }
-
- start = i;
- end = i + 1;
- }
- else if (chunkTag.startsWith("I")) {
- end = i + 1;
- }
- else if (chunkTag.startsWith("O")){
- if (start != -1) {
-
- addChunkAnnotation(tcas, tokenAnnotations, result[i - 1].substring(2), start, end);
-
- start = -1;
- end = -1;
- }
- }
- else {
- System.out.println("Unexpected tag: " + result[i]);
- }
- }
-
- if (start != -1) {
- addChunkAnnotation(tcas, tokenAnnotations, result[result.length - 1].substring(2), start, end);
- }
- }
-
- /**
- * Releases allocated resources.
- */
- public void destroy() {
- // dereference model to allow garbage collection
- mChunker = null;
- }
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.uima.chunker;
+
+import java.util.Iterator;
+
+import opennlp.tools.chunker.ChunkerME;
+import opennlp.tools.chunker.ChunkerModel;
+import opennlp.uima.util.AnnotatorUtil;
+import opennlp.uima.util.UimaUtil;
+
+import org.apache.uima.UimaContext;
+import org.apache.uima.analysis_component.CasAnnotator_ImplBase;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.FSIndex;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.resource.ResourceAccessException;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.util.Level;
+import org.apache.uima.util.Logger;
+
+/**
+ * OpenNLP Chunker annotator.
+ * <p>
+ * Mandatory parameters
+ * <table border=1>
+ * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ * <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.POSFeature</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.ChunkType</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.ChunkTagFeature</td></tr>
+ * </table>
+ * <p>
+ * Optional parameters
+ * <table border=1>
+ * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ * <tr><td>Integer</td> <td>opennlp.uima.BeamSize</td></tr>
+ * </table>
+ */
+public final class Chunker extends CasAnnotator_ImplBase {
+
+ /**
+ * The chunk type parameter.
+ */
+ public static final String CHUNK_TYPE_PARAMETER = "opennlp.uima.ChunkType";
+
+ /**
+ * The chunk tag feature parameter
+ */
+ public static final String CHUNK_TAG_FEATURE_PARAMETER =
+ "opennlp.uima.ChunkTagFeature";
+
+ private Type mTokenType;
+
+ private Type mChunkType;
+
+ private Feature mPosFeature;
+
+ private ChunkerME mChunker;
+
+ private UimaContext context;
+
+ private Logger mLogger;
+
+ private Feature mChunkFeature;
+
+ /**
+ * Initializes a new instance.
+ *
+ * Note: Use {@link #initialize(UimaContext) } to initialize
+ * this instance. Not use the constructor.
+ */
+ public Chunker() {
+ // must not be implemented !
+ }
+
+ /**
+ * Initializes the current instance with the given context.
+ *
+ * Note: Do all initialization in this method, do not use the constructor.
+ */
+ public void initialize(UimaContext context)
+ throws ResourceInitializationException {
+
+ super.initialize(context);
+
+ this.context = context;
+
+ mLogger = context.getLogger();
+
+ if (mLogger.isLoggable(Level.INFO)) {
+ mLogger.log(Level.INFO, "Initializing the OpenNLP Chunker annotator.");
+ }
+
+ ChunkerModel model;
+
+ try {
+ ChunkerModelResource modelResource =
+ (ChunkerModelResource) context.getResourceObject(UimaUtil.MODEL_PARAMETER);
+
+ model = modelResource.getModel();
+ }
+ catch (ResourceAccessException e) {
+ throw new ResourceInitializationException(e);
+ }
+
+ mChunker = new ChunkerME(model);
+ }
+
+ /**
+ * Initializes the type system.
+ */
+ public void typeSystemInit(TypeSystem typeSystem)
+ throws AnalysisEngineProcessException {
+
+ // chunk type
+ mChunkType = AnnotatorUtil.getRequiredTypeParameter(context, typeSystem,
+ CHUNK_TYPE_PARAMETER);
+
+ // chunk feature
+ mChunkFeature = AnnotatorUtil.getRequiredFeatureParameter(context, mChunkType,
+ CHUNK_TAG_FEATURE_PARAMETER, CAS.TYPE_NAME_STRING);
+
+ // token type
+ mTokenType = AnnotatorUtil.getRequiredTypeParameter(context, typeSystem,
+ UimaUtil.TOKEN_TYPE_PARAMETER);
+
+ // pos feature
+ mPosFeature = AnnotatorUtil.getRequiredFeatureParameter(context, mTokenType, UimaUtil.POS_FEATURE_PARAMETER,
+ CAS.TYPE_NAME_STRING);
+ }
+
+ private void addChunkAnnotation(CAS tcas, AnnotationFS tokenAnnotations[],
+ String tag, int start, int end) {
+ AnnotationFS chunk = tcas.createAnnotation(mChunkType,
+ tokenAnnotations[start].getBegin(), tokenAnnotations[end - 1].getEnd());
+
+ chunk.setStringValue(mChunkFeature, tag);
+
+ tcas.getIndexRepository().addFS(chunk);
+ }
+
+ /**
+ * Performs chunking on the given tcas object.
+ */
+ public void process(CAS tcas) {
+
+ FSIndex<AnnotationFS> tokenAnnotationIndex = tcas.getAnnotationIndex(mTokenType);
+
+ String tokens[] = new String[tokenAnnotationIndex.size()];
+ String pos[] = new String[tokenAnnotationIndex.size()];
+ AnnotationFS tokenAnnotations[] = new AnnotationFS[tokenAnnotationIndex
+ .size()];
+
+ int index = 0;
+
+ for (Iterator<AnnotationFS> tokenAnnotationIterator = tokenAnnotationIndex.iterator();
+ tokenAnnotationIterator.hasNext();) {
+
+ AnnotationFS tokenAnnotation = tokenAnnotationIterator.next();
+
+ tokenAnnotations[index] = tokenAnnotation;
+
+ tokens[index] = tokenAnnotation.getCoveredText();
+
+ pos[index++] = tokenAnnotation.getFeatureValueAsString(
+ mPosFeature);
+ }
+
+ String result[] = mChunker.chunk(tokens, pos);
+
+ int start = -1;
+ int end = -1;
+ for (int i = 0; i < result.length; i++) {
+
+ String chunkTag = result[i];
+
+ if (chunkTag.startsWith("B")) {
+ if (start != -1) {
+ addChunkAnnotation(tcas, tokenAnnotations, result[i - 1].substring(2),
+ start, end);
+ }
+
+ start = i;
+ end = i + 1;
+ }
+ else if (chunkTag.startsWith("I")) {
+ end = i + 1;
+ }
+ else if (chunkTag.startsWith("O")){
+ if (start != -1) {
+
+ addChunkAnnotation(tcas, tokenAnnotations, result[i - 1].substring(2), start, end);
+
+ start = -1;
+ end = -1;
+ }
+ }
+ else {
+ System.out.println("Unexpected tag: " + result[i]);
+ }
+ }
+
+ if (start != -1) {
+ addChunkAnnotation(tcas, tokenAnnotations, result[result.length - 1].substring(2), start, end);
+ }
+ }
+
+ /**
+ * Releases allocated resources.
+ */
+ public void destroy() {
+ // dereference model to allow garbage collection
+ mChunker = null;
+ }
}
\ No newline at end of file
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/Chunker.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerModelResource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerModelResourceImpl.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java?rev=1198880&r1=1198879&r2=1198880&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java (original)
+++ incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java Mon Nov 7 19:07:33 2011
@@ -1,233 +1,233 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreemnets. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.uima.chunker;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import opennlp.maxent.GIS;
-import opennlp.tools.chunker.ChunkSample;
-import opennlp.tools.chunker.ChunkerME;
-import opennlp.tools.chunker.ChunkerModel;
-import opennlp.tools.util.ObjectStreamUtils;
-import opennlp.uima.util.CasConsumerUtil;
-import opennlp.uima.util.ContainingConstraint;
-import opennlp.uima.util.OpennlpUtil;
-import opennlp.uima.util.UimaUtil;
-
-import org.apache.uima.UimaContext;
-import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.FSIndex;
-import org.apache.uima.cas.Feature;
-import org.apache.uima.cas.Type;
-import org.apache.uima.cas.TypeSystem;
-import org.apache.uima.cas.text.AnnotationFS;
-import org.apache.uima.collection.CasConsumer_ImplBase;
-import org.apache.uima.resource.ResourceInitializationException;
-import org.apache.uima.resource.ResourceProcessException;
-import org.apache.uima.util.Level;
-import org.apache.uima.util.Logger;
-import org.apache.uima.util.ProcessTrace;
-
-/**
- * OpenNLP Chunker trainer.
- * <p>
- * Mandatory parameters
- * <table border=1>
- * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- * <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.POSFeature</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.ChunkType</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.ChunkTagFeature</td></tr>
- * </table>
- */
-public class ChunkerTrainer extends CasConsumer_ImplBase {
-
- private List<ChunkSample> mChunkSamples = new ArrayList<ChunkSample>();
-
- private UimaContext mContext;
-
- private String mModelName;
-
- private Type mSentenceType;
-
- private Type mTokenType;
-
- private Feature mPOSFeature;
-
- private Type mChunkType;
-
- private Feature mChunkTagFeature;
-
- private Logger mLogger;
-
- private String language;
-
- /**
- * Initializes the current instance.
- */
- public void initialize() throws ResourceInitializationException {
-
- super.initialize();
-
- mContext = getUimaContext();
-
- mLogger = mContext.getLogger();
-
- if (mLogger.isLoggable(Level.INFO)) {
- mLogger.log(Level.INFO, "Initializing the OpenNLP Chunker Trainer.");
- }
-
- mModelName = CasConsumerUtil.getRequiredStringParameter(mContext,
- UimaUtil.MODEL_PARAMETER);
-
- language = CasConsumerUtil.getRequiredStringParameter(mContext,
- UimaUtil.LANGUAGE_PARAMETER);
- }
-
- /**
- * Initialize the current instance with the given type system.
- */
- public void typeSystemInit(TypeSystem typeSystem)
- throws ResourceInitializationException {
- String sentenceTypeName =
- CasConsumerUtil.getRequiredStringParameter(mContext,
- UimaUtil.SENTENCE_TYPE_PARAMETER);
-
- mSentenceType = CasConsumerUtil.getType(typeSystem, sentenceTypeName);
-
- String chunkTypeName = CasConsumerUtil.getRequiredStringParameter(mContext,
- Chunker.CHUNK_TYPE_PARAMETER);
-
- mChunkType = CasConsumerUtil.getType(typeSystem, chunkTypeName);
-
- String chunkTagFeature = CasConsumerUtil.getRequiredStringParameter(
- mContext, Chunker.CHUNK_TAG_FEATURE_PARAMETER);
-
- mChunkTagFeature = mChunkType.getFeatureByBaseName(chunkTagFeature);
-
- CasConsumerUtil.checkFeatureType(mChunkTagFeature, CAS.TYPE_NAME_STRING);
-
- String tokenTypeName = CasConsumerUtil.getRequiredStringParameter(mContext,
- UimaUtil.TOKEN_TYPE_PARAMETER);
-
- mTokenType = CasConsumerUtil.getType(typeSystem, tokenTypeName);
-
- String posFeatureName = CasConsumerUtil.getRequiredStringParameter(mContext,
- UimaUtil.POS_FEATURE_PARAMETER);
-
- mPOSFeature = mTokenType.getFeatureByBaseName(posFeatureName);
-
- CasConsumerUtil.checkFeatureType(mPOSFeature, CAS.TYPE_NAME_STRING);
- }
-
- /**
- * Process the given CAS object.
- */
- public void processCas(CAS cas) {
-
- FSIndex<AnnotationFS> sentenceIndex = cas.getAnnotationIndex(mSentenceType);
-
- Iterator<AnnotationFS> sentenceIterator = sentenceIndex.iterator();
- while (sentenceIterator.hasNext()) {
- AnnotationFS sentenceAnnotation = (AnnotationFS) sentenceIterator.next();
-
- processSentence(cas, sentenceAnnotation);
- }
- }
-
- private void processSentence(CAS tcas, AnnotationFS sentence) {
- FSIndex<AnnotationFS> chunkIndex = tcas.getAnnotationIndex(mChunkType);
-
- ContainingConstraint containingConstraint =
- new ContainingConstraint(sentence);
-
- Iterator<AnnotationFS> chunkIterator = tcas.createFilteredIterator(
- chunkIndex.iterator(), containingConstraint);
-
- while (chunkIterator.hasNext()) {
- AnnotationFS chunkAnnotation = (AnnotationFS) chunkIterator.next();
- processChunk(tcas, (chunkAnnotation));
- }
- }
-
- private void processChunk(CAS tcas, AnnotationFS chunk) {
-
- String chunkTag = chunk.getFeatureValueAsString(mChunkTagFeature);
-
- FSIndex<AnnotationFS> tokenIndex = tcas.getAnnotationIndex(mTokenType);
-
- ContainingConstraint containingConstraint =
- new ContainingConstraint(chunk);
-
- Iterator<AnnotationFS> tokenIterator = tcas.createFilteredIterator(tokenIndex.iterator(),
- containingConstraint);
-
- List<String> tokens = new ArrayList<String>();
- List<String> tags = new ArrayList<String>();;
- List<String> chunkTags = new ArrayList<String>();;
-
- while (tokenIterator.hasNext()) {
- AnnotationFS tokenAnnotation = tokenIterator.next();
-
- tokens.add(tokenAnnotation.getCoveredText().trim());
- tags.add(tokenAnnotation.getFeatureValueAsString(mPOSFeature));
- chunkTags.add(chunkTag);
- }
-
- mChunkSamples.add(new ChunkSample(tokens, tags, chunkTags));
- }
-
- /**
- * Called if the processing is finished, this method
- * does the training.
- */
- public void collectionProcessComplete(ProcessTrace trace)
- throws ResourceProcessException, IOException {
- GIS.PRINT_MESSAGES = false;
-
- ChunkerModel chunkerModel = ChunkerME.train(language, ObjectStreamUtils.createObjectStream(mChunkSamples), 100, 5);
-
- // dereference to allow garbage collection
- mChunkSamples = null;
-
- File modelFile = new File(getUimaContextAdmin().getResourceManager()
- .getDataPath() + File.separatorChar + mModelName);
-
- OpennlpUtil.serialize(chunkerModel, modelFile);
- }
-
- /**
- * The trainer is not stateless.
- */
- public boolean isStateless() {
- return false;
- }
-
- /**
- * Releases allocated resources.
- */
- public void destroy() {
- mChunkSamples = null;
- }
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.uima.chunker;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import opennlp.maxent.GIS;
+import opennlp.tools.chunker.ChunkSample;
+import opennlp.tools.chunker.ChunkerME;
+import opennlp.tools.chunker.ChunkerModel;
+import opennlp.tools.util.ObjectStreamUtils;
+import opennlp.uima.util.CasConsumerUtil;
+import opennlp.uima.util.ContainingConstraint;
+import opennlp.uima.util.OpennlpUtil;
+import opennlp.uima.util.UimaUtil;
+
+import org.apache.uima.UimaContext;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.FSIndex;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.collection.CasConsumer_ImplBase;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.ResourceProcessException;
+import org.apache.uima.util.Level;
+import org.apache.uima.util.Logger;
+import org.apache.uima.util.ProcessTrace;
+
+/**
+ * OpenNLP Chunker trainer.
+ * <p>
+ * Mandatory parameters
+ * <table border=1>
+ * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ * <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.POSFeature</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.ChunkType</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.ChunkTagFeature</td></tr>
+ * </table>
+ */
+public class ChunkerTrainer extends CasConsumer_ImplBase {
+
+ private List<ChunkSample> mChunkSamples = new ArrayList<ChunkSample>();
+
+ private UimaContext mContext;
+
+ private String mModelName;
+
+ private Type mSentenceType;
+
+ private Type mTokenType;
+
+ private Feature mPOSFeature;
+
+ private Type mChunkType;
+
+ private Feature mChunkTagFeature;
+
+ private Logger mLogger;
+
+ private String language;
+
+ /**
+ * Initializes the current instance.
+ */
+ public void initialize() throws ResourceInitializationException {
+
+ super.initialize();
+
+ mContext = getUimaContext();
+
+ mLogger = mContext.getLogger();
+
+ if (mLogger.isLoggable(Level.INFO)) {
+ mLogger.log(Level.INFO, "Initializing the OpenNLP Chunker Trainer.");
+ }
+
+ mModelName = CasConsumerUtil.getRequiredStringParameter(mContext,
+ UimaUtil.MODEL_PARAMETER);
+
+ language = CasConsumerUtil.getRequiredStringParameter(mContext,
+ UimaUtil.LANGUAGE_PARAMETER);
+ }
+
+ /**
+ * Initialize the current instance with the given type system.
+ */
+ public void typeSystemInit(TypeSystem typeSystem)
+ throws ResourceInitializationException {
+ String sentenceTypeName =
+ CasConsumerUtil.getRequiredStringParameter(mContext,
+ UimaUtil.SENTENCE_TYPE_PARAMETER);
+
+ mSentenceType = CasConsumerUtil.getType(typeSystem, sentenceTypeName);
+
+ String chunkTypeName = CasConsumerUtil.getRequiredStringParameter(mContext,
+ Chunker.CHUNK_TYPE_PARAMETER);
+
+ mChunkType = CasConsumerUtil.getType(typeSystem, chunkTypeName);
+
+ String chunkTagFeature = CasConsumerUtil.getRequiredStringParameter(
+ mContext, Chunker.CHUNK_TAG_FEATURE_PARAMETER);
+
+ mChunkTagFeature = mChunkType.getFeatureByBaseName(chunkTagFeature);
+
+ CasConsumerUtil.checkFeatureType(mChunkTagFeature, CAS.TYPE_NAME_STRING);
+
+ String tokenTypeName = CasConsumerUtil.getRequiredStringParameter(mContext,
+ UimaUtil.TOKEN_TYPE_PARAMETER);
+
+ mTokenType = CasConsumerUtil.getType(typeSystem, tokenTypeName);
+
+ String posFeatureName = CasConsumerUtil.getRequiredStringParameter(mContext,
+ UimaUtil.POS_FEATURE_PARAMETER);
+
+ mPOSFeature = mTokenType.getFeatureByBaseName(posFeatureName);
+
+ CasConsumerUtil.checkFeatureType(mPOSFeature, CAS.TYPE_NAME_STRING);
+ }
+
+ /**
+ * Process the given CAS object.
+ */
+ public void processCas(CAS cas) {
+
+ FSIndex<AnnotationFS> sentenceIndex = cas.getAnnotationIndex(mSentenceType);
+
+ Iterator<AnnotationFS> sentenceIterator = sentenceIndex.iterator();
+ while (sentenceIterator.hasNext()) {
+ AnnotationFS sentenceAnnotation = (AnnotationFS) sentenceIterator.next();
+
+ processSentence(cas, sentenceAnnotation);
+ }
+ }
+
+ private void processSentence(CAS tcas, AnnotationFS sentence) {
+ FSIndex<AnnotationFS> chunkIndex = tcas.getAnnotationIndex(mChunkType);
+
+ ContainingConstraint containingConstraint =
+ new ContainingConstraint(sentence);
+
+ Iterator<AnnotationFS> chunkIterator = tcas.createFilteredIterator(
+ chunkIndex.iterator(), containingConstraint);
+
+ while (chunkIterator.hasNext()) {
+ AnnotationFS chunkAnnotation = (AnnotationFS) chunkIterator.next();
+ processChunk(tcas, (chunkAnnotation));
+ }
+ }
+
+ private void processChunk(CAS tcas, AnnotationFS chunk) {
+
+ String chunkTag = chunk.getFeatureValueAsString(mChunkTagFeature);
+
+ FSIndex<AnnotationFS> tokenIndex = tcas.getAnnotationIndex(mTokenType);
+
+ ContainingConstraint containingConstraint =
+ new ContainingConstraint(chunk);
+
+ Iterator<AnnotationFS> tokenIterator = tcas.createFilteredIterator(tokenIndex.iterator(),
+ containingConstraint);
+
+ List<String> tokens = new ArrayList<String>();
+ List<String> tags = new ArrayList<String>();;
+ List<String> chunkTags = new ArrayList<String>();;
+
+ while (tokenIterator.hasNext()) {
+ AnnotationFS tokenAnnotation = tokenIterator.next();
+
+ tokens.add(tokenAnnotation.getCoveredText().trim());
+ tags.add(tokenAnnotation.getFeatureValueAsString(mPOSFeature));
+ chunkTags.add(chunkTag);
+ }
+
+ mChunkSamples.add(new ChunkSample(tokens, tags, chunkTags));
+ }
+
+ /**
+ * Called if the processing is finished, this method
+ * does the training.
+ */
+ public void collectionProcessComplete(ProcessTrace trace)
+ throws ResourceProcessException, IOException {
+ GIS.PRINT_MESSAGES = false;
+
+ ChunkerModel chunkerModel = ChunkerME.train(language, ObjectStreamUtils.createObjectStream(mChunkSamples), 100, 5);
+
+ // dereference to allow garbage collection
+ mChunkSamples = null;
+
+ File modelFile = new File(getUimaContextAdmin().getResourceManager()
+ .getDataPath() + File.separatorChar + mModelName);
+
+ OpennlpUtil.serialize(chunkerModel, modelFile);
+ }
+
+ /**
+ * The trainer is not stateless.
+ */
+ public boolean isStateless() {
+ return false;
+ }
+
+ /**
+ * Releases allocated resources.
+ */
+ public void destroy() {
+ mChunkSamples = null;
+ }
}
\ No newline at end of file
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/package.html
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/doccat/AbstractDocumentCategorizer.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/doccat/DoccatModelResource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/doccat/DoccatModelResourceImpl.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizer.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizerTrainer.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/doccat/LanguageDetector.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/AbstractNameFinder.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/DictionaryNameFinder.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinder.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinder.java?rev=1198880&r1=1198879&r2=1198880&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinder.java (original)
+++ incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinder.java Mon Nov 7 19:07:33 2011
@@ -1,213 +1,213 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreemnets. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.uima.namefind;
-
-import java.util.List;
-
-import opennlp.tools.namefind.NameFinderME;
-import opennlp.tools.namefind.TokenNameFinderModel;
-import opennlp.tools.util.Span;
-import opennlp.tools.util.eval.Mean;
-import opennlp.uima.util.AnnotatorUtil;
-import opennlp.uima.util.UimaUtil;
-
-import org.apache.uima.UimaContext;
-import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
-import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.Feature;
-import org.apache.uima.cas.FeatureStructure;
-import org.apache.uima.cas.Type;
-import org.apache.uima.cas.TypeSystem;
-import org.apache.uima.cas.text.AnnotationFS;
-import org.apache.uima.resource.ResourceAccessException;
-import org.apache.uima.resource.ResourceInitializationException;
-
-/**
- * OpenNLP Name annotator.
- * <p>
- * Mandatory parameters
- * <table border=1>
- * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- * <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.NameType</td> <td>The full name of the name type</td></tr>
- * </table>
- * <p>
- * Optional parameters
- * <table border=1>
- * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- * <tr><td>String</td> <td>opennlp.uima.ProbabilityFeature</td> <td>The name of the double
- * probability feature (not set by default)</td></tr>
- * <tr><td>Integer</td> <td>opennlp.uima.BeamSize</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.DocumentConfidenceType</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.DocumentConfidenceType</td></tr>
- *
- * </table>
- */
-public final class NameFinder extends AbstractNameFinder {
-
- public static final String NAME_TYPE_PARAMETER = "opennlp.uima.NameType";
-
- public static final String TOKEN_PATTERN_OPTIMIZATION =
- "opennlp.uima.TokenPatternOptimization";
-
- // Token feature
- public static final String TOKEN_FEATURE_PARAMETER =
- "opennlp.uima.namefinder.TokenFeature";
-
- public static final String TOKEN_FEATURE_PREV_WINDOW_SIZE_PARAMETER =
- TOKEN_FEATURE_PARAMETER + ".previousWindowSize";
-
- public static final String TOKEN_FEATURE_NEXT_WINDOW_SIZE_PARAMETER =
- TOKEN_FEATURE_PARAMETER + ".nextWindowSize";
-
- // Token class feature
- public static final String TOKEN_CLASS_FEATURE_PARAMETER =
- "opennlp.uima.namefinder.TokenClassFeature";
-
- public static final String TOKEN_CLASS_FEATURE_PREV_WINDOW_SIZE_PARAMETER =
- TOKEN_CLASS_FEATURE_PARAMETER + ".previousWindowSize";
-
- public static final String TOKEN_CLASS_FEATURE_NEXT_WINDOW_SIZE_PARAMETER =
- TOKEN_CLASS_FEATURE_PARAMETER + ".nextWindowSize";
-
- private NameFinderME mNameFinder;
-
- private Feature probabilityFeature;
-
- private Type documentConfidenceType;
- private Feature documentConfidenceNameTypeFeature;
- private Feature documentConfidenceFeature;
-
- private Mean documentConfidence = new Mean();
-
- /**
- * Initializes a new instance.
- *
- * Note: Use {@link #initialize(UimaContext) } to initialize
- * this instance. Not use the constructor.
- */
- public NameFinder() {
- super("OpenNLP Maxent Name annotator");
- }
-
- /**
- * Initializes the current instance with the given context.
- *
- * Note: Do all initialization in this method, do not use the constructor.
- */
- public void initialize()
- throws ResourceInitializationException {
-
- super.initialize();
-
- TokenNameFinderModel model;
-
- try {
- TokenNameFinderModelResource modelResource =
- (TokenNameFinderModelResource) context.getResourceObject(UimaUtil.MODEL_PARAMETER);
-
- model = modelResource.getModel();
- }
- catch (ResourceAccessException e) {
- throw new ResourceInitializationException(e);
- }
-
- Integer beamSize = AnnotatorUtil.getOptionalIntegerParameter(context,
- UimaUtil.BEAM_SIZE_PARAMETER);
-
- if (beamSize == null)
- beamSize = NameFinderME.DEFAULT_BEAM_SIZE;
-
- mNameFinder = new NameFinderME(model, beamSize);
- }
-
- /**
- * Initializes the type system.
- */
- public void typeSystemInit(TypeSystem typeSystem)
- throws AnalysisEngineProcessException {
-
- super.typeSystemInit(typeSystem);
-
- probabilityFeature = AnnotatorUtil.getOptionalFeatureParameter(context, mNameType,
- UimaUtil.PROBABILITY_FEATURE_PARAMETER, CAS.TYPE_NAME_DOUBLE);
-
- documentConfidenceType = AnnotatorUtil.getOptionalTypeParameter(context, typeSystem,
- "opennlp.uima.DocumentConfidenceType");
- if (documentConfidenceType != null) {
- documentConfidenceNameTypeFeature = AnnotatorUtil.getRequiredFeature(
- documentConfidenceType, "nameType");
- documentConfidenceFeature = AnnotatorUtil.getRequiredFeature(
- documentConfidenceType, "confidence");
- }
- }
-
- protected Span[] find(CAS cas, String[] tokens) {
-
- Span names[] = mNameFinder.find(tokens);
-
- double probs[] = mNameFinder.probs();
-
- for (int i = 0; i < probs.length; i++) {
- documentConfidence.add(probs[i]);
- }
-
- return names;
- }
-
- protected void postProcessAnnotations(Span detectedNames[],
- AnnotationFS[] nameAnnotations) {
-
- if (probabilityFeature != null) {
- double[] probs = mNameFinder.probs(detectedNames);
-
- for (int i = 0; i < nameAnnotations.length; i++) {
- nameAnnotations[i].setDoubleValue(probabilityFeature, probs[i]);
- }
- }
- }
-
- protected void documentDone(CAS cas) {
-
- // TODO: Create confidence FS
- // contains String name type
- // contains Double prob
- if (documentConfidenceType != null) {
- FeatureStructure confidenceFS = cas.createFS(documentConfidenceType);
- confidenceFS.setDoubleValue(documentConfidenceFeature,
- documentConfidence.mean());
- confidenceFS.setStringValue(documentConfidenceNameTypeFeature,
- mNameType.getName());
- cas.addFsToIndexes(confidenceFS);
- }
-
- // Clears the adaptive data which was created for the current document
- mNameFinder.clearAdaptiveData();
-
- documentConfidence = new Mean();
- }
-
- /**
- * Releases allocated resources.
- */
- public void destroy() {
- mNameFinder = null;
- }
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.uima.namefind;
+
+import java.util.List;
+
+import opennlp.tools.namefind.NameFinderME;
+import opennlp.tools.namefind.TokenNameFinderModel;
+import opennlp.tools.util.Span;
+import opennlp.tools.util.eval.Mean;
+import opennlp.uima.util.AnnotatorUtil;
+import opennlp.uima.util.UimaUtil;
+
+import org.apache.uima.UimaContext;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.FeatureStructure;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.resource.ResourceAccessException;
+import org.apache.uima.resource.ResourceInitializationException;
+
+/**
+ * OpenNLP Name annotator.
+ * <p>
+ * Mandatory parameters
+ * <table border=1>
+ * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ * <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.NameType</td> <td>The full name of the name type</td></tr>
+ * </table>
+ * <p>
+ * Optional parameters
+ * <table border=1>
+ * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ * <tr><td>String</td> <td>opennlp.uima.ProbabilityFeature</td> <td>The name of the double
+ * probability feature (not set by default)</td></tr>
+ * <tr><td>Integer</td> <td>opennlp.uima.BeamSize</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.DocumentConfidenceType</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.DocumentConfidenceType</td></tr>
+ *
+ * </table>
+ */
+public final class NameFinder extends AbstractNameFinder {
+
+ public static final String NAME_TYPE_PARAMETER = "opennlp.uima.NameType";
+
+ public static final String TOKEN_PATTERN_OPTIMIZATION =
+ "opennlp.uima.TokenPatternOptimization";
+
+ // Token feature
+ public static final String TOKEN_FEATURE_PARAMETER =
+ "opennlp.uima.namefinder.TokenFeature";
+
+ public static final String TOKEN_FEATURE_PREV_WINDOW_SIZE_PARAMETER =
+ TOKEN_FEATURE_PARAMETER + ".previousWindowSize";
+
+ public static final String TOKEN_FEATURE_NEXT_WINDOW_SIZE_PARAMETER =
+ TOKEN_FEATURE_PARAMETER + ".nextWindowSize";
+
+ // Token class feature
+ public static final String TOKEN_CLASS_FEATURE_PARAMETER =
+ "opennlp.uima.namefinder.TokenClassFeature";
+
+ public static final String TOKEN_CLASS_FEATURE_PREV_WINDOW_SIZE_PARAMETER =
+ TOKEN_CLASS_FEATURE_PARAMETER + ".previousWindowSize";
+
+ public static final String TOKEN_CLASS_FEATURE_NEXT_WINDOW_SIZE_PARAMETER =
+ TOKEN_CLASS_FEATURE_PARAMETER + ".nextWindowSize";
+
+ private NameFinderME mNameFinder;
+
+ private Feature probabilityFeature;
+
+ private Type documentConfidenceType;
+ private Feature documentConfidenceNameTypeFeature;
+ private Feature documentConfidenceFeature;
+
+ private Mean documentConfidence = new Mean();
+
+ /**
+ * Initializes a new instance.
+ *
+ * Note: Use {@link #initialize(UimaContext) } to initialize
+ * this instance. Not use the constructor.
+ */
+ public NameFinder() {
+ super("OpenNLP Maxent Name annotator");
+ }
+
+ /**
+ * Initializes the current instance with the given context.
+ *
+ * Note: Do all initialization in this method, do not use the constructor.
+ */
+ public void initialize()
+ throws ResourceInitializationException {
+
+ super.initialize();
+
+ TokenNameFinderModel model;
+
+ try {
+ TokenNameFinderModelResource modelResource =
+ (TokenNameFinderModelResource) context.getResourceObject(UimaUtil.MODEL_PARAMETER);
+
+ model = modelResource.getModel();
+ }
+ catch (ResourceAccessException e) {
+ throw new ResourceInitializationException(e);
+ }
+
+ Integer beamSize = AnnotatorUtil.getOptionalIntegerParameter(context,
+ UimaUtil.BEAM_SIZE_PARAMETER);
+
+ if (beamSize == null)
+ beamSize = NameFinderME.DEFAULT_BEAM_SIZE;
+
+ mNameFinder = new NameFinderME(model, beamSize);
+ }
+
+ /**
+ * Initializes the type system.
+ */
+ public void typeSystemInit(TypeSystem typeSystem)
+ throws AnalysisEngineProcessException {
+
+ super.typeSystemInit(typeSystem);
+
+ probabilityFeature = AnnotatorUtil.getOptionalFeatureParameter(context, mNameType,
+ UimaUtil.PROBABILITY_FEATURE_PARAMETER, CAS.TYPE_NAME_DOUBLE);
+
+ documentConfidenceType = AnnotatorUtil.getOptionalTypeParameter(context, typeSystem,
+ "opennlp.uima.DocumentConfidenceType");
+ if (documentConfidenceType != null) {
+ documentConfidenceNameTypeFeature = AnnotatorUtil.getRequiredFeature(
+ documentConfidenceType, "nameType");
+ documentConfidenceFeature = AnnotatorUtil.getRequiredFeature(
+ documentConfidenceType, "confidence");
+ }
+ }
+
+ protected Span[] find(CAS cas, String[] tokens) {
+
+ Span names[] = mNameFinder.find(tokens);
+
+ double probs[] = mNameFinder.probs();
+
+ for (int i = 0; i < probs.length; i++) {
+ documentConfidence.add(probs[i]);
+ }
+
+ return names;
+ }
+
+ protected void postProcessAnnotations(Span detectedNames[],
+ AnnotationFS[] nameAnnotations) {
+
+ if (probabilityFeature != null) {
+ double[] probs = mNameFinder.probs(detectedNames);
+
+ for (int i = 0; i < nameAnnotations.length; i++) {
+ nameAnnotations[i].setDoubleValue(probabilityFeature, probs[i]);
+ }
+ }
+ }
+
+ protected void documentDone(CAS cas) {
+
+ // TODO: Create confidence FS
+ // contains String name type
+ // contains Double prob
+ if (documentConfidenceType != null) {
+ FeatureStructure confidenceFS = cas.createFS(documentConfidenceType);
+ confidenceFS.setDoubleValue(documentConfidenceFeature,
+ documentConfidence.mean());
+ confidenceFS.setStringValue(documentConfidenceNameTypeFeature,
+ mNameType.getName());
+ cas.addFsToIndexes(confidenceFS);
+ }
+
+ // Clears the adaptive data which was created for the current document
+ mNameFinder.clearAdaptiveData();
+
+ documentConfidence = new Mean();
+ }
+
+ /**
+ * Releases allocated resources.
+ */
+ public void destroy() {
+ mNameFinder = null;
+ }
}
\ No newline at end of file
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinder.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java?rev=1198880&r1=1198879&r2=1198880&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java (original)
+++ incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java Mon Nov 7 19:07:33 2011
@@ -1,378 +1,378 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreemnets. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.uima.namefind;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-
-import opennlp.maxent.GIS;
-import opennlp.tools.namefind.NameFinderME;
-import opennlp.tools.namefind.NameSample;
-import opennlp.tools.namefind.NameSampleDataStream;
-import opennlp.tools.namefind.TokenNameFinderModel;
-import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.ObjectStreamUtils;
-import opennlp.tools.util.PlainTextByLineStream;
-import opennlp.tools.util.Span;
-import opennlp.uima.util.CasConsumerUtil;
-import opennlp.uima.util.ContainingConstraint;
-import opennlp.uima.util.OpennlpUtil;
-import opennlp.uima.util.UimaUtil;
-
-import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.FSIndex;
-import org.apache.uima.cas.Type;
-import org.apache.uima.cas.TypeSystem;
-import org.apache.uima.cas.text.AnnotationFS;
-import org.apache.uima.collection.CasConsumer_ImplBase;
-import org.apache.uima.resource.ResourceInitializationException;
-import org.apache.uima.resource.ResourceProcessException;
-import org.apache.uima.util.Level;
-import org.apache.uima.util.Logger;
-import org.apache.uima.util.ProcessTrace;
-
-/**
- * OpenNLP NameFinder trainer.
- * <p>
- * Mandatory parameters
- * <table border=1>
- * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- * <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.Language</td> <td>The language code</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.NameType</td> <td>The full name of the name type</td></tr>
- * </table>
- *
- * Optional parameters
- * <table border=1>
- * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- * <tr><td>String</td> <td>opennlp.uima.AdditionalTrainingDataFile</td> <td>Training file which contains additional data in the OpenNLP format</td></tr>
- * <tr><td>String</td> <td>opennlp.uima.AdditionalTrainingDataEncoding</td> <td>Encoding of the additional training data</td></tr>
- * <tr><td>Integer</td> <td>opennlp.uima.Cutoff</td> <td>(default=5)</td></tr>
- * <tr><td>Integer</td> <td>opennlp.uima.Iterations</td> <td>(default=100)</td></tr>
- * </table>
- * <p>
- */
-public final class NameFinderTrainer extends CasConsumer_ImplBase {
-
- private Logger logger;
-
- private String modelPath;
-
- private String additionalTrainingDataFile;
-
- private String additionalTrainingDataEncoding;
-
- private Type sentenceType;
-
- private Type tokenType;
-
- private Type nameType;
-
- private String language;
-
- private int cutoff;
-
- private int iterations;
-
- // TODO: Keeping all events in memory limits the size of the training corpus
- // Possible solutions:
- // - Write all events to disk
- // - Directly start indexing with a blocking sample stream, the indexer will then write everything
- // to disk or could store the events much more space efficient in memory
-
- private List<NameSample> nameFinderSamples = new ArrayList<NameSample>();
-
- /**
- * Initializes the current instance.
- */
- public void initialize() throws ResourceInitializationException {
-
- super.initialize();
-
- logger = getUimaContext().getLogger();
-
- if (logger.isLoggable(Level.INFO)) {
- logger.log(Level.INFO, "Initializing the OpenNLP Name Trainer.");
- }
-
- modelPath = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
- UimaUtil.MODEL_PARAMETER);
-
- language = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
- UimaUtil.LANGUAGE_PARAMETER);
-
- cutoff = CasConsumerUtil.getOptionalIntegerParameter(getUimaContext(), UimaUtil.CUTOFF_PARAMETER, 5);
- iterations = CasConsumerUtil.getOptionalIntegerParameter(getUimaContext(), UimaUtil.ITERATIONS_PARAMETER, 100);
-
- additionalTrainingDataFile = CasConsumerUtil.getOptionalStringParameter(
- getUimaContext(), UimaUtil.ADDITIONAL_TRAINING_DATA_FILE);
-
- // If the additional training data is specified, the encoding must be provided!
- if (additionalTrainingDataFile != null) {
- additionalTrainingDataEncoding = CasConsumerUtil.getRequiredStringParameter(
- getUimaContext(), UimaUtil.ADDITIONAL_TRAINING_DATA_ENCODING);
- }
- }
-
- /**
- * Initialize the current instance with the given type system.
- */
- public void typeSystemInit(TypeSystem typeSystem)
- throws ResourceInitializationException {
-
- String sentenceTypeName =
- CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
- UimaUtil.SENTENCE_TYPE_PARAMETER);
-
- sentenceType = CasConsumerUtil.getType(typeSystem, sentenceTypeName);
-
- String tokenTypeName = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
- UimaUtil.TOKEN_TYPE_PARAMETER);
-
- tokenType = CasConsumerUtil.getType(typeSystem, tokenTypeName);
-
- String nameTypeName = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
- NameFinder.NAME_TYPE_PARAMETER);
-
- nameType = CasConsumerUtil.getType(typeSystem, nameTypeName);
- }
-
- /**
- * Creates a {@link List} from an {@link Iterator}.
- *
- * @param <T>
- * @param it
- * @return
- */
- private static <T> List<T> iteratorToList(Iterator<T> it) {
- List<T> list = new LinkedList<T>();
-
- while (it.hasNext()) {
- list.add(it.next());
- }
-
- return list;
- }
-
- private static boolean isContaining(AnnotationFS annotation,
- AnnotationFS containtedAnnotation) {
- boolean isStartContaining = annotation.getBegin() <= containtedAnnotation
- .getBegin();
- if (!isStartContaining) {
- return false;
- }
-
- boolean isEndContaining = annotation.getEnd() >= containtedAnnotation
- .getEnd();
- if (!isEndContaining) {
- return false;
- }
-
- return true;
- }
-
- /**
- * Creates the name spans out of a list of token annotations and a list of entity annotations.
- * <p>
- * The name spans for the name finder use a token index and not on a character index which
- * is used by the entity annotations.
- *
- * @param tokenList
- * @param entityAnnotations
- * @return
- */
- private static Span[] createNames(List<AnnotationFS> tokenList, List<AnnotationFS> entityAnnotations) {
-
- List<Span> nameList = new LinkedList<Span>();
-
- AnnotationFS currentEntity = null;
-
- int startIndex = -1;
- int index = 0;
- for (Iterator<AnnotationFS> tokenIterator = tokenList.iterator(); tokenIterator.hasNext();) {
- AnnotationFS token = (AnnotationFS) tokenIterator.next();
-
- for (Iterator<AnnotationFS> it = entityAnnotations.iterator(); it.hasNext();) {
-
- AnnotationFS entity = (AnnotationFS) it.next();
-
- if (!isContaining(entity, token)) {
- // ... end of an entity
- if (currentEntity == entity) {
- nameList.add(new Span(startIndex, index));
-
- startIndex = -1;
- currentEntity = null;
- // break;
- } else {
- continue;
- }
- }
-
- // is this token start of new entity
- if (currentEntity == null && isContaining(entity, token)) {
- startIndex = index;
-
- currentEntity = entity;
- }
- }
-
- index++;
- }
-
- if (currentEntity != null) {
- Span name = new Span(startIndex, index);
- nameList.add(name);
- }
-
- return nameList.toArray(new Span[nameList.size()]);
- }
-
- /**
- * Process the given CAS object.
- */
- public void processCas(CAS cas) {
- FSIndex<AnnotationFS> sentenceIndex = cas.getAnnotationIndex(sentenceType);
-
- Iterator<AnnotationFS> sentenceIterator = sentenceIndex.iterator();
- while (sentenceIterator.hasNext()) {
- AnnotationFS sentenceAnnotation = sentenceIterator.next();
-
- ContainingConstraint sentenceContainingConstraint = new ContainingConstraint(
- sentenceAnnotation);
-
- FSIndex<AnnotationFS> tokenAnnotations = cas.getAnnotationIndex(tokenType);
-
- Iterator<AnnotationFS> containingTokens = cas.createFilteredIterator(tokenAnnotations
- .iterator(), sentenceContainingConstraint);
-
- FSIndex<AnnotationFS> allNames = cas.getAnnotationIndex(nameType);
-
- Iterator<AnnotationFS> containingNames = cas.createFilteredIterator(allNames.iterator(),
- sentenceContainingConstraint);
-
- List<AnnotationFS> tokenList = iteratorToList(containingTokens);
-
- Span names[] = createNames(tokenList, iteratorToList(containingNames));
-
- // create token array
- String tokenArray[] = new String[tokenList.size()];
-
- for (int i = 0; i < tokenArray.length; i++) {
- tokenArray[i] = ((AnnotationFS) tokenList.get(i))
- .getCoveredText();
- }
-
- NameSample traingSentence = new NameSample(tokenArray, names, null, false);
-
- if (traingSentence.getSentence().length != 0) {
- nameFinderSamples.add(traingSentence);
- }
- else {
- if (logger.isLoggable(Level.INFO)) {
- logger.log(Level.INFO, "Sentence without tokens: " +
- sentenceAnnotation.getCoveredText());
- }
- }
- }
- }
-
- /**
- * Called if the processing is finished, this method
- * does the training.
- */
- public void collectionProcessComplete(ProcessTrace trace)
- throws ResourceProcessException, IOException {
-
- if (logger.isLoggable(Level.INFO)) {
- logger.log(Level.INFO, "Collected " + nameFinderSamples.size() +
- " name samples.");
- }
-
- GIS.PRINT_MESSAGES = false;
-
- // create training stream ...
- ObjectStream<NameSample> samples = ObjectStreamUtils.createObjectStream(nameFinderSamples);
-
- InputStream additionalTrainingDataIn = null;
- TokenNameFinderModel nameModel;
- try {
- if (additionalTrainingDataFile != null) {
-
- if (logger.isLoggable(Level.INFO)) {
- logger.log(Level.INFO, "Using addional training data file: " + additionalTrainingDataFile);
- }
-
- additionalTrainingDataIn = new FileInputStream(additionalTrainingDataFile);
-
- // TODO: Make encoding configurable, otherwise use UTF-8 as default!
- ObjectStream<NameSample> additionalSamples = new NameSampleDataStream(
- new PlainTextByLineStream(new InputStreamReader(additionalTrainingDataIn, additionalTrainingDataEncoding)));
-
- samples = ObjectStreamUtils.createObjectStream(samples, additionalSamples);
- }
-
- // TODO: Make sure its possible to pass custom feature generator
- // User could subclass this trainer to provide a custom feature generator
- nameModel = NameFinderME.train(language, null,
- samples, Collections.EMPTY_MAP, iterations, cutoff);
-
- }
- finally {
- if (additionalTrainingDataIn != null)
- additionalTrainingDataIn.close();
- }
-
- // dereference to allow garbage collection
- nameFinderSamples = null;
-
- File modelFile = new File(getUimaContextAdmin().getResourceManager()
- .getDataPath() + File.separatorChar + modelPath);
-
- OpennlpUtil.serialize(nameModel, modelFile);
-
- if (logger.isLoggable(Level.INFO)) {
- logger.log(Level.INFO, "Model was written to: " + modelFile.getAbsolutePath());
- }
- }
-
- /**
- * The trainer is not stateless.
- */
- public boolean isStateless() {
- return false;
- }
-
- /**
- * Destroys the current instance.
- */
- public void destroy() {
- // dereference to allow garbage collection
- nameFinderSamples = null;
- }
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.uima.namefind;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import opennlp.maxent.GIS;
+import opennlp.tools.namefind.NameFinderME;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.namefind.NameSampleDataStream;
+import opennlp.tools.namefind.TokenNameFinderModel;
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.ObjectStreamUtils;
+import opennlp.tools.util.PlainTextByLineStream;
+import opennlp.tools.util.Span;
+import opennlp.uima.util.CasConsumerUtil;
+import opennlp.uima.util.ContainingConstraint;
+import opennlp.uima.util.OpennlpUtil;
+import opennlp.uima.util.UimaUtil;
+
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.FSIndex;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.collection.CasConsumer_ImplBase;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.ResourceProcessException;
+import org.apache.uima.util.Level;
+import org.apache.uima.util.Logger;
+import org.apache.uima.util.ProcessTrace;
+
+/**
+ * OpenNLP NameFinder trainer.
+ * <p>
+ * Mandatory parameters
+ * <table border=1>
+ * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ * <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.Language</td> <td>The language code</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.NameType</td> <td>The full name of the name type</td></tr>
+ * </table>
+ *
+ * Optional parameters
+ * <table border=1>
+ * <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ * <tr><td>String</td> <td>opennlp.uima.AdditionalTrainingDataFile</td> <td>Training file which contains additional data in the OpenNLP format</td></tr>
+ * <tr><td>String</td> <td>opennlp.uima.AdditionalTrainingDataEncoding</td> <td>Encoding of the additional training data</td></tr>
+ * <tr><td>Integer</td> <td>opennlp.uima.Cutoff</td> <td>(default=5)</td></tr>
+ * <tr><td>Integer</td> <td>opennlp.uima.Iterations</td> <td>(default=100)</td></tr>
+ * </table>
+ * <p>
+ */
+public final class NameFinderTrainer extends CasConsumer_ImplBase {
+
+ private Logger logger;
+
+ private String modelPath;
+
+ private String additionalTrainingDataFile;
+
+ private String additionalTrainingDataEncoding;
+
+ private Type sentenceType;
+
+ private Type tokenType;
+
+ private Type nameType;
+
+ private String language;
+
+ private int cutoff;
+
+ private int iterations;
+
+ // TODO: Keeping all events in memory limits the size of the training corpus
+ // Possible solutions:
+ // - Write all events to disk
+ // - Directly start indexing with a blocking sample stream, the indexer will then write everything
+ // to disk or could store the events much more space efficient in memory
+
+ private List<NameSample> nameFinderSamples = new ArrayList<NameSample>();
+
+ /**
+ * Initializes the current instance.
+ */
+ public void initialize() throws ResourceInitializationException {
+
+ super.initialize();
+
+ logger = getUimaContext().getLogger();
+
+ if (logger.isLoggable(Level.INFO)) {
+ logger.log(Level.INFO, "Initializing the OpenNLP Name Trainer.");
+ }
+
+ modelPath = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
+ UimaUtil.MODEL_PARAMETER);
+
+ language = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
+ UimaUtil.LANGUAGE_PARAMETER);
+
+ cutoff = CasConsumerUtil.getOptionalIntegerParameter(getUimaContext(), UimaUtil.CUTOFF_PARAMETER, 5);
+ iterations = CasConsumerUtil.getOptionalIntegerParameter(getUimaContext(), UimaUtil.ITERATIONS_PARAMETER, 100);
+
+ additionalTrainingDataFile = CasConsumerUtil.getOptionalStringParameter(
+ getUimaContext(), UimaUtil.ADDITIONAL_TRAINING_DATA_FILE);
+
+ // If the additional training data is specified, the encoding must be provided!
+ if (additionalTrainingDataFile != null) {
+ additionalTrainingDataEncoding = CasConsumerUtil.getRequiredStringParameter(
+ getUimaContext(), UimaUtil.ADDITIONAL_TRAINING_DATA_ENCODING);
+ }
+ }
+
+ /**
+ * Initialize the current instance with the given type system.
+ */
+ public void typeSystemInit(TypeSystem typeSystem)
+ throws ResourceInitializationException {
+
+ String sentenceTypeName =
+ CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
+ UimaUtil.SENTENCE_TYPE_PARAMETER);
+
+ sentenceType = CasConsumerUtil.getType(typeSystem, sentenceTypeName);
+
+ String tokenTypeName = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
+ UimaUtil.TOKEN_TYPE_PARAMETER);
+
+ tokenType = CasConsumerUtil.getType(typeSystem, tokenTypeName);
+
+ String nameTypeName = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
+ NameFinder.NAME_TYPE_PARAMETER);
+
+ nameType = CasConsumerUtil.getType(typeSystem, nameTypeName);
+ }
+
+ /**
+ * Creates a {@link List} from an {@link Iterator}.
+ *
+ * @param <T>
+ * @param it
+ * @return
+ */
+ private static <T> List<T> iteratorToList(Iterator<T> it) {
+ List<T> list = new LinkedList<T>();
+
+ while (it.hasNext()) {
+ list.add(it.next());
+ }
+
+ return list;
+ }
+
+ private static boolean isContaining(AnnotationFS annotation,
+ AnnotationFS containtedAnnotation) {
+ boolean isStartContaining = annotation.getBegin() <= containtedAnnotation
+ .getBegin();
+ if (!isStartContaining) {
+ return false;
+ }
+
+ boolean isEndContaining = annotation.getEnd() >= containtedAnnotation
+ .getEnd();
+ if (!isEndContaining) {
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Creates the name spans out of a list of token annotations and a list of entity annotations.
+ * <p>
+ * The name spans for the name finder use a token index and not on a character index which
+ * is used by the entity annotations.
+ *
+ * @param tokenList
+ * @param entityAnnotations
+ * @return
+ */
+ private static Span[] createNames(List<AnnotationFS> tokenList, List<AnnotationFS> entityAnnotations) {
+
+ List<Span> nameList = new LinkedList<Span>();
+
+ AnnotationFS currentEntity = null;
+
+ int startIndex = -1;
+ int index = 0;
+ for (Iterator<AnnotationFS> tokenIterator = tokenList.iterator(); tokenIterator.hasNext();) {
+ AnnotationFS token = (AnnotationFS) tokenIterator.next();
+
+ for (Iterator<AnnotationFS> it = entityAnnotations.iterator(); it.hasNext();) {
+
+ AnnotationFS entity = (AnnotationFS) it.next();
+
+ if (!isContaining(entity, token)) {
+ // ... end of an entity
+ if (currentEntity == entity) {
+ nameList.add(new Span(startIndex, index));
+
+ startIndex = -1;
+ currentEntity = null;
+ // break;
+ } else {
+ continue;
+ }
+ }
+
+ // is this token start of new entity
+ if (currentEntity == null && isContaining(entity, token)) {
+ startIndex = index;
+
+ currentEntity = entity;
+ }
+ }
+
+ index++;
+ }
+
+ if (currentEntity != null) {
+ Span name = new Span(startIndex, index);
+ nameList.add(name);
+ }
+
+ return nameList.toArray(new Span[nameList.size()]);
+ }
+
+ /**
+ * Process the given CAS object.
+ */
+ public void processCas(CAS cas) {
+ FSIndex<AnnotationFS> sentenceIndex = cas.getAnnotationIndex(sentenceType);
+
+ Iterator<AnnotationFS> sentenceIterator = sentenceIndex.iterator();
+ while (sentenceIterator.hasNext()) {
+ AnnotationFS sentenceAnnotation = sentenceIterator.next();
+
+ ContainingConstraint sentenceContainingConstraint = new ContainingConstraint(
+ sentenceAnnotation);
+
+ FSIndex<AnnotationFS> tokenAnnotations = cas.getAnnotationIndex(tokenType);
+
+ Iterator<AnnotationFS> containingTokens = cas.createFilteredIterator(tokenAnnotations
+ .iterator(), sentenceContainingConstraint);
+
+ FSIndex<AnnotationFS> allNames = cas.getAnnotationIndex(nameType);
+
+ Iterator<AnnotationFS> containingNames = cas.createFilteredIterator(allNames.iterator(),
+ sentenceContainingConstraint);
+
+ List<AnnotationFS> tokenList = iteratorToList(containingTokens);
+
+ Span names[] = createNames(tokenList, iteratorToList(containingNames));
+
+ // create token array
+ String tokenArray[] = new String[tokenList.size()];
+
+ for (int i = 0; i < tokenArray.length; i++) {
+ tokenArray[i] = ((AnnotationFS) tokenList.get(i))
+ .getCoveredText();
+ }
+
+ NameSample traingSentence = new NameSample(tokenArray, names, null, false);
+
+ if (traingSentence.getSentence().length != 0) {
+ nameFinderSamples.add(traingSentence);
+ }
+ else {
+ if (logger.isLoggable(Level.INFO)) {
+ logger.log(Level.INFO, "Sentence without tokens: " +
+ sentenceAnnotation.getCoveredText());
+ }
+ }
+ }
+ }
+
+ /**
+ * Called if the processing is finished, this method
+ * does the training.
+ */
+ public void collectionProcessComplete(ProcessTrace trace)
+ throws ResourceProcessException, IOException {
+
+ if (logger.isLoggable(Level.INFO)) {
+ logger.log(Level.INFO, "Collected " + nameFinderSamples.size() +
+ " name samples.");
+ }
+
+ GIS.PRINT_MESSAGES = false;
+
+ // create training stream ...
+ ObjectStream<NameSample> samples = ObjectStreamUtils.createObjectStream(nameFinderSamples);
+
+ InputStream additionalTrainingDataIn = null;
+ TokenNameFinderModel nameModel;
+ try {
+ if (additionalTrainingDataFile != null) {
+
+ if (logger.isLoggable(Level.INFO)) {
+ logger.log(Level.INFO, "Using addional training data file: " + additionalTrainingDataFile);
+ }
+
+ additionalTrainingDataIn = new FileInputStream(additionalTrainingDataFile);
+
+ // TODO: Make encoding configurable, otherwise use UTF-8 as default!
+ ObjectStream<NameSample> additionalSamples = new NameSampleDataStream(
+ new PlainTextByLineStream(new InputStreamReader(additionalTrainingDataIn, additionalTrainingDataEncoding)));
+
+ samples = ObjectStreamUtils.createObjectStream(samples, additionalSamples);
+ }
+
+ // TODO: Make sure its possible to pass custom feature generator
+ // User could subclass this trainer to provide a custom feature generator
+ nameModel = NameFinderME.train(language, null,
+ samples, Collections.EMPTY_MAP, iterations, cutoff);
+
+ }
+ finally {
+ if (additionalTrainingDataIn != null)
+ additionalTrainingDataIn.close();
+ }
+
+ // dereference to allow garbage collection
+ nameFinderSamples = null;
+
+ File modelFile = new File(getUimaContextAdmin().getResourceManager()
+ .getDataPath() + File.separatorChar + modelPath);
+
+ OpennlpUtil.serialize(nameModel, modelFile);
+
+ if (logger.isLoggable(Level.INFO)) {
+ logger.log(Level.INFO, "Model was written to: " + modelFile.getAbsolutePath());
+ }
+ }
+
+ /**
+ * The trainer is not stateless.
+ */
+ public boolean isStateless() {
+ return false;
+ }
+
+ /**
+ * Destroys the current instance.
+ */
+ public void destroy() {
+ // dereference to allow garbage collection
+ nameFinderSamples = null;
+ }
}
\ No newline at end of file
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/TokenNameFinderModelResource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/TokenNameFinderModelResourceImpl.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/package.html
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/normalizer/Normalizer.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/normalizer/NumberUtil.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/normalizer/StringDictionary.java
------------------------------------------------------------------------------
svn:eol-style = native