You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2011/11/07 20:07:50 UTC

svn commit: r1198880 [4/7] - in /incubator/opennlp/trunk: ./ opennlp-distr/ opennlp-distr/src/main/assembly/ opennlp-distr/src/main/readme/ opennlp-docs/ opennlp-docs/src/docbkx/ opennlp-docs/src/docbkx/css/ opennlp-docs/src/main/resources/xsl/ opennlp...

Modified: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/Chunker.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/Chunker.java?rev=1198880&r1=1198879&r2=1198880&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/Chunker.java (original)
+++ incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/Chunker.java Mon Nov  7 19:07:33 2011
@@ -1,237 +1,237 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreemnets.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.uima.chunker;
-
-import java.util.Iterator;
-
-import opennlp.tools.chunker.ChunkerME;
-import opennlp.tools.chunker.ChunkerModel;
-import opennlp.uima.util.AnnotatorUtil;
-import opennlp.uima.util.UimaUtil;
-
-import org.apache.uima.UimaContext;
-import org.apache.uima.analysis_component.CasAnnotator_ImplBase;
-import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
-import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.FSIndex;
-import org.apache.uima.cas.Feature;
-import org.apache.uima.cas.Type;
-import org.apache.uima.cas.TypeSystem;
-import org.apache.uima.cas.text.AnnotationFS;
-import org.apache.uima.resource.ResourceAccessException;
-import org.apache.uima.resource.ResourceInitializationException;
-import org.apache.uima.util.Level;
-import org.apache.uima.util.Logger;
-
-/**
- * OpenNLP Chunker annotator.
- * <p>
- * Mandatory parameters
- * <table border=1>
- *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- *   <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.POSFeature</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.ChunkType</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.ChunkTagFeature</td></tr>
- * </table>
- * <p>
- * Optional parameters
- * <table border=1>
- *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- *   <tr><td>Integer</td> <td>opennlp.uima.BeamSize</td></tr>
- *   </table>
- */
-public final class Chunker extends CasAnnotator_ImplBase {
-  
-  /**
-   * The chunk type parameter.
-   */
-  public static final String CHUNK_TYPE_PARAMETER = "opennlp.uima.ChunkType";
-  
-  /**
-   * The chunk tag feature parameter
-   */
-  public static final String CHUNK_TAG_FEATURE_PARAMETER = 
-      "opennlp.uima.ChunkTagFeature";
-  
-  private Type mTokenType;
-
-  private Type mChunkType;
-
-  private Feature mPosFeature;
-
-  private ChunkerME mChunker;
-  
-  private UimaContext context;
-  
-  private Logger mLogger;
-
-  private Feature mChunkFeature;
-  
-  /**
-   * Initializes a new instance.
-   *
-   * Note: Use {@link #initialize(UimaContext) } to initialize 
-   * this instance. Not use the constructor.
-   */
-  public Chunker() {
-    // must not be implemented !
-  }
-  
-  /**
-   * Initializes the current instance with the given context.
-   * 
-   * Note: Do all initialization in this method, do not use the constructor.
-   */
-  public void initialize(UimaContext context)
-      throws ResourceInitializationException {
-
-    super.initialize(context);
-    
-	this.context = context;
-	  
-    mLogger = context.getLogger();  
-    
-    if (mLogger.isLoggable(Level.INFO)) {
-      mLogger.log(Level.INFO, "Initializing the OpenNLP Chunker annotator.");
-    }    
-    
-    ChunkerModel model;
-    
-    try {
-      ChunkerModelResource modelResource = 
-            (ChunkerModelResource) context.getResourceObject(UimaUtil.MODEL_PARAMETER);
-        
-        model = modelResource.getModel();
-    }
-    catch (ResourceAccessException e) {
-        throw new ResourceInitializationException(e);
-    }
-    
-    mChunker = new ChunkerME(model);
-  }
-
-  /**
-   * Initializes the type system.
-   */
-  public void typeSystemInit(TypeSystem typeSystem)
-      throws AnalysisEngineProcessException {
-    
-    // chunk type 
-	mChunkType = AnnotatorUtil.getRequiredTypeParameter(context, typeSystem,
-        CHUNK_TYPE_PARAMETER);
-    
-    // chunk feature
-    mChunkFeature = AnnotatorUtil.getRequiredFeatureParameter(context, mChunkType,
-    		CHUNK_TAG_FEATURE_PARAMETER, CAS.TYPE_NAME_STRING);
-    
-    // token type
-    mTokenType = AnnotatorUtil.getRequiredTypeParameter(context, typeSystem,
-        UimaUtil.TOKEN_TYPE_PARAMETER);
-
-    // pos feature
-    mPosFeature = AnnotatorUtil.getRequiredFeatureParameter(context, mTokenType, UimaUtil.POS_FEATURE_PARAMETER, 
-    		CAS.TYPE_NAME_STRING);
-  }
-
-  private void addChunkAnnotation(CAS tcas, AnnotationFS tokenAnnotations[],
-      String tag, int start, int end) {
-    AnnotationFS chunk = tcas.createAnnotation(mChunkType,
-        tokenAnnotations[start].getBegin(), tokenAnnotations[end - 1].getEnd());
-
-    chunk.setStringValue(mChunkFeature, tag);
-
-    tcas.getIndexRepository().addFS(chunk);
-  }
-  
-  /**
-   * Performs chunking on the given tcas object.
-   */
-  public void process(CAS tcas) {
-    
-    FSIndex<AnnotationFS> tokenAnnotationIndex = tcas.getAnnotationIndex(mTokenType);
-
-    String tokens[] = new String[tokenAnnotationIndex.size()];
-    String pos[] = new String[tokenAnnotationIndex.size()];
-    AnnotationFS tokenAnnotations[] = new AnnotationFS[tokenAnnotationIndex
-        .size()];
-
-    int index = 0;
-
-    for (Iterator<AnnotationFS> tokenAnnotationIterator = tokenAnnotationIndex.iterator();
-        tokenAnnotationIterator.hasNext();) {
-      
-      AnnotationFS tokenAnnotation = tokenAnnotationIterator.next();
-
-      tokenAnnotations[index] = tokenAnnotation;
-
-      tokens[index] = tokenAnnotation.getCoveredText();
-
-      pos[index++] = tokenAnnotation.getFeatureValueAsString(
-          mPosFeature);
-    }
-
-    String result[] = mChunker.chunk(tokens, pos);
-
-    int start = -1;
-    int end = -1;
-    for (int i = 0; i < result.length; i++) {
-    	
-      String chunkTag = result[i];	
-    
-      if (chunkTag.startsWith("B")) {
-        if (start != -1) {
-          addChunkAnnotation(tcas, tokenAnnotations, result[i - 1].substring(2),
-        		  start, end);
-        }
-
-        start = i;
-        end = i + 1;
-      }
-      else if (chunkTag.startsWith("I")) {
-    	  end = i + 1;
-      }
-      else if (chunkTag.startsWith("O")){
-        if (start != -1) {
-          
-          addChunkAnnotation(tcas, tokenAnnotations, result[i - 1].substring(2), start, end);
-        	
-          start = -1;
-          end = -1;
-        }
-      }
-      else {
-    	  System.out.println("Unexpected tag: " + result[i]);
-      }
-    }
-    
-    if (start != -1) {
-      addChunkAnnotation(tcas, tokenAnnotations, result[result.length - 1].substring(2), start, end);
-    }      
-  }
-
-  /**
-   * Releases allocated resources.
-   */
-  public void destroy() {
-    // dereference model to allow garbage collection 
-    mChunker = null;
-  }
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.uima.chunker;
+
+import java.util.Iterator;
+
+import opennlp.tools.chunker.ChunkerME;
+import opennlp.tools.chunker.ChunkerModel;
+import opennlp.uima.util.AnnotatorUtil;
+import opennlp.uima.util.UimaUtil;
+
+import org.apache.uima.UimaContext;
+import org.apache.uima.analysis_component.CasAnnotator_ImplBase;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.FSIndex;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.resource.ResourceAccessException;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.util.Level;
+import org.apache.uima.util.Logger;
+
+/**
+ * OpenNLP Chunker annotator.
+ * <p>
+ * Mandatory parameters
+ * <table border=1>
+ *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.POSFeature</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.ChunkType</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.ChunkTagFeature</td></tr>
+ * </table>
+ * <p>
+ * Optional parameters
+ * <table border=1>
+ *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ *   <tr><td>Integer</td> <td>opennlp.uima.BeamSize</td></tr>
+ *   </table>
+ */
+public final class Chunker extends CasAnnotator_ImplBase {
+  
+  /**
+   * The chunk type parameter.
+   */
+  public static final String CHUNK_TYPE_PARAMETER = "opennlp.uima.ChunkType";
+  
+  /**
+   * The chunk tag feature parameter
+   */
+  public static final String CHUNK_TAG_FEATURE_PARAMETER = 
+      "opennlp.uima.ChunkTagFeature";
+  
+  private Type mTokenType;
+
+  private Type mChunkType;
+
+  private Feature mPosFeature;
+
+  private ChunkerME mChunker;
+  
+  private UimaContext context;
+  
+  private Logger mLogger;
+
+  private Feature mChunkFeature;
+  
+  /**
+   * Initializes a new instance.
+   *
+   * Note: Use {@link #initialize(UimaContext) } to initialize 
+   * this instance. Not use the constructor.
+   */
+  public Chunker() {
+    // must not be implemented !
+  }
+  
+  /**
+   * Initializes the current instance with the given context.
+   * 
+   * Note: Do all initialization in this method, do not use the constructor.
+   */
+  public void initialize(UimaContext context)
+      throws ResourceInitializationException {
+
+    super.initialize(context);
+    
+	this.context = context;
+	  
+    mLogger = context.getLogger();  
+    
+    if (mLogger.isLoggable(Level.INFO)) {
+      mLogger.log(Level.INFO, "Initializing the OpenNLP Chunker annotator.");
+    }    
+    
+    ChunkerModel model;
+    
+    try {
+      ChunkerModelResource modelResource = 
+            (ChunkerModelResource) context.getResourceObject(UimaUtil.MODEL_PARAMETER);
+        
+        model = modelResource.getModel();
+    }
+    catch (ResourceAccessException e) {
+        throw new ResourceInitializationException(e);
+    }
+    
+    mChunker = new ChunkerME(model);
+  }
+
+  /**
+   * Initializes the type system.
+   */
+  public void typeSystemInit(TypeSystem typeSystem)
+      throws AnalysisEngineProcessException {
+    
+    // chunk type 
+	mChunkType = AnnotatorUtil.getRequiredTypeParameter(context, typeSystem,
+        CHUNK_TYPE_PARAMETER);
+    
+    // chunk feature
+    mChunkFeature = AnnotatorUtil.getRequiredFeatureParameter(context, mChunkType,
+    		CHUNK_TAG_FEATURE_PARAMETER, CAS.TYPE_NAME_STRING);
+    
+    // token type
+    mTokenType = AnnotatorUtil.getRequiredTypeParameter(context, typeSystem,
+        UimaUtil.TOKEN_TYPE_PARAMETER);
+
+    // pos feature
+    mPosFeature = AnnotatorUtil.getRequiredFeatureParameter(context, mTokenType, UimaUtil.POS_FEATURE_PARAMETER, 
+    		CAS.TYPE_NAME_STRING);
+  }
+
+  private void addChunkAnnotation(CAS tcas, AnnotationFS tokenAnnotations[],
+      String tag, int start, int end) {
+    AnnotationFS chunk = tcas.createAnnotation(mChunkType,
+        tokenAnnotations[start].getBegin(), tokenAnnotations[end - 1].getEnd());
+
+    chunk.setStringValue(mChunkFeature, tag);
+
+    tcas.getIndexRepository().addFS(chunk);
+  }
+  
+  /**
+   * Performs chunking on the given tcas object.
+   */
+  public void process(CAS tcas) {
+    
+    FSIndex<AnnotationFS> tokenAnnotationIndex = tcas.getAnnotationIndex(mTokenType);
+
+    String tokens[] = new String[tokenAnnotationIndex.size()];
+    String pos[] = new String[tokenAnnotationIndex.size()];
+    AnnotationFS tokenAnnotations[] = new AnnotationFS[tokenAnnotationIndex
+        .size()];
+
+    int index = 0;
+
+    for (Iterator<AnnotationFS> tokenAnnotationIterator = tokenAnnotationIndex.iterator();
+        tokenAnnotationIterator.hasNext();) {
+      
+      AnnotationFS tokenAnnotation = tokenAnnotationIterator.next();
+
+      tokenAnnotations[index] = tokenAnnotation;
+
+      tokens[index] = tokenAnnotation.getCoveredText();
+
+      pos[index++] = tokenAnnotation.getFeatureValueAsString(
+          mPosFeature);
+    }
+
+    String result[] = mChunker.chunk(tokens, pos);
+
+    int start = -1;
+    int end = -1;
+    for (int i = 0; i < result.length; i++) {
+    	
+      String chunkTag = result[i];	
+    
+      if (chunkTag.startsWith("B")) {
+        if (start != -1) {
+          addChunkAnnotation(tcas, tokenAnnotations, result[i - 1].substring(2),
+        		  start, end);
+        }
+
+        start = i;
+        end = i + 1;
+      }
+      else if (chunkTag.startsWith("I")) {
+    	  end = i + 1;
+      }
+      else if (chunkTag.startsWith("O")){
+        if (start != -1) {
+          
+          addChunkAnnotation(tcas, tokenAnnotations, result[i - 1].substring(2), start, end);
+        	
+          start = -1;
+          end = -1;
+        }
+      }
+      else {
+    	  System.out.println("Unexpected tag: " + result[i]);
+      }
+    }
+    
+    if (start != -1) {
+      addChunkAnnotation(tcas, tokenAnnotations, result[result.length - 1].substring(2), start, end);
+    }      
+  }
+
+  /**
+   * Releases allocated resources.
+   */
+  public void destroy() {
+    // dereference model to allow garbage collection 
+    mChunker = null;
+  }
 }
\ No newline at end of file

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/Chunker.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerModelResource.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerModelResourceImpl.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java?rev=1198880&r1=1198879&r2=1198880&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java (original)
+++ incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java Mon Nov  7 19:07:33 2011
@@ -1,233 +1,233 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreemnets.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.uima.chunker;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import opennlp.maxent.GIS;
-import opennlp.tools.chunker.ChunkSample;
-import opennlp.tools.chunker.ChunkerME;
-import opennlp.tools.chunker.ChunkerModel;
-import opennlp.tools.util.ObjectStreamUtils;
-import opennlp.uima.util.CasConsumerUtil;
-import opennlp.uima.util.ContainingConstraint;
-import opennlp.uima.util.OpennlpUtil;
-import opennlp.uima.util.UimaUtil;
-
-import org.apache.uima.UimaContext;
-import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.FSIndex;
-import org.apache.uima.cas.Feature;
-import org.apache.uima.cas.Type;
-import org.apache.uima.cas.TypeSystem;
-import org.apache.uima.cas.text.AnnotationFS;
-import org.apache.uima.collection.CasConsumer_ImplBase;
-import org.apache.uima.resource.ResourceInitializationException;
-import org.apache.uima.resource.ResourceProcessException;
-import org.apache.uima.util.Level;
-import org.apache.uima.util.Logger;
-import org.apache.uima.util.ProcessTrace;
-
-/**
- * OpenNLP Chunker trainer.
- * <p>
- * Mandatory parameters
- * <table border=1>
- *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- *   <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.POSFeature</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.ChunkType</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.ChunkTagFeature</td></tr>
- * </table>
- */
-public class ChunkerTrainer extends CasConsumer_ImplBase {
-
-  private List<ChunkSample> mChunkSamples = new ArrayList<ChunkSample>();
-  
-  private UimaContext mContext;
-
-  private String mModelName;
-  
-  private Type mSentenceType;
-
-  private Type mTokenType;
-  
-  private Feature mPOSFeature;
-  
-  private Type mChunkType;
-  
-  private Feature mChunkTagFeature;
-  
-  private Logger mLogger;
-  
-  private String language;
-  
-  /**
-   * Initializes the current instance.
-   */
-  public void initialize() throws ResourceInitializationException {
-    
-    super.initialize();
-    
-    mContext = getUimaContext();
-
-    mLogger = mContext.getLogger();
-    
-    if (mLogger.isLoggable(Level.INFO)) {
-      mLogger.log(Level.INFO, "Initializing the OpenNLP Chunker Trainer.");
-    }    
-
-    mModelName = CasConsumerUtil.getRequiredStringParameter(mContext,
-        UimaUtil.MODEL_PARAMETER);
-    
-    language = CasConsumerUtil.getRequiredStringParameter(mContext,
-        UimaUtil.LANGUAGE_PARAMETER);
-  }
-  
-  /**
-   * Initialize the current instance with the given type system.
-   */
-  public void typeSystemInit(TypeSystem typeSystem) 
-        throws ResourceInitializationException {
-    String sentenceTypeName = 
-        CasConsumerUtil.getRequiredStringParameter(mContext,
-        UimaUtil.SENTENCE_TYPE_PARAMETER);
-
-    mSentenceType = CasConsumerUtil.getType(typeSystem, sentenceTypeName);
-
-    String chunkTypeName = CasConsumerUtil.getRequiredStringParameter(mContext,
-        Chunker.CHUNK_TYPE_PARAMETER);
-    
-    mChunkType = CasConsumerUtil.getType(typeSystem, chunkTypeName);
-    
-    String chunkTagFeature = CasConsumerUtil.getRequiredStringParameter(
-        mContext, Chunker.CHUNK_TAG_FEATURE_PARAMETER);
-    
-    mChunkTagFeature = mChunkType.getFeatureByBaseName(chunkTagFeature);
-      
-    CasConsumerUtil.checkFeatureType(mChunkTagFeature, CAS.TYPE_NAME_STRING);
-    
-    String tokenTypeName = CasConsumerUtil.getRequiredStringParameter(mContext,
-        UimaUtil.TOKEN_TYPE_PARAMETER);
-
-    mTokenType = CasConsumerUtil.getType(typeSystem, tokenTypeName);
-    
-    String posFeatureName = CasConsumerUtil.getRequiredStringParameter(mContext,
-        UimaUtil.POS_FEATURE_PARAMETER);
-    
-    mPOSFeature = mTokenType.getFeatureByBaseName(posFeatureName);
-    
-    CasConsumerUtil.checkFeatureType(mPOSFeature, CAS.TYPE_NAME_STRING);
-  }
-  
-  /**
-   * Process the given CAS object.
-   */
-  public void processCas(CAS cas) {
-    
-    FSIndex<AnnotationFS> sentenceIndex = cas.getAnnotationIndex(mSentenceType);
-
-    Iterator<AnnotationFS> sentenceIterator = sentenceIndex.iterator();
-    while (sentenceIterator.hasNext()) {
-      AnnotationFS sentenceAnnotation = (AnnotationFS) sentenceIterator.next();
-
-      processSentence(cas, sentenceAnnotation);
-    }
-  }
-  
-  private void processSentence(CAS tcas, AnnotationFS sentence) {
-    FSIndex<AnnotationFS> chunkIndex = tcas.getAnnotationIndex(mChunkType);
-    
-    ContainingConstraint containingConstraint = 
-      new ContainingConstraint(sentence);
-
-    Iterator<AnnotationFS> chunkIterator = tcas.createFilteredIterator(
-        chunkIndex.iterator(), containingConstraint);
-    
-    while (chunkIterator.hasNext()) {
-      AnnotationFS chunkAnnotation = (AnnotationFS) chunkIterator.next();
-      processChunk(tcas, (chunkAnnotation));
-    }
-  }
-  
-  private void processChunk(CAS tcas, AnnotationFS chunk) {
-    
-    String chunkTag = chunk.getFeatureValueAsString(mChunkTagFeature);
-    
-    FSIndex<AnnotationFS> tokenIndex = tcas.getAnnotationIndex(mTokenType);
-    
-    ContainingConstraint containingConstraint = 
-      new ContainingConstraint(chunk);
-    
-    Iterator<AnnotationFS> tokenIterator = tcas.createFilteredIterator(tokenIndex.iterator(), 
-        containingConstraint);
-    
-    List<String> tokens = new ArrayList<String>();
-    List<String> tags = new ArrayList<String>();;
-    List<String> chunkTags = new ArrayList<String>();;
-    
-    while (tokenIterator.hasNext()) {
-      AnnotationFS tokenAnnotation = tokenIterator.next();
-      
-      tokens.add(tokenAnnotation.getCoveredText().trim());
-      tags.add(tokenAnnotation.getFeatureValueAsString(mPOSFeature));
-      chunkTags.add(chunkTag);
-    }
-    
-    mChunkSamples.add(new ChunkSample(tokens, tags, chunkTags));
-  }
-  
-  /**
-   * Called if the processing is finished, this method
-   * does the training.
-   */
-  public void collectionProcessComplete(ProcessTrace trace) 
-      throws ResourceProcessException, IOException {
-    GIS.PRINT_MESSAGES = false;
-    
-    ChunkerModel chunkerModel = ChunkerME.train(language, ObjectStreamUtils.createObjectStream(mChunkSamples), 100, 5);
-    
-    // dereference to allow garbage collection
-    mChunkSamples  = null;
-    
-    File modelFile = new File(getUimaContextAdmin().getResourceManager()
-        .getDataPath() + File.separatorChar + mModelName);
-
-    OpennlpUtil.serialize(chunkerModel, modelFile);
-  }
-  
-  /**
-   * The trainer is not stateless.
-   */
-  public boolean isStateless() {
-    return false;
-  }
-  
-  /**
-   * Releases allocated resources.
-   */
-  public void destroy() {
-    mChunkSamples = null;
-  }
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.uima.chunker;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import opennlp.maxent.GIS;
+import opennlp.tools.chunker.ChunkSample;
+import opennlp.tools.chunker.ChunkerME;
+import opennlp.tools.chunker.ChunkerModel;
+import opennlp.tools.util.ObjectStreamUtils;
+import opennlp.uima.util.CasConsumerUtil;
+import opennlp.uima.util.ContainingConstraint;
+import opennlp.uima.util.OpennlpUtil;
+import opennlp.uima.util.UimaUtil;
+
+import org.apache.uima.UimaContext;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.FSIndex;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.collection.CasConsumer_ImplBase;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.ResourceProcessException;
+import org.apache.uima.util.Level;
+import org.apache.uima.util.Logger;
+import org.apache.uima.util.ProcessTrace;
+
+/**
+ * OpenNLP Chunker trainer.
+ * <p>
+ * Mandatory parameters
+ * <table border=1>
+ *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.POSFeature</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.ChunkType</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.ChunkTagFeature</td></tr>
+ * </table>
+ */
+public class ChunkerTrainer extends CasConsumer_ImplBase {
+
+  private List<ChunkSample> mChunkSamples = new ArrayList<ChunkSample>();
+  
+  private UimaContext mContext;
+
+  private String mModelName;
+  
+  private Type mSentenceType;
+
+  private Type mTokenType;
+  
+  private Feature mPOSFeature;
+  
+  private Type mChunkType;
+  
+  private Feature mChunkTagFeature;
+  
+  private Logger mLogger;
+  
+  private String language;
+  
+  /**
+   * Initializes the current instance.
+   */
+  public void initialize() throws ResourceInitializationException {
+    
+    super.initialize();
+    
+    mContext = getUimaContext();
+
+    mLogger = mContext.getLogger();
+    
+    if (mLogger.isLoggable(Level.INFO)) {
+      mLogger.log(Level.INFO, "Initializing the OpenNLP Chunker Trainer.");
+    }    
+
+    mModelName = CasConsumerUtil.getRequiredStringParameter(mContext,
+        UimaUtil.MODEL_PARAMETER);
+    
+    language = CasConsumerUtil.getRequiredStringParameter(mContext,
+        UimaUtil.LANGUAGE_PARAMETER);
+  }
+  
+  /**
+   * Initialize the current instance with the given type system.
+   */
+  public void typeSystemInit(TypeSystem typeSystem) 
+        throws ResourceInitializationException {
+    String sentenceTypeName = 
+        CasConsumerUtil.getRequiredStringParameter(mContext,
+        UimaUtil.SENTENCE_TYPE_PARAMETER);
+
+    mSentenceType = CasConsumerUtil.getType(typeSystem, sentenceTypeName);
+
+    String chunkTypeName = CasConsumerUtil.getRequiredStringParameter(mContext,
+        Chunker.CHUNK_TYPE_PARAMETER);
+    
+    mChunkType = CasConsumerUtil.getType(typeSystem, chunkTypeName);
+    
+    String chunkTagFeature = CasConsumerUtil.getRequiredStringParameter(
+        mContext, Chunker.CHUNK_TAG_FEATURE_PARAMETER);
+    
+    mChunkTagFeature = mChunkType.getFeatureByBaseName(chunkTagFeature);
+      
+    CasConsumerUtil.checkFeatureType(mChunkTagFeature, CAS.TYPE_NAME_STRING);
+    
+    String tokenTypeName = CasConsumerUtil.getRequiredStringParameter(mContext,
+        UimaUtil.TOKEN_TYPE_PARAMETER);
+
+    mTokenType = CasConsumerUtil.getType(typeSystem, tokenTypeName);
+    
+    String posFeatureName = CasConsumerUtil.getRequiredStringParameter(mContext,
+        UimaUtil.POS_FEATURE_PARAMETER);
+    
+    mPOSFeature = mTokenType.getFeatureByBaseName(posFeatureName);
+    
+    CasConsumerUtil.checkFeatureType(mPOSFeature, CAS.TYPE_NAME_STRING);
+  }
+  
+  /**
+   * Process the given CAS object.
+   */
+  public void processCas(CAS cas) {
+    
+    FSIndex<AnnotationFS> sentenceIndex = cas.getAnnotationIndex(mSentenceType);
+
+    Iterator<AnnotationFS> sentenceIterator = sentenceIndex.iterator();
+    while (sentenceIterator.hasNext()) {
+      AnnotationFS sentenceAnnotation = (AnnotationFS) sentenceIterator.next();
+
+      processSentence(cas, sentenceAnnotation);
+    }
+  }
+  
+  private void processSentence(CAS tcas, AnnotationFS sentence) {
+    FSIndex<AnnotationFS> chunkIndex = tcas.getAnnotationIndex(mChunkType);
+    
+    ContainingConstraint containingConstraint = 
+      new ContainingConstraint(sentence);
+
+    Iterator<AnnotationFS> chunkIterator = tcas.createFilteredIterator(
+        chunkIndex.iterator(), containingConstraint);
+    
+    while (chunkIterator.hasNext()) {
+      AnnotationFS chunkAnnotation = (AnnotationFS) chunkIterator.next();
+      processChunk(tcas, (chunkAnnotation));
+    }
+  }
+  
+  private void processChunk(CAS tcas, AnnotationFS chunk) {
+    
+    String chunkTag = chunk.getFeatureValueAsString(mChunkTagFeature);
+    
+    FSIndex<AnnotationFS> tokenIndex = tcas.getAnnotationIndex(mTokenType);
+    
+    ContainingConstraint containingConstraint = 
+      new ContainingConstraint(chunk);
+    
+    Iterator<AnnotationFS> tokenIterator = tcas.createFilteredIterator(tokenIndex.iterator(), 
+        containingConstraint);
+    
+    List<String> tokens = new ArrayList<String>();
+    List<String> tags = new ArrayList<String>();;
+    List<String> chunkTags = new ArrayList<String>();;
+    
+    while (tokenIterator.hasNext()) {
+      AnnotationFS tokenAnnotation = tokenIterator.next();
+      
+      tokens.add(tokenAnnotation.getCoveredText().trim());
+      tags.add(tokenAnnotation.getFeatureValueAsString(mPOSFeature));
+      chunkTags.add(chunkTag);
+    }
+    
+    mChunkSamples.add(new ChunkSample(tokens, tags, chunkTags));
+  }
+  
+  /**
+   * Called if the processing is finished, this method
+   * does the training.
+   */
+  public void collectionProcessComplete(ProcessTrace trace) 
+      throws ResourceProcessException, IOException {
+    GIS.PRINT_MESSAGES = false;
+    
+    ChunkerModel chunkerModel = ChunkerME.train(language, ObjectStreamUtils.createObjectStream(mChunkSamples), 100, 5);
+    
+    // dereference to allow garbage collection
+    mChunkSamples  = null;
+    
+    File modelFile = new File(getUimaContextAdmin().getResourceManager()
+        .getDataPath() + File.separatorChar + mModelName);
+
+    OpennlpUtil.serialize(chunkerModel, modelFile);
+  }
+  
+  /**
+   * The trainer is not stateless.
+   */
+  public boolean isStateless() {
+    return false;
+  }
+  
+  /**
+   * Releases allocated resources.
+   */
+  public void destroy() {
+    mChunkSamples = null;
+  }
 }
\ No newline at end of file

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/chunker/package.html
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/doccat/AbstractDocumentCategorizer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/doccat/DoccatModelResource.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/doccat/DoccatModelResourceImpl.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizerTrainer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/doccat/LanguageDetector.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/AbstractNameFinder.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/DictionaryNameFinder.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinder.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinder.java?rev=1198880&r1=1198879&r2=1198880&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinder.java (original)
+++ incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinder.java Mon Nov  7 19:07:33 2011
@@ -1,213 +1,213 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreemnets.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.uima.namefind;
-
-import java.util.List;
-
-import opennlp.tools.namefind.NameFinderME;
-import opennlp.tools.namefind.TokenNameFinderModel;
-import opennlp.tools.util.Span;
-import opennlp.tools.util.eval.Mean;
-import opennlp.uima.util.AnnotatorUtil;
-import opennlp.uima.util.UimaUtil;
-
-import org.apache.uima.UimaContext;
-import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
-import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.Feature;
-import org.apache.uima.cas.FeatureStructure;
-import org.apache.uima.cas.Type;
-import org.apache.uima.cas.TypeSystem;
-import org.apache.uima.cas.text.AnnotationFS;
-import org.apache.uima.resource.ResourceAccessException;
-import org.apache.uima.resource.ResourceInitializationException;
-
-/**
- * OpenNLP Name annotator.
- * <p>
- * Mandatory parameters
- * <table border=1>
- *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- *   <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.NameType</td> <td>The full name of the name type</td></tr>
- *  </table>
- * <p>
- * Optional parameters
- * <table border=1>
- *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- *   <tr><td>String</td> <td>opennlp.uima.ProbabilityFeature</td> <td>The name of the double 
- *       probability feature (not set by default)</td></tr>
- *   <tr><td>Integer</td> <td>opennlp.uima.BeamSize</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.DocumentConfidenceType</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.DocumentConfidenceType</td></tr>
- *   
- * </table>
- */
-public final class NameFinder extends AbstractNameFinder {
-    
-  public static final String NAME_TYPE_PARAMETER = "opennlp.uima.NameType";
-
-  public static final String TOKEN_PATTERN_OPTIMIZATION = 
-      "opennlp.uima.TokenPatternOptimization";
-
-  // Token feature 
-  public static final String TOKEN_FEATURE_PARAMETER = 
-      "opennlp.uima.namefinder.TokenFeature";
-  
-  public static final String TOKEN_FEATURE_PREV_WINDOW_SIZE_PARAMETER = 
-      TOKEN_FEATURE_PARAMETER + ".previousWindowSize";
-  
-  public static final String TOKEN_FEATURE_NEXT_WINDOW_SIZE_PARAMETER =
-      TOKEN_FEATURE_PARAMETER + ".nextWindowSize";
-  
-  // Token class feature
-  public static final String TOKEN_CLASS_FEATURE_PARAMETER = 
-      "opennlp.uima.namefinder.TokenClassFeature";
-  
-  public static final String TOKEN_CLASS_FEATURE_PREV_WINDOW_SIZE_PARAMETER =
-      TOKEN_CLASS_FEATURE_PARAMETER + ".previousWindowSize";
-  
-  public static final String TOKEN_CLASS_FEATURE_NEXT_WINDOW_SIZE_PARAMETER =
-      TOKEN_CLASS_FEATURE_PARAMETER + ".nextWindowSize";
-  
-  private NameFinderME mNameFinder;
-
-  private Feature probabilityFeature;
-  
-  private Type documentConfidenceType;
-  private Feature documentConfidenceNameTypeFeature;
-  private Feature documentConfidenceFeature;
-  
-  private Mean documentConfidence = new Mean();
-  
-  /**
-   * Initializes a new instance.
-   *
-   * Note: Use {@link #initialize(UimaContext) } to initialize 
-   * this instance. Not use the constructor.
-   */
-  public NameFinder() {
-    super("OpenNLP Maxent Name annotator");
-  }
-  
-  /**
-   * Initializes the current instance with the given context.
-   * 
-   * Note: Do all initialization in this method, do not use the constructor.
-   */
-  public void initialize()
-      throws ResourceInitializationException {  
-
-    super.initialize();
-    
-    TokenNameFinderModel model;
-    
-    try {
-      TokenNameFinderModelResource modelResource = 
-            (TokenNameFinderModelResource) context.getResourceObject(UimaUtil.MODEL_PARAMETER);
-        
-        model = modelResource.getModel();
-    }
-    catch (ResourceAccessException e) {
-        throw new ResourceInitializationException(e);
-    }
-
-    Integer beamSize = AnnotatorUtil.getOptionalIntegerParameter(context, 
-        UimaUtil.BEAM_SIZE_PARAMETER);
-
-    if (beamSize == null)
-      beamSize = NameFinderME.DEFAULT_BEAM_SIZE;
-    
-    mNameFinder = new NameFinderME(model, beamSize);
-  }
-
-  /**
-   * Initializes the type system.
-   */
-  public void typeSystemInit(TypeSystem typeSystem)
-      throws AnalysisEngineProcessException {
-
-    super.typeSystemInit(typeSystem);
-    
-    probabilityFeature = AnnotatorUtil.getOptionalFeatureParameter(context, mNameType,
-    		UimaUtil.PROBABILITY_FEATURE_PARAMETER, CAS.TYPE_NAME_DOUBLE);
-    
-    documentConfidenceType = AnnotatorUtil.getOptionalTypeParameter(context, typeSystem,
-        "opennlp.uima.DocumentConfidenceType");    
-    if (documentConfidenceType != null) {
-      documentConfidenceNameTypeFeature = AnnotatorUtil.getRequiredFeature(
-          documentConfidenceType, "nameType");
-      documentConfidenceFeature = AnnotatorUtil.getRequiredFeature(
-          documentConfidenceType, "confidence");
-    }
-  }
-
-  protected Span[] find(CAS cas, String[] tokens) {
-
-    Span names[] = mNameFinder.find(tokens);
-
-    double probs[] = mNameFinder.probs();
-
-    for (int i = 0; i < probs.length; i++) {
-      documentConfidence.add(probs[i]);
-    }
-
-    return names;
-  }
-  
-  protected void postProcessAnnotations(Span detectedNames[],
-      AnnotationFS[] nameAnnotations) {
-
-    if (probabilityFeature != null) {
-      double[] probs = mNameFinder.probs(detectedNames);
-
-      for (int i = 0; i < nameAnnotations.length; i++) {
-        nameAnnotations[i].setDoubleValue(probabilityFeature, probs[i]);
-      }
-    }
-  }
-
-  protected void documentDone(CAS cas) {
-
-    // TODO: Create confidence FS
-    // contains String name type
-    // contains Double prob
-    if (documentConfidenceType != null) {
-      FeatureStructure confidenceFS = cas.createFS(documentConfidenceType);
-      confidenceFS.setDoubleValue(documentConfidenceFeature,
-          documentConfidence.mean());
-      confidenceFS.setStringValue(documentConfidenceNameTypeFeature,
-          mNameType.getName());
-      cas.addFsToIndexes(confidenceFS);
-    }
-
-    // Clears the adaptive data which was created for the current document
-    mNameFinder.clearAdaptiveData();
-
-    documentConfidence = new Mean();
-  }
-  
-  /**
-   * Releases allocated resources.
-   */
-  public void destroy() {
-    mNameFinder = null;
-  }
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.uima.namefind;
+
+import java.util.List;
+
+import opennlp.tools.namefind.NameFinderME;
+import opennlp.tools.namefind.TokenNameFinderModel;
+import opennlp.tools.util.Span;
+import opennlp.tools.util.eval.Mean;
+import opennlp.uima.util.AnnotatorUtil;
+import opennlp.uima.util.UimaUtil;
+
+import org.apache.uima.UimaContext;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.FeatureStructure;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.resource.ResourceAccessException;
+import org.apache.uima.resource.ResourceInitializationException;
+
+/**
+ * OpenNLP Name annotator.
+ * <p>
+ * Mandatory parameters
+ * <table border=1>
+ *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.NameType</td> <td>The full name of the name type</td></tr>
+ *  </table>
+ * <p>
+ * Optional parameters
+ * <table border=1>
+ *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.ProbabilityFeature</td> <td>The name of the double 
+ *       probability feature (not set by default)</td></tr>
+ *   <tr><td>Integer</td> <td>opennlp.uima.BeamSize</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.DocumentConfidenceType</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.DocumentConfidenceType</td></tr>
+ *   
+ * </table>
+ */
+public final class NameFinder extends AbstractNameFinder {
+    
+  public static final String NAME_TYPE_PARAMETER = "opennlp.uima.NameType";
+
+  public static final String TOKEN_PATTERN_OPTIMIZATION = 
+      "opennlp.uima.TokenPatternOptimization";
+
+  // Token feature 
+  public static final String TOKEN_FEATURE_PARAMETER = 
+      "opennlp.uima.namefinder.TokenFeature";
+  
+  public static final String TOKEN_FEATURE_PREV_WINDOW_SIZE_PARAMETER = 
+      TOKEN_FEATURE_PARAMETER + ".previousWindowSize";
+  
+  public static final String TOKEN_FEATURE_NEXT_WINDOW_SIZE_PARAMETER =
+      TOKEN_FEATURE_PARAMETER + ".nextWindowSize";
+  
+  // Token class feature
+  public static final String TOKEN_CLASS_FEATURE_PARAMETER = 
+      "opennlp.uima.namefinder.TokenClassFeature";
+  
+  public static final String TOKEN_CLASS_FEATURE_PREV_WINDOW_SIZE_PARAMETER =
+      TOKEN_CLASS_FEATURE_PARAMETER + ".previousWindowSize";
+  
+  public static final String TOKEN_CLASS_FEATURE_NEXT_WINDOW_SIZE_PARAMETER =
+      TOKEN_CLASS_FEATURE_PARAMETER + ".nextWindowSize";
+  
+  private NameFinderME mNameFinder;
+
+  private Feature probabilityFeature;
+  
+  private Type documentConfidenceType;
+  private Feature documentConfidenceNameTypeFeature;
+  private Feature documentConfidenceFeature;
+  
+  private Mean documentConfidence = new Mean();
+  
+  /**
+   * Initializes a new instance.
+   *
+   * Note: Use {@link #initialize(UimaContext) } to initialize 
+   * this instance. Not use the constructor.
+   */
+  public NameFinder() {
+    super("OpenNLP Maxent Name annotator");
+  }
+  
+  /**
+   * Initializes the current instance with the given context.
+   * 
+   * Note: Do all initialization in this method, do not use the constructor.
+   */
+  public void initialize()
+      throws ResourceInitializationException {  
+
+    super.initialize();
+    
+    TokenNameFinderModel model;
+    
+    try {
+      TokenNameFinderModelResource modelResource = 
+            (TokenNameFinderModelResource) context.getResourceObject(UimaUtil.MODEL_PARAMETER);
+        
+        model = modelResource.getModel();
+    }
+    catch (ResourceAccessException e) {
+        throw new ResourceInitializationException(e);
+    }
+
+    Integer beamSize = AnnotatorUtil.getOptionalIntegerParameter(context, 
+        UimaUtil.BEAM_SIZE_PARAMETER);
+
+    if (beamSize == null)
+      beamSize = NameFinderME.DEFAULT_BEAM_SIZE;
+    
+    mNameFinder = new NameFinderME(model, beamSize);
+  }
+
+  /**
+   * Initializes the type system.
+   */
+  public void typeSystemInit(TypeSystem typeSystem)
+      throws AnalysisEngineProcessException {
+
+    super.typeSystemInit(typeSystem);
+    
+    probabilityFeature = AnnotatorUtil.getOptionalFeatureParameter(context, mNameType,
+    		UimaUtil.PROBABILITY_FEATURE_PARAMETER, CAS.TYPE_NAME_DOUBLE);
+    
+    documentConfidenceType = AnnotatorUtil.getOptionalTypeParameter(context, typeSystem,
+        "opennlp.uima.DocumentConfidenceType");    
+    if (documentConfidenceType != null) {
+      documentConfidenceNameTypeFeature = AnnotatorUtil.getRequiredFeature(
+          documentConfidenceType, "nameType");
+      documentConfidenceFeature = AnnotatorUtil.getRequiredFeature(
+          documentConfidenceType, "confidence");
+    }
+  }
+
+  protected Span[] find(CAS cas, String[] tokens) {
+
+    Span names[] = mNameFinder.find(tokens);
+
+    double probs[] = mNameFinder.probs();
+
+    for (int i = 0; i < probs.length; i++) {
+      documentConfidence.add(probs[i]);
+    }
+
+    return names;
+  }
+  
+  protected void postProcessAnnotations(Span detectedNames[],
+      AnnotationFS[] nameAnnotations) {
+
+    if (probabilityFeature != null) {
+      double[] probs = mNameFinder.probs(detectedNames);
+
+      for (int i = 0; i < nameAnnotations.length; i++) {
+        nameAnnotations[i].setDoubleValue(probabilityFeature, probs[i]);
+      }
+    }
+  }
+
+  protected void documentDone(CAS cas) {
+
+    // TODO: Create confidence FS
+    // contains String name type
+    // contains Double prob
+    if (documentConfidenceType != null) {
+      FeatureStructure confidenceFS = cas.createFS(documentConfidenceType);
+      confidenceFS.setDoubleValue(documentConfidenceFeature,
+          documentConfidence.mean());
+      confidenceFS.setStringValue(documentConfidenceNameTypeFeature,
+          mNameType.getName());
+      cas.addFsToIndexes(confidenceFS);
+    }
+
+    // Clears the adaptive data which was created for the current document
+    mNameFinder.clearAdaptiveData();
+
+    documentConfidence = new Mean();
+  }
+  
+  /**
+   * Releases allocated resources.
+   */
+  public void destroy() {
+    mNameFinder = null;
+  }
 }
\ No newline at end of file

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinder.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java?rev=1198880&r1=1198879&r2=1198880&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java (original)
+++ incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java Mon Nov  7 19:07:33 2011
@@ -1,378 +1,378 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreemnets.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */ 
-
-package opennlp.uima.namefind;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-
-import opennlp.maxent.GIS;
-import opennlp.tools.namefind.NameFinderME;
-import opennlp.tools.namefind.NameSample;
-import opennlp.tools.namefind.NameSampleDataStream;
-import opennlp.tools.namefind.TokenNameFinderModel;
-import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.ObjectStreamUtils;
-import opennlp.tools.util.PlainTextByLineStream;
-import opennlp.tools.util.Span;
-import opennlp.uima.util.CasConsumerUtil;
-import opennlp.uima.util.ContainingConstraint;
-import opennlp.uima.util.OpennlpUtil;
-import opennlp.uima.util.UimaUtil;
-
-import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.FSIndex;
-import org.apache.uima.cas.Type;
-import org.apache.uima.cas.TypeSystem;
-import org.apache.uima.cas.text.AnnotationFS;
-import org.apache.uima.collection.CasConsumer_ImplBase;
-import org.apache.uima.resource.ResourceInitializationException;
-import org.apache.uima.resource.ResourceProcessException;
-import org.apache.uima.util.Level;
-import org.apache.uima.util.Logger;
-import org.apache.uima.util.ProcessTrace;
-
-/**
- * OpenNLP NameFinder trainer.
- * <p>
- * Mandatory parameters
- * <table border=1>
- *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- *   <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.Language</td> <td>The language code</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.NameType</td> <td>The full name of the name type</td></tr>
- *  </table>
- *  
- * Optional parameters
- * <table border=1>
- *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
- *   <tr><td>String</td> <td>opennlp.uima.AdditionalTrainingDataFile</td> <td>Training file which contains additional data in the OpenNLP format</td></tr>
- *   <tr><td>String</td> <td>opennlp.uima.AdditionalTrainingDataEncoding</td> <td>Encoding of the additional training data</td></tr>
- *   <tr><td>Integer</td> <td>opennlp.uima.Cutoff</td> <td>(default=5)</td></tr>
- *   <tr><td>Integer</td> <td>opennlp.uima.Iterations</td> <td>(default=100)</td></tr>
- * </table>
- * <p>
- */
-public final class NameFinderTrainer extends CasConsumer_ImplBase {
-    
-  private Logger logger;
-  
-  private String modelPath;
-  
-  private String additionalTrainingDataFile;
-  
-  private String additionalTrainingDataEncoding;
-  
-  private Type sentenceType;
-
-  private Type tokenType;
-
-  private Type nameType;
-  
-  private String language;
-  
-  private int cutoff;
-  
-  private int iterations;
-  
-  // TODO: Keeping all events in memory limits the size of the training corpus
-  // Possible solutions:
-  // - Write all events to disk
-  // - Directly start indexing with a blocking sample stream, the indexer will then write everything
-  //   to disk or could store the events much more space efficient in memory
-  
-  private List<NameSample> nameFinderSamples = new ArrayList<NameSample>();
-  
-  /**
-   * Initializes the current instance.
-   */
-  public void initialize() throws ResourceInitializationException {
-    
-    super.initialize();
-    
-    logger = getUimaContext().getLogger();
-    
-    if (logger.isLoggable(Level.INFO)) {
-      logger.log(Level.INFO, "Initializing the OpenNLP Name Trainer.");
-    } 
-    
-    modelPath = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
-        UimaUtil.MODEL_PARAMETER);
-    
-    language = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
-        UimaUtil.LANGUAGE_PARAMETER);
-    
-    cutoff = CasConsumerUtil.getOptionalIntegerParameter(getUimaContext(), UimaUtil.CUTOFF_PARAMETER, 5);
-    iterations = CasConsumerUtil.getOptionalIntegerParameter(getUimaContext(), UimaUtil.ITERATIONS_PARAMETER, 100);
-    
-    additionalTrainingDataFile = CasConsumerUtil.getOptionalStringParameter(
-        getUimaContext(), UimaUtil.ADDITIONAL_TRAINING_DATA_FILE);
-    
-    // If the additional training data is specified, the encoding must be provided!
-    if (additionalTrainingDataFile != null) {
-      additionalTrainingDataEncoding = CasConsumerUtil.getRequiredStringParameter(
-          getUimaContext(), UimaUtil.ADDITIONAL_TRAINING_DATA_ENCODING);
-    }
-  }
-
-  /**
-   * Initialize the current instance with the given type system.
-   */
-  public void typeSystemInit(TypeSystem typeSystem)
-      throws ResourceInitializationException {
-
-    String sentenceTypeName = 
-        CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
-        UimaUtil.SENTENCE_TYPE_PARAMETER);
-
-    sentenceType = CasConsumerUtil.getType(typeSystem, sentenceTypeName);
-
-    String tokenTypeName = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
-        UimaUtil.TOKEN_TYPE_PARAMETER);
-
-    tokenType = CasConsumerUtil.getType(typeSystem, tokenTypeName);
-
-    String nameTypeName = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
-        NameFinder.NAME_TYPE_PARAMETER);
-    
-    nameType = CasConsumerUtil.getType(typeSystem, nameTypeName);
-  }
-
-  /**
-   * Creates a {@link List} from an {@link Iterator}.
-   * 
-   * @param <T>
-   * @param it
-   * @return
-   */
-  private static <T> List<T> iteratorToList(Iterator<T> it) {
-    List<T> list = new LinkedList<T>();
-    
-    while (it.hasNext()) {
-      list.add(it.next());
-    }
-    
-    return list;
-  }
-
-  private static boolean isContaining(AnnotationFS annotation,
-      AnnotationFS containtedAnnotation) {
-    boolean isStartContaining = annotation.getBegin() <= containtedAnnotation
-        .getBegin();
-    if (!isStartContaining) {
-      return false;
-    }
-
-    boolean isEndContaining = annotation.getEnd() >= containtedAnnotation
-        .getEnd();
-    if (!isEndContaining) {
-      return false;
-    }
-
-    return true;
-  }
-  
-  /**
-   * Creates the name spans out of a list of token annotations and a list of entity annotations.
-   * <p>
-   * The name spans for the name finder use a token index and not on a character index which
-   * is used by the entity annotations.
-   * 
-   * @param tokenList
-   * @param entityAnnotations
-   * @return
-   */
-  private static Span[] createNames(List<AnnotationFS> tokenList, List<AnnotationFS> entityAnnotations) {
-
-    List<Span> nameList = new LinkedList<Span>();
-
-    AnnotationFS currentEntity = null;
-
-    int startIndex = -1;
-    int index = 0;
-    for (Iterator<AnnotationFS> tokenIterator = tokenList.iterator(); tokenIterator.hasNext();) {
-      AnnotationFS token = (AnnotationFS) tokenIterator.next();
-
-      for (Iterator<AnnotationFS> it = entityAnnotations.iterator(); it.hasNext();) {
-
-        AnnotationFS entity = (AnnotationFS) it.next();
-
-        if (!isContaining(entity, token)) {
-          // ... end of an entity
-          if (currentEntity == entity) {
-            nameList.add(new Span(startIndex, index));
-
-            startIndex = -1;
-            currentEntity = null;
-            // break;
-          } else {
-            continue;
-          }
-        }
-
-        // is this token start of new entity
-        if (currentEntity == null && isContaining(entity, token)) {
-          startIndex = index;
-
-          currentEntity = entity;
-        }
-      }
-
-      index++;
-    }
-
-    if (currentEntity != null) {
-      Span name = new Span(startIndex, index);
-      nameList.add(name);
-    }
-
-    return nameList.toArray(new Span[nameList.size()]);
-  }
-  
-  /**
-   * Process the given CAS object.
-   */
-  public void processCas(CAS cas) {
-    FSIndex<AnnotationFS> sentenceIndex = cas.getAnnotationIndex(sentenceType);
-
-    Iterator<AnnotationFS> sentenceIterator = sentenceIndex.iterator();
-    while (sentenceIterator.hasNext()) {
-      AnnotationFS sentenceAnnotation = sentenceIterator.next();
-
-      ContainingConstraint sentenceContainingConstraint = new ContainingConstraint(
-          sentenceAnnotation);
-      
-      FSIndex<AnnotationFS> tokenAnnotations = cas.getAnnotationIndex(tokenType);
-      
-      Iterator<AnnotationFS> containingTokens = cas.createFilteredIterator(tokenAnnotations
-          .iterator(), sentenceContainingConstraint);
-      
-      FSIndex<AnnotationFS> allNames = cas.getAnnotationIndex(nameType);
-      
-      Iterator<AnnotationFS> containingNames = cas.createFilteredIterator(allNames.iterator(),
-          sentenceContainingConstraint);
-      
-      List<AnnotationFS> tokenList = iteratorToList(containingTokens);
-      
-      Span names[] = createNames(tokenList, iteratorToList(containingNames));
-      
-      // create token array
-      String tokenArray[] = new String[tokenList.size()];
-      
-      for (int i = 0; i < tokenArray.length; i++) {
-        tokenArray[i] = ((AnnotationFS) tokenList.get(i))
-            .getCoveredText();
-      }
-      
-      NameSample traingSentence = new NameSample(tokenArray, names, null, false);
-      
-      if (traingSentence.getSentence().length != 0) {
-      	nameFinderSamples.add(traingSentence);
-      }
-      else {
-      	if (logger.isLoggable(Level.INFO)) {
-      		logger.log(Level.INFO, "Sentence without tokens: " + 
-      				sentenceAnnotation.getCoveredText());
-      	}
-      }
-    }
-  }
-  
-  /**
-   * Called if the processing is finished, this method
-   * does the training.
-   */
-  public void collectionProcessComplete(ProcessTrace trace)
-      throws ResourceProcessException, IOException {
-   
-    if (logger.isLoggable(Level.INFO)) {
-      logger.log(Level.INFO, "Collected " + nameFinderSamples.size() + 
-          " name samples.");
-    }
-    
-    GIS.PRINT_MESSAGES = false;
-    
-    // create training stream ... 
-    ObjectStream<NameSample> samples = ObjectStreamUtils.createObjectStream(nameFinderSamples);
-    
-    InputStream additionalTrainingDataIn = null;
-    TokenNameFinderModel nameModel;
-    try {
-      if (additionalTrainingDataFile != null) {
-        
-        if (logger.isLoggable(Level.INFO)) {
-          logger.log(Level.INFO, "Using addional training data file: " + additionalTrainingDataFile); 
-        }
-        
-        additionalTrainingDataIn = new FileInputStream(additionalTrainingDataFile);
-        
-        // TODO: Make encoding configurable, otherwise use UTF-8 as default!
-        ObjectStream<NameSample> additionalSamples = new NameSampleDataStream(
-            new PlainTextByLineStream(new InputStreamReader(additionalTrainingDataIn, additionalTrainingDataEncoding)));
-        
-        samples = ObjectStreamUtils.createObjectStream(samples, additionalSamples);
-      }
-      
-      // TODO: Make sure its possible to pass custom feature generator
-      // User could subclass this trainer to provide a custom feature generator
-      nameModel = NameFinderME.train(language, null,
-          samples, Collections.EMPTY_MAP, iterations, cutoff);
-      
-    }
-    finally {
-      if (additionalTrainingDataIn != null)
-        additionalTrainingDataIn.close();
-    }
-    
-    // dereference to allow garbage collection
-    nameFinderSamples = null;
-
-    File modelFile = new File(getUimaContextAdmin().getResourceManager()
-        .getDataPath() + File.separatorChar + modelPath);
-
-    OpennlpUtil.serialize(nameModel, modelFile);
-    
-    if (logger.isLoggable(Level.INFO)) {
-      logger.log(Level.INFO, "Model was written to: " + modelFile.getAbsolutePath());
-    }
-  }
-  
-  /**
-   * The trainer is not stateless.
-   */
-  public boolean isStateless() {
-    return false;
-  }
-  
-  /**
-   * Destroys the current instance.
-   */
-  public void destroy() {
-    // dereference to allow garbage collection
-    nameFinderSamples = null;
-  }
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ 
+
+package opennlp.uima.namefind;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import opennlp.maxent.GIS;
+import opennlp.tools.namefind.NameFinderME;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.namefind.NameSampleDataStream;
+import opennlp.tools.namefind.TokenNameFinderModel;
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.ObjectStreamUtils;
+import opennlp.tools.util.PlainTextByLineStream;
+import opennlp.tools.util.Span;
+import opennlp.uima.util.CasConsumerUtil;
+import opennlp.uima.util.ContainingConstraint;
+import opennlp.uima.util.OpennlpUtil;
+import opennlp.uima.util.UimaUtil;
+
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.FSIndex;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.collection.CasConsumer_ImplBase;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.ResourceProcessException;
+import org.apache.uima.util.Level;
+import org.apache.uima.util.Logger;
+import org.apache.uima.util.ProcessTrace;
+
+/**
+ * OpenNLP NameFinder trainer.
+ * <p>
+ * Mandatory parameters
+ * <table border=1>
+ *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.ModelName</td> <td>The name of the model file</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.Language</td> <td>The language code</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.SentenceType</td> <td>The full name of the sentence type</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.TokenType</td> <td>The full name of the token type</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.NameType</td> <td>The full name of the name type</td></tr>
+ *  </table>
+ *  
+ * Optional parameters
+ * <table border=1>
+ *   <tr><th>Type</th> <th>Name</th> <th>Description</th></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.AdditionalTrainingDataFile</td> <td>Training file which contains additional data in the OpenNLP format</td></tr>
+ *   <tr><td>String</td> <td>opennlp.uima.AdditionalTrainingDataEncoding</td> <td>Encoding of the additional training data</td></tr>
+ *   <tr><td>Integer</td> <td>opennlp.uima.Cutoff</td> <td>(default=5)</td></tr>
+ *   <tr><td>Integer</td> <td>opennlp.uima.Iterations</td> <td>(default=100)</td></tr>
+ * </table>
+ * <p>
+ */
+public final class NameFinderTrainer extends CasConsumer_ImplBase {
+    
+  private Logger logger;
+  
+  private String modelPath;
+  
+  private String additionalTrainingDataFile;
+  
+  private String additionalTrainingDataEncoding;
+  
+  private Type sentenceType;
+
+  private Type tokenType;
+
+  private Type nameType;
+  
+  private String language;
+  
+  private int cutoff;
+  
+  private int iterations;
+  
+  // TODO: Keeping all events in memory limits the size of the training corpus
+  // Possible solutions:
+  // - Write all events to disk
+  // - Directly start indexing with a blocking sample stream, the indexer will then write everything
+  //   to disk or could store the events much more space efficient in memory
+  
+  private List<NameSample> nameFinderSamples = new ArrayList<NameSample>();
+  
+  /**
+   * Initializes the current instance.
+   */
+  public void initialize() throws ResourceInitializationException {
+    
+    super.initialize();
+    
+    logger = getUimaContext().getLogger();
+    
+    if (logger.isLoggable(Level.INFO)) {
+      logger.log(Level.INFO, "Initializing the OpenNLP Name Trainer.");
+    } 
+    
+    modelPath = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
+        UimaUtil.MODEL_PARAMETER);
+    
+    language = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
+        UimaUtil.LANGUAGE_PARAMETER);
+    
+    cutoff = CasConsumerUtil.getOptionalIntegerParameter(getUimaContext(), UimaUtil.CUTOFF_PARAMETER, 5);
+    iterations = CasConsumerUtil.getOptionalIntegerParameter(getUimaContext(), UimaUtil.ITERATIONS_PARAMETER, 100);
+    
+    additionalTrainingDataFile = CasConsumerUtil.getOptionalStringParameter(
+        getUimaContext(), UimaUtil.ADDITIONAL_TRAINING_DATA_FILE);
+    
+    // If the additional training data is specified, the encoding must be provided!
+    if (additionalTrainingDataFile != null) {
+      additionalTrainingDataEncoding = CasConsumerUtil.getRequiredStringParameter(
+          getUimaContext(), UimaUtil.ADDITIONAL_TRAINING_DATA_ENCODING);
+    }
+  }
+
+  /**
+   * Initialize the current instance with the given type system.
+   */
+  public void typeSystemInit(TypeSystem typeSystem)
+      throws ResourceInitializationException {
+
+    String sentenceTypeName = 
+        CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
+        UimaUtil.SENTENCE_TYPE_PARAMETER);
+
+    sentenceType = CasConsumerUtil.getType(typeSystem, sentenceTypeName);
+
+    String tokenTypeName = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
+        UimaUtil.TOKEN_TYPE_PARAMETER);
+
+    tokenType = CasConsumerUtil.getType(typeSystem, tokenTypeName);
+
+    String nameTypeName = CasConsumerUtil.getRequiredStringParameter(getUimaContext(),
+        NameFinder.NAME_TYPE_PARAMETER);
+    
+    nameType = CasConsumerUtil.getType(typeSystem, nameTypeName);
+  }
+
+  /**
+   * Creates a {@link List} from an {@link Iterator}.
+   * 
+   * @param <T>
+   * @param it
+   * @return
+   */
+  private static <T> List<T> iteratorToList(Iterator<T> it) {
+    List<T> list = new LinkedList<T>();
+    
+    while (it.hasNext()) {
+      list.add(it.next());
+    }
+    
+    return list;
+  }
+
+  private static boolean isContaining(AnnotationFS annotation,
+      AnnotationFS containtedAnnotation) {
+    boolean isStartContaining = annotation.getBegin() <= containtedAnnotation
+        .getBegin();
+    if (!isStartContaining) {
+      return false;
+    }
+
+    boolean isEndContaining = annotation.getEnd() >= containtedAnnotation
+        .getEnd();
+    if (!isEndContaining) {
+      return false;
+    }
+
+    return true;
+  }
+  
+  /**
+   * Creates the name spans out of a list of token annotations and a list of entity annotations.
+   * <p>
+   * The name spans for the name finder use a token index and not on a character index which
+   * is used by the entity annotations.
+   * 
+   * @param tokenList
+   * @param entityAnnotations
+   * @return
+   */
+  private static Span[] createNames(List<AnnotationFS> tokenList, List<AnnotationFS> entityAnnotations) {
+
+    List<Span> nameList = new LinkedList<Span>();
+
+    AnnotationFS currentEntity = null;
+
+    int startIndex = -1;
+    int index = 0;
+    for (Iterator<AnnotationFS> tokenIterator = tokenList.iterator(); tokenIterator.hasNext();) {
+      AnnotationFS token = (AnnotationFS) tokenIterator.next();
+
+      for (Iterator<AnnotationFS> it = entityAnnotations.iterator(); it.hasNext();) {
+
+        AnnotationFS entity = (AnnotationFS) it.next();
+
+        if (!isContaining(entity, token)) {
+          // ... end of an entity
+          if (currentEntity == entity) {
+            nameList.add(new Span(startIndex, index));
+
+            startIndex = -1;
+            currentEntity = null;
+            // break;
+          } else {
+            continue;
+          }
+        }
+
+        // is this token start of new entity
+        if (currentEntity == null && isContaining(entity, token)) {
+          startIndex = index;
+
+          currentEntity = entity;
+        }
+      }
+
+      index++;
+    }
+
+    if (currentEntity != null) {
+      Span name = new Span(startIndex, index);
+      nameList.add(name);
+    }
+
+    return nameList.toArray(new Span[nameList.size()]);
+  }
+  
+  /**
+   * Process the given CAS object.
+   */
+  public void processCas(CAS cas) {
+    FSIndex<AnnotationFS> sentenceIndex = cas.getAnnotationIndex(sentenceType);
+
+    Iterator<AnnotationFS> sentenceIterator = sentenceIndex.iterator();
+    while (sentenceIterator.hasNext()) {
+      AnnotationFS sentenceAnnotation = sentenceIterator.next();
+
+      ContainingConstraint sentenceContainingConstraint = new ContainingConstraint(
+          sentenceAnnotation);
+      
+      FSIndex<AnnotationFS> tokenAnnotations = cas.getAnnotationIndex(tokenType);
+      
+      Iterator<AnnotationFS> containingTokens = cas.createFilteredIterator(tokenAnnotations
+          .iterator(), sentenceContainingConstraint);
+      
+      FSIndex<AnnotationFS> allNames = cas.getAnnotationIndex(nameType);
+      
+      Iterator<AnnotationFS> containingNames = cas.createFilteredIterator(allNames.iterator(),
+          sentenceContainingConstraint);
+      
+      List<AnnotationFS> tokenList = iteratorToList(containingTokens);
+      
+      Span names[] = createNames(tokenList, iteratorToList(containingNames));
+      
+      // create token array
+      String tokenArray[] = new String[tokenList.size()];
+      
+      for (int i = 0; i < tokenArray.length; i++) {
+        tokenArray[i] = ((AnnotationFS) tokenList.get(i))
+            .getCoveredText();
+      }
+      
+      NameSample traingSentence = new NameSample(tokenArray, names, null, false);
+      
+      if (traingSentence.getSentence().length != 0) {
+      	nameFinderSamples.add(traingSentence);
+      }
+      else {
+      	if (logger.isLoggable(Level.INFO)) {
+      		logger.log(Level.INFO, "Sentence without tokens: " + 
+      				sentenceAnnotation.getCoveredText());
+      	}
+      }
+    }
+  }
+  
+  /**
+   * Called if the processing is finished, this method
+   * does the training.
+   */
+  public void collectionProcessComplete(ProcessTrace trace)
+      throws ResourceProcessException, IOException {
+   
+    if (logger.isLoggable(Level.INFO)) {
+      logger.log(Level.INFO, "Collected " + nameFinderSamples.size() + 
+          " name samples.");
+    }
+    
+    GIS.PRINT_MESSAGES = false;
+    
+    // create training stream ... 
+    ObjectStream<NameSample> samples = ObjectStreamUtils.createObjectStream(nameFinderSamples);
+    
+    InputStream additionalTrainingDataIn = null;
+    TokenNameFinderModel nameModel;
+    try {
+      if (additionalTrainingDataFile != null) {
+        
+        if (logger.isLoggable(Level.INFO)) {
+          logger.log(Level.INFO, "Using addional training data file: " + additionalTrainingDataFile); 
+        }
+        
+        additionalTrainingDataIn = new FileInputStream(additionalTrainingDataFile);
+        
+        // TODO: Make encoding configurable, otherwise use UTF-8 as default!
+        ObjectStream<NameSample> additionalSamples = new NameSampleDataStream(
+            new PlainTextByLineStream(new InputStreamReader(additionalTrainingDataIn, additionalTrainingDataEncoding)));
+        
+        samples = ObjectStreamUtils.createObjectStream(samples, additionalSamples);
+      }
+      
+      // TODO: Make sure its possible to pass custom feature generator
+      // User could subclass this trainer to provide a custom feature generator
+      nameModel = NameFinderME.train(language, null,
+          samples, Collections.EMPTY_MAP, iterations, cutoff);
+      
+    }
+    finally {
+      if (additionalTrainingDataIn != null)
+        additionalTrainingDataIn.close();
+    }
+    
+    // dereference to allow garbage collection
+    nameFinderSamples = null;
+
+    File modelFile = new File(getUimaContextAdmin().getResourceManager()
+        .getDataPath() + File.separatorChar + modelPath);
+
+    OpennlpUtil.serialize(nameModel, modelFile);
+    
+    if (logger.isLoggable(Level.INFO)) {
+      logger.log(Level.INFO, "Model was written to: " + modelFile.getAbsolutePath());
+    }
+  }
+  
+  /**
+   * The trainer is not stateless.
+   */
+  public boolean isStateless() {
+    return false;
+  }
+  
+  /**
+   * Destroys the current instance.
+   */
+  public void destroy() {
+    // dereference to allow garbage collection
+    nameFinderSamples = null;
+  }
 }
\ No newline at end of file

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/TokenNameFinderModelResource.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/TokenNameFinderModelResourceImpl.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/namefind/package.html
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/normalizer/Normalizer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/normalizer/NumberUtil.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/opennlp/trunk/opennlp-uima/src/main/java/opennlp/uima/normalizer/StringDictionary.java
------------------------------------------------------------------------------
    svn:eol-style = native