You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by tw...@apache.org on 2008/02/29 14:25:31 UTC

svn commit: r632315 - in /incubator/uima/sandbox/trunk/Tagger: ./ desc/ docbook/hmmTaggerUsersGuide/ resources/english/ resources/german/ src/main/java/org/apache/uima/examples/tagger/ src/main/java/org/apache/uima/examples/tagger/trainAndTest/

Author: twgoetz
Date: Fri Feb 29 05:25:15 2008
New Revision: 632315

URL: http://svn.apache.org/viewvc?rev=632315&view=rev
Log:
Jira UIMA-791: apply patch.

https://issues.apache.org/jira/browse/UIMA-791

Added:
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/NGram.java
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/GrobMappingTueba.java
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/MappingInterface.java
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TagMappingBrown.java
Removed:
    incubator/uima/sandbox/trunk/Tagger/params.txt
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/GrobMapping.java
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/MappingInterface.java
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/TagMapping.java
Modified:
    incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml
    incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml
    incubator/uima/sandbox/trunk/Tagger/docbook/hmmTaggerUsersGuide/hmmTaggerUsersGuide.xml
    incubator/uima/sandbox/trunk/Tagger/resources/english/BrownModel.dat
    incubator/uima/sandbox/trunk/Tagger/resources/german/TuebaModel.dat
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Viterbi.java
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/BrownReader.java
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/CorpusReader.java
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/ModelGeneration.java
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/SuffixTree.java
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TT_FormatReader.java
    incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TaggerEvaluation.java
    incubator/uima/sandbox/trunk/Tagger/tagger.properties

Modified: incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml (original)
+++ incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml Fri Feb 29 05:25:15 2008
@@ -1,145 +1,103 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-    
-    http://www.apache.org/licenses/LICENSE-2.0
-    
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.    
--->
-
-<analysisEngineDescription
-	xmlns="http://uima.apache.org/resourceSpecifier">
-	<frameworkImplementation>
-		org.apache.uima.java
-	</frameworkImplementation>
-	<primitive>true</primitive>
-	<annotatorImplementationName>
-		org.apache.uima.examples.tagger.HMMTagger
-	</annotatorImplementationName>
-	<analysisEngineMetaData>
-		<name>Hidden Markov Model - Part of Speech Tagger</name>
-		<description>
-			A configuration of the HmmTaggerAnnotator that looks for
+<analysisEngineDescription xmlns="http://uima.apache.org/resourceSpecifier">
+  <frameworkImplementation>org.apache.uima.java</frameworkImplementation>
+  <primitive>true</primitive>
+  <annotatorImplementationName>org.apache.uima.examples.tagger.HMMTagger</annotatorImplementationName>
+  <analysisEngineMetaData>
+    <name>Hidden Markov Model - Part of Speech Tagger</name>
+    <description>A configuration of the HmmTaggerAnnotator that looks for
 			parts of speech of identified tokens within existing
 			Sentence and Token annotations. See also
-			WhitespaceTokenizer.xml.
-		</description>
-		<version>1.0</version>
-		<vendor>The Apache Software Foundation</vendor>
-		<configurationParameters>
-            <configurationParameter>
-                <name>NGRAM_SIZE</name>
-                <type>Integer</type>
-                <multiValued>false</multiValued>
-                <mandatory>true</mandatory>
-            </configurationParameter>
-		</configurationParameters>
-		<configurationParameterSettings>
-            <nameValuePair>
-                <name>NGRAM_SIZE</name>
-                <value>
-                    <integer>3</integer>
-                </value>
-            </nameValuePair>
-		</configurationParameterSettings>
-		<typeSystemDescription>
-			<types>
-				<typeDescription>
-					<name>org.apache.uima.TokenAnnotation</name>
-					<description>Single token annotation</description>
-					<supertypeName>uima.tcas.Annotation</supertypeName>
-					<features>
-						<featureDescription>
-							<name>posTag</name>
-							<description>
-								contains part-of-speech of a
-								corresponding token
-							</description>
-							<rangeTypeName>
-								uima.cas.String
-							</rangeTypeName>
-						</featureDescription>
-					</features>
-				</typeDescription>
-				<typeDescription>
-					<name>org.apache.uima.SentenceAnnotation</name>
-					<description>sentence annotation</description>
-					<supertypeName>uima.tcas.Annotation</supertypeName>
-				</typeDescription>
-			</types>
-		</typeSystemDescription>
-		<typePriorities />
-		<fsIndexCollection />
-		<capabilities>
-			<capability>
-				<inputs>
-					<type>org.apache.uima.TokenAnnotation</type>
-					<type allAnnotatorFeatures="true">
-						org.apache.uima.SentenceAnnotation
-					</type>
-					<feature>
-						org.apache.uima.TokenAnnotation:end
-					</feature>
-					<feature>
-						org.apache.uima.TokenAnnotation:begin
-					</feature>
-				</inputs>
-				<outputs>
-					<type>org.apache.uima.TokenAnnotation</type>
-					<feature>
-						org.apache.uima.TokenAnnotation:posTag
-					</feature>
-					<feature>
-						org.apache.uima.TokenAnnotation:end
-					</feature>
-					<feature>
-						org.apache.uima.TokenAnnotation:begin
-					</feature>
-				</outputs>
-				<languagesSupported />
-			</capability>
-		</capabilities>
-		<operationalProperties>
-			<modifiesCas>true</modifiesCas>
-			<multipleDeploymentAllowed>true</multipleDeploymentAllowed>
-			<outputsNewCASes>false</outputsNewCASes>
-		</operationalProperties>
-	</analysisEngineMetaData>
-	<externalResourceDependencies>
-        <externalResourceDependency>
-            <key>Model</key>
-            <description>HMM Tagger model file</description>
-            <interfaceName>org.apache.uima.examples.tagger.IModelResource</interfaceName>
-            <optional>false</optional>
-        </externalResourceDependency>
-    </externalResourceDependencies>
-    <resourceManagerConfiguration>
-        <externalResources>
-            <externalResource>
-                <name>ModelFile</name>
-                <description>HMM Tagger model file</description>
-                <fileResourceSpecifier>
-                    <fileUrl>file:english/BrownModel.dat</fileUrl>
-                </fileResourceSpecifier>
-                <implementationName>org.apache.uima.examples.tagger.ModelResource</implementationName>
-            </externalResource>
-        </externalResources>
-        <externalResourceBindings>
-            <externalResourceBinding>
-                <key>Model</key>
-                <resourceName>ModelFile</resourceName>
-            </externalResourceBinding>
-        </externalResourceBindings>
-   </resourceManagerConfiguration>
+			WhitespaceTokenizer.xml.</description>
+    <version>1.0</version>
+    <vendor>The Apache Software Foundation</vendor>
+    <configurationParameters>
+      <configurationParameter>
+        <name>NGRAM_SIZE</name>
+        <type>Integer</type>
+        <multiValued>false</multiValued>
+        <mandatory>true</mandatory>
+      </configurationParameter>
+    </configurationParameters>
+    <configurationParameterSettings>
+      <nameValuePair>
+        <name>NGRAM_SIZE</name>
+        <value>
+          <integer>3</integer>
+        </value>
+      </nameValuePair>
+    </configurationParameterSettings>
+    <typeSystemDescription>
+      <types>
+        <typeDescription>
+          <name>org.apache.uima.TokenAnnotation</name>
+          <description>Single token annotation</description>
+          <supertypeName>uima.tcas.Annotation</supertypeName>
+          <features>
+            <featureDescription>
+              <name>posTag</name>
+              <description>contains part-of-speech of a
+								corresponding token</description>
+              <rangeTypeName>uima.cas.String</rangeTypeName>
+            </featureDescription>
+          </features>
+        </typeDescription>
+        <typeDescription>
+          <name>org.apache.uima.SentenceAnnotation</name>
+          <description>sentence annotation</description>
+          <supertypeName>uima.tcas.Annotation</supertypeName>
+        </typeDescription>
+      </types>
+    </typeSystemDescription>
+    <typePriorities/>
+    <fsIndexCollection/>
+    <capabilities>
+      <capability>
+        <inputs>
+          <type>org.apache.uima.TokenAnnotation</type>
+          <type allAnnotatorFeatures="true">org.apache.uima.SentenceAnnotation</type>
+          <feature>org.apache.uima.TokenAnnotation:end</feature>
+          <feature>org.apache.uima.TokenAnnotation:begin</feature>
+        </inputs>
+        <outputs>
+          <type>org.apache.uima.TokenAnnotation</type>
+          <feature>org.apache.uima.TokenAnnotation:posTag</feature>
+          <feature>org.apache.uima.TokenAnnotation:end</feature>
+          <feature>org.apache.uima.TokenAnnotation:begin</feature>
+        </outputs>
+        <languagesSupported/>
+      </capability>
+    </capabilities>
+    <operationalProperties>
+      <modifiesCas>true</modifiesCas>
+      <multipleDeploymentAllowed>true</multipleDeploymentAllowed>
+      <outputsNewCASes>false</outputsNewCASes>
+    </operationalProperties>
+  </analysisEngineMetaData>
+  <externalResourceDependencies>
+    <externalResourceDependency>
+      <key>Model</key>
+      <description>HMM Tagger model file</description>
+      <interfaceName>org.apache.uima.examples.tagger.IModelResource</interfaceName>
+      <optional>false</optional>
+    </externalResourceDependency>
+  </externalResourceDependencies>
+  <resourceManagerConfiguration>
+    <externalResources>
+      <externalResource>
+        <name>ModelFile</name>
+        <description>HMM Tagger model file</description>
+        <fileResourceSpecifier>
+          <fileUrl>file:english/BrownModel.dat</fileUrl>
+        </fileResourceSpecifier>
+        <implementationName>org.apache.uima.examples.tagger.ModelResource</implementationName>
+      </externalResource>
+    </externalResources>
+    <externalResourceBindings>
+      <externalResourceBinding>
+        <key>Model</key>
+        <resourceName>ModelFile</resourceName>
+      </externalResourceBinding>
+    </externalResourceBindings>
+  </resourceManagerConfiguration>
 </analysisEngineDescription>

Modified: incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml (original)
+++ incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml Fri Feb 29 05:25:15 2008
@@ -1,23 +1,23 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!--
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-    
-    http://www.apache.org/licenses/LICENSE-2.0
-    
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.    
--->
-
+<!--
+	Licensed to the Apache Software Foundation (ASF) under one
+	or more contributor license agreements.  See the NOTICE file
+	distributed with this work for additional information
+	regarding copyright ownership.  The ASF licenses this file
+	to you under the Apache License, Version 2.0 (the
+	"License"); you may not use this file except in compliance
+	with the License.  You may obtain a copy of the License at
+	
+	http://www.apache.org/licenses/LICENSE-2.0
+	
+	Unless required by applicable law or agreed to in writing,
+	software distributed under the License is distributed on an
+	"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+	KIND, either express or implied.  See the License for the
+	specific language governing permissions and limitations
+	under the License.    
+-->
+
 <analysisEngineDescription
 	xmlns="http://uima.apache.org/resourceSpecifier">
 	<frameworkImplementation>

Modified: incubator/uima/sandbox/trunk/Tagger/docbook/hmmTaggerUsersGuide/hmmTaggerUsersGuide.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/docbook/hmmTaggerUsersGuide/hmmTaggerUsersGuide.xml?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/docbook/hmmTaggerUsersGuide/hmmTaggerUsersGuide.xml (original)
+++ incubator/uima/sandbox/trunk/Tagger/docbook/hmmTaggerUsersGuide/hmmTaggerUsersGuide.xml Fri Feb 29 05:25:15 2008
@@ -27,7 +27,7 @@
 <book lang="en">
 
 	<title>Tagger Annotator Documentation</title>
-    <subtitle>.. work in progress..</subtitle>
+	
 
 	<xi:include xmlns:xi="http://www.w3.org/2001/XInclude"
 		href="../../../SandboxDocs/src/docbook/book_info.xml" />
@@ -121,23 +121,18 @@
 		<section id="sandbox.tagger.annotatorDescriptor.configParam">
 			<title>Configuration Parameters</title>
 			<para>
-				The HMM tagger annotator requires the following
-				configuration parameter:
+				The HMM tagger annotator (
+				<code>HmmTagger.xml</code>
+				) requires the following configuration parameters:
 			</para>
 			<para>
 				<itemizedlist>
 					<listitem>
 						<para>
-							<code>PARAM_FILE</code>
-							- This parameter is a String defining the
-							name of a tagger's parameter file (default
-							is
-							<code>tagger.properties</code>
-							), which is in Java properties file format.
-							The parameter file must be specified using a
-							relative path that is available in the UIMA
-							datapath or in the classpath.
-
+							<code>NGRAM_SIZE</code>
+							- this parameter is an Integer, defining
+							whether a bi- or trigram model should be
+							used for tagging (default is N=3).
 							<programlisting><emphasis><![CDATA[    <configurationParameters>
       <configurationParameter>
         <name>NGRAM_SIZE</name>
@@ -154,7 +149,14 @@
         </value>
       </nameValuePair>
     </configurationParameterSettings>]]></emphasis></programlisting>
-
+	</para>
+						</listitem>
+				  	
+				  	
+	<listitem>
+						<para>
+							<code>ModelFile</code>
+							- binary file containing the statistical model which should be used for tagging is defined as an external resource
 <programlisting><emphasis><![CDATA[   
     <externalResources>
       <externalResource>
@@ -167,9 +169,14 @@
       </externalResource>
     </externalResources>]]></emphasis></programlisting>
 
+Thus, one can easily use a different model by changing the <code>fileUrl</code> line: 
+<code>file:german/TuebaModel.dat</code>. 
+(NB. <emphasis>New models must be located in the <code>resources</code> folder</emphasis>.)
+After these two parameters have been set, the tagger is ready to use.  
+	
 							</para>
 						</listitem>
-				  	</itemizedlist>
+				  	</itemizedlist>
 				  	</para>
 				
 			</section>
@@ -206,7 +213,7 @@
 	<chapter id="sandbox.tagger.unittest">
 		<title>Functionality Test</title>
 		<para>
-			The <code>TaggerTest</code> is a JUnit test file (available in the folder <code>unittests</code>), 
+			The <code>TaggerTest</code> is a JUnit test file (available in the <code>test</code> folder), 
 			which provides an opportunity to test provided models for English and German, 
 			as well as the basic functionality of the tagger. In order to check whether 
 			the tagger's configuration is correct, just run this file as JUnit Test and you should get the following output:
@@ -237,7 +244,7 @@
 	</chapter>
 	
 	<chapter id="sandbox.tagger.tagger">
-		<title>Overview of the Tagger Package</title>
+		<title>Overview of the Tagger package</title>
 		<para>
 			The package <code>org.apache.uima.examples.tagger</code> contains:
 			<itemizedlist>
@@ -247,21 +254,14 @@
 							<orderedlist>
 								<listitem>
 									<para>
-									 <code>MappingInterface</code>
-									 - defines mapping for a tagset. For example, one may wish to map a more detailed tagset 
-									 to a less distinctive one (i.e. tell a program to tag all verbs as just <code>VERB</code>
-									 instead of differentiating between <code>verb infinitive</code>, <code>verb imperative</code>, etc.
-									
-									Two sample implementations for <code>MappingInterface</code> are included, 
-									namely <code>TagMapping</code> (mapping reducing Brown corpus tagset from more than 400 tags to ca. 93) and 
-									<code>GrobMapping</code>(mapping German STTS tagset from 54 tags to basic 
-									11 categories plus special symbols and punctuation)
- 									</para>
+									<code>IModelResource</code>
+									- model resource interface
+									 </para>
 								</listitem>
 								<listitem>
 									<para>
 									 <code>Tagger</code>
-									 - general tagger interface in case one would want to integrate further tagger types. 
+									 - general tagger interface, in case one would want to integrate further tagger types. 
 									</para>
 								</listitem>
 							</orderedlist>
@@ -269,7 +269,7 @@
 							</para>
 					</listitem>
 					<listitem>
-						<para> two basic classes:
+						<para> three classes:
 						<orderedlist>
 								<listitem>
 									<para>
@@ -285,13 +285,18 @@
 									 - implementation of the Viterbi Algorithm. This class makes up the core of the tagger.
 									</para>
 								</listitem>
-
+								<listitem>
+									<para>
+									 <code>ModelResource.java</code>
+									 - implementation of the <code>IModelResource</code>
+									</para>
+								</listitem>
 							</orderedlist>
 						
 						</para>
 					</listitem>
-			</itemizedlist>  
-		</para>
+			</itemizedlist>
+				</para>
 	</chapter>
 	
 	
@@ -305,15 +310,29 @@
 		
 			 
 			<itemizedlist>
-			
-			
+				<listitem>
+						<para>
+							<code>MappingInterface</code>
+									 - defines mapping for a tagset. For example, one may wish to map a more detailed tagset 
+									 to a less distinctive one (i.e. tell a program to tag all verbs as just <code>VERB</code>
+									 instead of differentiating between <code>verb infinitive</code>, <code>verb imperative</code>, etc.
+									
+									Two sample implementations for <code>MappingInterface</code> are included, 
+									namely <code>TagMappingBrown</code> (mapping reducing Brown corpus tagset from more than 400 tags to 93) and 
+									<code>GrobMappingTueba</code>(mapping German STTS tagset from 54 tags to basic 11 categories plus special symbols and punctuation)
+ 						</para>
+					</listitem>
+					
 					<listitem>
 						<para>
 							<code>ModelGeneration</code>
-							- trains an N-gram model for the tagger, iterating over the files from some predefined 
-							training directory. Writes the resulting model to a binary file. At the moment,
+							- trains an N-gram model for the tagger, iterating over a List of <code>Token</code>s. 
+							Writes the resulting model to a binary file. At the moment,
 							only bi-and trigram models are supported. Further N-grams can be easily integrated.
-							
+							<code>ModelGeneration</code> is not concerned with the fact, 
+							whether the training corpus is given as a single file or as a directory containing a number of files,
+							as this is a <code>CORPUS_READER</code> implementation issue. Two supplied readers include both a reader 
+							for a corpus as a single file (<code>TT_FormatReader</code>code>) or as a directory (<code>BrownReader</code>code>)
 							</para>
 					</listitem>
 					<listitem>
@@ -338,7 +357,7 @@
 									<para>
 									 <code>TT_FormatReader</code>
 									-  for the corpora in TreeTagger format, i.e. one word per line 
-									with tags separated from the words by tab. 
+									with tags separated from the words by tabs. 
 									</para>
 							</listitem>
 							
@@ -347,75 +366,67 @@
 							
 							</para>
 					</listitem>
-		
-					
-					<listitem>
-						<para>
-							<code>SuffixTree</code>
-							- Java implementation of the Ukkonen's suffix tree, used to gain suffix statistics for unknown words.
-						</para>
-					</listitem>
-					
-					<listitem>
-						<para>
-							 <code>Token</code>
-							 - is used as a kind of an interface between 
-							 the text and the annotator. Every tokenized unit of text should be defined as a <code>Token</code>
-							 .
-						</para>
-					</listitem>
-					
-						<listitem>
-						<para>
-							<code>TaggerEvaluation</code>
-							- can be used to evaluate the tagger and/or new models on a small manually annotated corpus. </para>
-					</listitem>
 					
 				</itemizedlist>
 				</para>
 		<para>
-		To train a new model, one should specify .... 
-		</para>
-	
-	<para>
-	An example of a parameter file for training is shown in the listing below:
-				
-				<programlisting><emphasis><![CDATA[## This is the default tagger.properties file
+		To train a new model, one should  adjust a number of parameters in the <code>"tagger.properties"</code> file, 
+		which is in Java properties file format (see <xref linkend="properties.file"/>). After the parameters are set, you just need to run 
+		<code>ModelGeneration.java</code>
+				<programlisting id="properties.file" xreflabel="tagger.properties file"><emphasis><![CDATA[######## This is the default tagger.properties file
+######## This file is used for training and testing only,
+######## The configuration for tagging is directly tuned in the descriptor "HmmTagger.xml"
+
+
+##########################  BOTH FOR TRAINING AND EVALUATION  ################################
+
+######## THESE ARE THE DEFAULT MODEL FILES FOR GERMAN AND ENGLISH
+######## You can either uncomment one of them, if you want to replace given models with your own one,
+
+#MODEL_FILE = resources/german/TuebaModel.dat
+#MODEL_FILE = resources/english/BrownModel.dat
+
+######## or specify a completely different name
+MODEL_FILE = 
 
-## if mapping of tags is desired
+######## If mapping of tags is desired, uncomment the following
 #DO_MAPPING = true
 
-## Basic mapping for the Brown corpus (nltk distribution) tagset: to get 93 tags out of ca. 460
-#MAPPING = org.apache.uima.examples.tagger.TagMapping
 
+#######	EXAMPLES OF MAPPING CLASSES 
 
+## Basic mapping for the Brown corpus (nltk distribution) tagset: to get 93 tags out of 473
+#MAPPING = org.apache.uima.examples.tagger.TagMappingBrown
 ## Basic mapping for STTS tagset: from 54 tags onto the basic ca. 15 classes plus punctuation
-#MAPPING = org.apache.uima.examples.tagger.GrobMapping]]></emphasis>
+#MAPPING = org.apache.uima.examples.tagger.GrobMappingTueba
+
+## If you implement your own mapping, you should specify here in the same manner as above a java-path to the class  
+MAPPING = 
+
+####### FILE CONTAINING TRAINING CORPUS:
+####### can be in specified either as an absolute or as a relative path
+####### e.g. FILE = ../../tueba_tigerFormat.txt or FILE = C:/Data/tueba.txt
+FILE = 
+
+######## If corpus is in a different format and cannot be read with the provided READERS,
+######## you should specify here a java-path to the class (s. examples below) 
+
+#CORPUS_READER = org.apache.uima.examples.tagger.trainAndTest.TT_FormatReader
+#CORPUS_READER = org.apache.uima.examples.tagger.trainAndTest.BrownReader
+CORPUS_READER = 
+
+#################      ONLY FOR EVALUATION   ###############################
+
+######### GOLD STANDARD CORPUS FILE: 
+######### can be specified as an absolute or as a relative path 
+## e.g. GOLD_STANDARD = ../../tueba_tigerFormat.txt or GOLD_STANDARD = C:/Data/tueba.txt
+GOLD_STANDARD = 
+
+######### Here we specify whether one intends to test a bi- or a trigram model (default is a trigram model)  
+N=3
+]]></emphasis>
 				</programlisting>
-				
-				Thus, parameter file defines the following values:
-				<itemizedlist>
-					<listitem>
-						<para>
-							<code>MODEL</code>
-							- defines the relative path to the file containing the trained model
-						</para>
-					</listitem>
-					<listitem>
-						<para>
-							<code>N</code>
-							- sets up whether we want to use a bi- (N=2) or a trigram (N=3) model for tagging (default is trigram)
-						</para>
-					</listitem>
-					<listitem>
-						<para>
-							<code>MAPPING</code>
-							- this parameter is useful in two scenarios: either if mapping of tags is desired after tagging
-							is performed with a given model, or if one is willing to train and test a model (s. <xref linkend="sandbox.tagger.training"/>) 
-							for a modified tagset than the one used by a manually annotated corpus.   
-							</para>
-					</listitem>
-				</itemizedlist>
+			
 				</para>
 	
 	</chapter>
@@ -423,8 +434,23 @@
 	<chapter id="sandbox.tagger.evaluation">
 		<title>Evaluation</title>
 		<para>
-			To evaluate performance if a "gold standard" corpus is given, ....
-		
+		To evaluate performance if a "gold standard" corpus is available, one can use the following provided file:
+		<itemizedlist>
+			<listitem>
+				<para>
+					<code>TaggerEvaluation.java</code>
+						- can be used to evaluate the tagger and/or new models on a manually annotated corpus. 
+				</para>
+			</listitem>
+		</itemizedlist>
+		</para>
+		<para>
+		<code>HMMTagger</code> was evaluated for English and German. For English, it was trained on 80% of the Brown corpus 
+		(180,000 tokens) and tested on the rest unseen 20%. The achieved accuracy was about 96%, test corpus contained 4.5% of unknown tokens.
+		</para>
+		<para>
+		For German, it achieves between 95% and 96% accuracy when trained and tested on the same type of corpus, i.e. with 80% of corpus used for training and 20% for testing.
+		The accuracy goes a bit down when tagging is performed for different types of corpora than the training one, mostly due to the growing number of unknown words.  
 		</para>
 	</chapter>
 

Modified: incubator/uima/sandbox/trunk/Tagger/resources/english/BrownModel.dat
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/resources/english/BrownModel.dat?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
Binary files - no diff available.

Modified: incubator/uima/sandbox/trunk/Tagger/resources/german/TuebaModel.dat
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/resources/german/TuebaModel.dat?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
Binary files - no diff available.

Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java Fri Feb 29 05:25:15 2008
@@ -34,6 +34,7 @@
 import org.apache.uima.analysis_engine.annotator.AnnotatorConfigurationException;
 import org.apache.uima.cas.FSIterator;
 import org.apache.uima.cas.text.AnnotationIndex;
+import org.apache.uima.examples.tagger.trainAndTest.MappingInterface;
 import org.apache.uima.examples.tagger.trainAndTest.ModelGeneration;
 import org.apache.uima.jcas.JCas;
 import org.apache.uima.resource.ResourceInitializationException;
@@ -184,13 +185,7 @@
       List<String> wordTagList = Viterbi.process(this.N, wordList, this.my_model.suffix_tree, this.my_model.suffix_tree_capitalized, this.my_model.transition_probs,
               this.my_model.word_probs, this.my_model.lambdas2, this.my_model.lambdas3, this.my_model.theta);
 
-      
-      /*
-      if (this.MAPPING != null){
-        wordTagList = this.MAPPING.map_tags(wordTagList);
-      }
-      */
-      
+     
       try {
         for (int i = 0; i < tokenList.size(); i++) {
           TokenAnnotation token = tokenList.get(i);

Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/NGram.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/NGram.java?rev=632315&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/NGram.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/NGram.java Fri Feb 29 05:25:15 2008
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.examples.tagger;
+
+import java.io.Serializable;
+
+
+// Helper class
+
+public class NGram implements Serializable{
+
+  private static final long serialVersionUID = 1L;
+  public String tag1;
+  public String tag2;
+  public String tag3;
+  
+ 
+  public NGram(){
+    this(null,null,null);
+  }
+  public NGram(String tag1){
+    this(tag1, null,null);
+  }
+  
+  public NGram(String tag1, String tag2){
+    this(tag1, tag2, null);
+  }
+  
+  public NGram(String tag1, String tag2, String tag3 ){
+    this.tag1 = tag1;
+    this.tag2 = tag2;
+    this.tag3 = tag3;
+  }
+  
+ 
+ 
+  @Override
+  public boolean equals(Object o) {
+    if (o instanceof NGram) {
+      NGram n = (NGram) o;
+      // case: trigram
+      if (this.tag2!=null && this.tag3!=null){
+        return (this.tag1.equals(n.tag1)) && (this.tag2.equals(n.tag2)) && (this.tag3.equals(n.tag3));
+      }
+      // case: bigram
+      else if (this.tag2!=null && this.tag3 == null){
+        return (this.tag1.equals(n.tag1)) && (this.tag2.equals(n.tag2));
+      } 
+      // case onegram
+      else {
+        return (this.tag1.equals(n.tag1));
+      }
+     } else{
+    return false;
+     }
+  }
+  
+  
+  
+  
+  public int hashCode(){
+    int hash = 7;
+    hash = 31 * hash + this.tag1.hashCode();
+    if (this.tag2!=null){
+      hash = 31 * hash + this.tag2.hashCode();
+    }
+    if (this.tag3!=null){
+     hash = 31 * hash + this.tag3.hashCode();
+    }
+    return hash;
+  }
+
+  
+  /* in case we would use char[] instead of Strings.. didn't help to reduce the model:(
+  public boolean equals(Object o) {
+    if (o instanceof NGram) {
+      NGram n = (NGram) o;
+      // case: trigram
+      if (this.tag2!=null && this.tag3!=null){
+        return (Arrays.equals(this.tag1, n.tag1)) && (Arrays.equals(this.tag2, n.tag2)) && (Arrays.equals(this.tag3, n.tag3));
+      }
+      // case: bigram
+      else if (this.tag2!=null && this.tag3 == null){
+        return (Arrays.equals(this.tag1, n.tag1)) && (Arrays.equals(this.tag2, n.tag2));
+      } 
+      // case onegram
+      else {
+        return (Arrays.equals(this.tag1, n.tag1));
+      }
+     } else{
+    return false;
+     }
+  }
+  
+  public int hashCode(){
+    int hash = 7;
+    hash = 31 * hash + Arrays.hashCode(this.tag1);
+    if (this.tag2!=null){
+      hash = 31 * hash + Arrays.hashCode(this.tag1);
+    }
+    if (this.tag3!=null){
+     hash = 31 * hash + Arrays.hashCode(this.tag1);
+    }
+    return hash;
+  }
+  
+  */
+  
+}
+
+

Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Viterbi.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Viterbi.java?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Viterbi.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Viterbi.java Fri Feb 29 05:25:15 2008
@@ -34,6 +34,9 @@
  */
 public class Viterbi {
 
+  static NGram ngram;
+  static NGram ngram2;
+
   @SuppressWarnings("unchecked")
   public static Map<String, List> init_probs(Map<String, Double> pos_s) {
     Map<String, List> init_probs = new HashMap<String, List>();
@@ -76,7 +79,7 @@
    */
   @SuppressWarnings("unchecked")
   public static List process(int N, List<String> sentence,
-      Map<String, Map<String, Double>> suffix_tree,Map<String,Map<String, Double>> suffix_tree_cap, Map<String, Double> transition_probs,
+      Map<String, Map<String, Double>> suffix_tree,Map<String,Map<String, Double>> suffix_tree_cap, Map<NGram, Double> transition_probs,
       Map<String, Map<String, Double>> word_probs, double[] lambdas2, double[] lambdas3, double theta) {
 
     // here we add something like an "end-of-sentence" tag at the beginning of the first sentence, to get probabilities for a certain tag to be at the beginning of the sentence.
@@ -191,13 +194,17 @@
       /** ********************************************************* */
       /* At INDUCTION step the algorithm looks at the total and local possibilities of the next
        * state(=tag), given a current observation (=token) If a token is known, we limit the coming
-       * tags by only possible ones for this token
+       * tags by only possible ones for this token from the dictionary;
+       * otherwise: we use suffix analysis to find possible POS tags for the unknown words 
        */
 
       Map<String, Double> possible_pos_next = new HashMap<String, Double>(); // possible pos of the
       // next token
       
-   /*   Matcher m2 = p.matcher(token);
+   // Here we tell the tagger that every identified number can only be a cardinal. 
+    //  It didn't improve the accuracy, even v.v. That is why it is uncommented.
+    /*
+    Matcher m2 = p.matcher(token);
     boolean b = m2.matches();
     
       if (b) {
@@ -209,7 +216,7 @@
       if (word_probs.containsKey(sentence.get(i + 1))) { // if the next token is known
         possible_pos_next = word_probs.get(sentence.get(i + 1)); // get possible POS of the next
      
-      } else   
+      } else   // if the token is unknown, then get possible tags from the suffix analysis
       {
           Map<String, Map<String, Double>> suffix_tree_local;
           // if a word is capitalized ... 
@@ -226,7 +233,7 @@
             if(suffix_tree_local.containsKey(suffix)){ 
               // get available POS with corresponding counts for the longest suffix
                 Map available_pos_zwischen =suffix_tree_local.get(suffix); 
-                // 
+                // smooth suffix probabilities
               for(int suf=j+1; suf<unknown.length;suf++){
                 String subsuffix = sentence.get(i+1).substring(suf, unknown.length);
                 Map pos = new HashMap();
@@ -241,9 +248,7 @@
                            // smooth suffix probability P(suffix|tag)
                            double zwischen_prob = (value + theta*(Double)available_pos_zwischen.get(key))/(1+theta);
                            available_pos_zwischen.put(key, zwischen_prob);
-                         }/* else {
-                           available_pos_zwischen.put(key, value);
-                         }*/
+                         }
                     }
                 } else {
                   Iterator posValuePairs2 = available_pos_zwischen.entrySet().iterator(); // iterate over words
@@ -261,7 +266,7 @@
               }
               possible_pos_next = available_pos_zwischen;
               break;
-              }
+              } // for unexpected cases which are neither in the dictionary nor in the suffix analysis 
             else if (j==unknown.length-1){
               possible_pos_next = word_probs.get("(");
           }
@@ -298,37 +303,37 @@
 
           // Double prob_local = (Double) probs.get(0); // total_probability for
           // forward algorihtm
-          List path_local = (List) probs.get(1); // path to that token
+          List<String> path_local = (List) probs.get(1); // path to that token
 
           Double vprob_local = (Double) probs.get(2); // get viterbi probability for this token
 
-          String ngram = null;
-          String ngram2 = null; // for smoothing of transition probs, at the moment for unknown grams only
-          
           /*
            * The following 4 lines can be modified to get any other number of N-grams
            */
 
           /* -- from here -- */
-       // um den ersten trigramm abzufangen
+          // um den ersten trigramm abzufangen
           if (N == 3 && i == 1) {
-            ngram = path_local.get(path_local.size() - 1) + "_" + key_next;
-       
-          }  else if (N == 2) {
+            // ngram = path_local.get(path_local.size() - 1) + "_" + key_next;
+             ngram = new NGram(path_local.get(path_local.size() - 1), key_next);       
             
-            ngram = path_local.get(path_local.size() - 1) + "_" + key_next;
+          }  else if (N == 2) {
             
+            //ngram = path_local.get(path_local.size() - 1) + "_" + key_next;
+              ngram = new NGram(path_local.get(path_local.size() - 1), key_next);
           } else if (N == 3 && i != 1) {
-            ngram = path_local.get(path_local.size() - 2) + "_"
-                + path_local.get(path_local.size() - 1) + "_" + key_next;
-            ngram2 = path_local.get(path_local.size() - 1) + "_" + key_next;
-            
+        //    ngram = path_local.get(path_local.size() - 2) + "_"
+        //        + path_local.get(path_local.size() - 1) + "_" + key_next;
+        //    ngram2 = path_local.get(path_local.size() - 1) + "_" + key_next;
+              ngram = new NGram(path_local.get(path_local.size() - 2),path_local.get(path_local.size() - 1), key_next);  
+              ngram2 = new NGram(path_local.get(path_local.size() - 1), key_next);
           } else {
             System.err.println("at the moment only bi-and trigramms are supported");
           }
 
           /* -- till here -- */
 
+
           double pp = 0;
           
           // smoothing only unknown n-grams strategy
@@ -346,7 +351,7 @@
              double lambda1 = lambdas2[0];
                  double lambda2 = lambdas2[1];
                
-                 ppp = (transition_probs.containsKey(ngram)) ? ((lambda2*transition_probs.get(ngram))+(lambda1*transition_probs.get(key_next))) : (lambda1*transition_probs.get(key_next));
+                 ppp = (transition_probs.containsKey(ngram)) ? ((lambda2*transition_probs.get(ngram))+(lambda1*transition_probs.get(new NGram(key_next)))) : (lambda1*transition_probs.get(new NGram(key_next)));
                  pp = Math.log(value_next) + Math.log(ppp); // P(t|w) * P(t1,t2,t3)
            }
            else if (N==3){
@@ -357,17 +362,17 @@
               if(transition_probs.containsKey(ngram)){
               
                 
-                ppp = (lambda3*transition_probs.get(ngram))+(lambda2*transition_probs.get(ngram2))+(lambda1*transition_probs.get(key_next));
+                ppp = (lambda3*transition_probs.get(ngram))+(lambda2*transition_probs.get(ngram2))+(lambda1*transition_probs.get(new NGram(key_next)));
               } else {
               //  System.out.println(ngram2);
-                 ppp = (transition_probs.containsKey(ngram2)) ? ((lambda2*transition_probs.get(ngram2))+(lambda1*transition_probs.get(key_next))) : (lambda1*transition_probs.get(key_next));
+                 ppp = (transition_probs.containsKey(ngram2)) ? ((lambda2*transition_probs.get(ngram2))+(lambda1*transition_probs.get(new NGram(key_next)))) : (lambda1*transition_probs.get(new NGram(key_next)));
               }pp = Math.log(value_next) + Math.log(ppp);
             }
            if (N ==2){
              double lambda1 = lambdas2[0];
                  double lambda2 = lambdas2[1];
                
-                 ppp = (transition_probs.containsKey(ngram)) ? ((lambda2*transition_probs.get(ngram))+(lambda1*transition_probs.get(key_next))) : (lambda1*transition_probs.get(key_next));
+                 ppp = (transition_probs.containsKey(ngram)) ? ((lambda2*transition_probs.get(ngram))+(lambda1*transition_probs.get(new NGram(key_next)))) : (lambda1*transition_probs.get(new NGram(key_next)));
                  pp = Math.log(value_next) + Math.log(ppp);
            }
    
@@ -396,7 +401,6 @@
       }
       all = m;
     }
-  //  System.out.println("cardinals found: "+cardinals);
     /** ****************************************************** */
     /***********************************************************************************************
      * Best sequence identification / Termination / Backtracking /

Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/BrownReader.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/BrownReader.java?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/BrownReader.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/BrownReader.java Fri Feb 29 05:25:15 2008
@@ -28,8 +28,6 @@
 import java.util.List;
 import java.util.regex.Pattern;
 
-import org.apache.uima.examples.tagger.MappingInterface;
-
 /**
  * Reader for Brown Corpus from NLTK Distribution (nltk.sourceforge.net)
  */
@@ -40,10 +38,6 @@
    * Reads Brown Corpus from NLTK Distribution Format. Iterates over all files in the directory,
    * which are in a sentence per line format, and returns all tokens in the collection in a List of
    * {@link Token}s}
-   * 
-   * @param directory
-   *          an array of file names
-   * @return a list of tokens from all files
    * 
    */
 

Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/CorpusReader.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/CorpusReader.java?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/CorpusReader.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/CorpusReader.java Fri Feb 29 05:25:15 2008
@@ -21,17 +21,14 @@
 
 
 import java.util.List;
-import org.apache.uima.examples.tagger.MappingInterface;
 
 
 /**
- * Reads (annotated) text file(s) and transforms every word into a {@code Token}-object 
+ * Reads (annotated) text file or directory and transforms every word into a {@code Token}-object 
  *
  */
 public interface CorpusReader {
 
- // public List<Token> corpus = new ArrayList<Token>();
- // public List<Token> read_corpus(String file);
   public List<Token> read_corpus(String file, MappingInterface mapping);
 }
 

Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/GrobMappingTueba.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/GrobMappingTueba.java?rev=632315&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/GrobMappingTueba.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/GrobMappingTueba.java Fri Feb 29 05:25:15 2008
@@ -0,0 +1,90 @@
+/*
+ *Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * 
+ */
+
+package org.apache.uima.examples.tagger.trainAndTest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.uima.examples.tagger.trainAndTest.Token;
+
+/**
+ *
+ */
+public class GrobMappingTueba implements MappingInterface{
+
+	    
+      /**
+       * Defines mapping for List<{@link Token}>
+       * E.g. if we need to map tags, given a list of {@code Tokens}, we need to map the 
+       * {@code pos} field of every {@code Token} to a different {@code pos}.  
+       */
+	    public List<Token> map_tags(List<Token> tokens){
+		  
+	    	
+		//    for (int i=0; i<sentences.size(); i++){ // iterate over sentences
+		      
+		         List<Token> tokens2 = new ArrayList<Token>(tokens.size());
+		         
+		         for (int x=0; x<tokens.size(); x++){ // iterate over tokens of the sentence with their corresponding POS
+		           Token current_token = (Token)tokens.get(x);
+		          
+		           if(current_token.pos.startsWith("N")){
+		        	   current_token.pos="Noun";
+		           } 
+		           if(current_token.pos.startsWith("V")){
+		        	   current_token.pos="Verb";
+		           } 
+		           if(current_token.pos.startsWith("ADJ")){
+		        	   current_token.pos="Adjective";
+		           } 
+		           if(current_token.pos.startsWith("P")){
+		        	   current_token.pos="Pronoun";
+		           } 
+		           if(current_token.pos.startsWith("KO")){
+		        	   current_token.pos="Conjunction";
+		           } 
+		           if(current_token.pos.startsWith("AP")){
+		        	   current_token.pos="Preposition";
+		           } 
+		           if(current_token.pos.startsWith("PTK")){
+		        	   current_token.pos="PTK";
+		           } 
+		           if(current_token.pos.startsWith("ADV")){
+		        	   current_token.pos="Adverb";
+		           } 
+		           
+		           if(current_token.pos.startsWith("ART")){
+		        	   current_token.pos="Article";
+		           } 
+		           
+		           if(current_token.pos.startsWith("ITJ")){
+		        	   current_token.pos="Interjection";
+		           } 
+		           
+		           Token zwischen = new Token(current_token.word, current_token.pos);
+		   
+		           tokens2.add(zwischen);
+		         }
+            return tokens2;
+            
+		  }
+	    
+}

Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/MappingInterface.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/MappingInterface.java?rev=632315&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/MappingInterface.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/MappingInterface.java Fri Feb 29 05:25:15 2008
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.examples.tagger.trainAndTest;
+
+import java.util.List;
+
+/**
+ * Defines mapping for a tagset. For example, one may wish to map a more detailed tagset to a less
+ * distinctive one (i.e. tell a program to tag all verbs as just {@code VERB} instead of
+ * differentiating between {@code verb infinitive}, {@code verb imperative}, etc ... Receives a
+ * List of {@code Tokens} to transform and returns a List of {@code Tokens} with mapped parts of
+ * speech
+ */
+public interface MappingInterface {
+
+  public List<Token> map_tags(List<Token> tokens);
+
+}

Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/ModelGeneration.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/ModelGeneration.java?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/ModelGeneration.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/ModelGeneration.java Fri Feb 29 05:25:15 2008
@@ -67,7 +67,7 @@
    * Map containing N-gram probabilities 
    */
   
-  public Map<String, Double> transition_probs = new HashMap<String, Double>() ;
+  public Map<NGram, Double> transition_probs = new HashMap<NGram, Double>() ;
   
   
   static List<String> posList = new ArrayList<String>(); 
@@ -78,16 +78,11 @@
   public double [] lambdas3 = new double[3];
   public double theta; // for suffix probabiliites smoothing
   
- // transient String InputDir;
+ 
   transient String OutputFile;
   transient List corpus; 
   
-  /**
-   * @param N N=1, 2 or 3
-   * @param InputDir input directory name
-   * @param OutputFile output file name
-   *    MapBrownToPenn TagMapping
-   */
+
   @SuppressWarnings("unchecked")
    public ModelGeneration(List<Token> corpus, String OutputFile) {
     this.OutputFile = OutputFile;
@@ -102,21 +97,7 @@
     this.word_probs = (Map) l.get(0);
     this.suffix_tree = (Map) l.get(1);
     this.suffix_tree_capitalized = (Map) l.get(2);
-     /*  Map test2 = get_ngrams(2);
-     Map test3 = get_ngrams(3);
-     Map test1 = get_ngrams(1);
-       Iterator<Entry<String, Map>> it = test2.entrySet().iterator(); // iterate over words
-       
-     for (int g = 0; g<test2.size(); g++){
-       
-       Map.Entry entry = (Map.Entry) it.next();
-          ArrayList  key = (ArrayList ) entry.getKey();
-        Object value = entry.getValue();
-       System.out.println(key.toString());
-       System.out.println(value);
-     }
-     System.out.println("Number of ngrams: "+(test2.size()+test1.size()+test3.size()));
-   */  
+  
      System.out.println("Number of different words "+word_probs.size());
      System.out.println("Number of non-capitalized suffixes: "+suffix_tree.size());
      System.out.println("Number of capitalized suffixes: "+suffix_tree_capitalized.size());
@@ -203,8 +184,8 @@
     int mapsize = word_counts.size();
     
     Iterator<Entry<String, Map<String, Double>>> keyValuePairs = word_counts.entrySet().iterator(); // iterate over words
-    Map<String, Double> pos_counts = get_ngrams(1);
-
+    Map<NGram, Double> pos_counts = get_ngrams(1);
+    
     for (int i = 0; i < mapsize; i++)
     {
       Map.Entry<String, Map<String, Double>> entry = keyValuePairs.next();
@@ -218,12 +199,13 @@
       for (int u = 0; u < pos2.size(); u++)
       {
        Map.Entry<String, Double> entry_pos = keyValuePairs_pos.next();
-       Object key2 = entry_pos.getKey(); // pos of a word
-        
+       String key2 = entry_pos.getKey(); // pos of a word
+       System.out.println(pos_counts.size()); 
        if (key2 != "count") {
       
         Object value2 = entry_pos.getValue(); // its count
-          double freq_pos=pos_counts.get(key2);
+          NGram ng = new NGram(key2);
+          double freq_pos=pos_counts.get(ng);
           Double val2 = (Double)value2 / freq_pos; // Prob(w|t) = freq(w,t)/freq(t)
           lokal.put((String)key2, val2); // save probability as a log2
          } else {
@@ -397,31 +379,31 @@
    * @return Map<String, Double> N-grams of parts-of-speech, where {@code N = 1, 2 or 3} 
    * @throws IllegalArgumentException
    */
-  static Map<String, Double> get_ngrams(int N) throws IllegalArgumentException{
+  static Map<NGram, Double> get_ngrams(int N) throws IllegalArgumentException{
    
-    Map<String, Double> ngrams1= new HashMap<String,Double>();
-    Map<String, Double> ngrams2= new HashMap<String,Double>();
-    Map<String, Double> ngrams3= new HashMap<String,Double>();
+    Map<NGram, Double> ngrams1= new HashMap<NGram,Double>();
+    Map<NGram, Double> ngrams2= new HashMap<NGram,Double>();
+    Map<NGram, Double> ngrams3= new HashMap<NGram,Double>();
     
     if (N==1){
       for (int y=0; y<posList.size(); y++){
-        Double freq=ngrams1.get(posList.get(y));
-        // if a given POS is already in the Map, then add its corresponding count, otherwise add a POS value with a count of 1 
-        ngrams1.put(posList.get(y), (freq == null) ? 1 : freq + 1);
-      }
+        NGram onegram = new NGram(posList.get(y));
+        Double freq = ngrams1.get(onegram);
+        ngrams1.put(onegram, (freq == null) ? 1 : freq + 1);
+        }
     }
     else if (N==2){
       for (int y=0; y<(posList.size()-1); y++){
-       String s2 = posList.get(y)+"_"+posList.get(y+1);
-       Double freq=ngrams2.get(s2);
-       ngrams2.put(s2, (freq == null) ? 1 : freq + 1);
+       NGram bigram = new NGram(posList.get(y),posList.get(y+1));
+       Double freq=ngrams2.get(bigram);
+       ngrams2.put(bigram, (freq == null) ? 1 : freq + 1);
       }
     }
     else if (N==3){
       for (int y=0; y<(posList.size()-2); y++){
-       String s3 = posList.get(y)+"_"+posList.get(y+1)+"_"+posList.get(y+2);
-       Double freq=ngrams3.get(s3);
-       ngrams3.put(s3, (freq == null) ? 1 : freq + 1);
+       NGram trigram = new NGram(posList.get(y),posList.get(y+1),posList.get(y+2));
+       Double freq=ngrams3.get(trigram);
+       ngrams3.put(trigram, (freq == null) ? 1 : freq + 1);
       }
     } else{ 
         throw new IllegalArgumentException ("N=1, N=2 or N=3, no further N-grams are supported at the moment");
@@ -434,15 +416,15 @@
   * Computes {@code transition_probs} using {@link #get_ngrams(int)} frequency counts for N-grams..  
   */
    
-  static Map<String, Double> unigrams;
-  static Map<String, Double> bigrams;
-  static Map<String, Double> trigrams;
+  static Map<NGram, Double> unigrams;
+  static Map<NGram, Double> bigrams;
+  static Map<NGram, Double> trigrams;
 
   @SuppressWarnings("unchecked")
-  static Map<String, Double> get_transition_probs(int N) throws IllegalArgumentException{
-  Map<String, Double> probs1= new HashMap<String, Double>();
-    Map<String, Double> probs2= new HashMap<String, Double>();
-    Map<String, Double> probs3= new HashMap<String, Double>();
+  static Map<NGram, Double> get_transition_probs(int N) throws IllegalArgumentException{
+    Map<NGram, Double> probs1= new HashMap<NGram, Double>();
+    Map<NGram, Double> probs2= new HashMap<NGram, Double>();
+    Map<NGram, Double> probs3= new HashMap<NGram, Double>();
     unigrams = get_ngrams(1);
     bigrams = get_ngrams(2);
     trigrams = get_ngrams(3);
@@ -452,14 +434,10 @@
         for (int i = 0; i < unigrams.size(); i++) // for all bigrams
         {
           Map.Entry entry = (Map.Entry) keyValuePairs.next();
-          Object key = entry.getKey(); // get a bigram
-       //   String [] t = ((String)key).split("_");
-          double freq1 =  unigrams.get(key); // get a count of a  unigram
-
-    //      Object freq2 = entry.getValue(); // get  a count of a bigram
-          
+          Object key = entry.getKey(); // get an unigram
+          double freq1 =  unigrams.get(key); // get a count of a  unigram 
           double prob1 = freq1 / tokens_count_all_corpus; // Prob(key) = freq(tag)/freq(all_tags)?
-          probs1.put((String)key, prob1); // save probability as a log2 : Math.log(prob1)
+          probs1.put((NGram)key, prob1); // save probability as a log2 : Math.log(prob1)
         
         }
     }
@@ -470,14 +448,17 @@
       for (int i = 0; i < bigrams.size(); i++) // for all bigrams
       {
         Map.Entry entry = (Map.Entry) keyValuePairs.next();
-        Object key = entry.getKey(); // get a bigram
-        String [] t = ((String)key).split("_");
-        double freq1 =  unigrams.get(t[0]); // get a count of a preceding unigram
+        NGram key = (NGram) entry.getKey(); // get a bigram
+        double freq1 =  unigrams.get(new NGram(key.tag1)); // get a count of a preceding unigram
+
+        
+        //  String [] t = ((String)key).split("_");
+        // double freq1 =  unigrams.get(t[0]); // get a count of a preceding unigram
 
         Object freq2 = entry.getValue(); // get  a count of a bigram
         
         double prob2 = (Double)freq2 / freq1; // Prob(t2|t1) = freq(t1,t2)/freq(t1)
-        probs2.put((String)key, prob2); // save probability as a log2: Math.log(prob2)
+        probs2.put((NGram)key, prob2); 
       } 
     } else if (N==3){ // for trigram models
       
@@ -485,16 +466,19 @@
       for (int i = 0; i < trigrams.size(); i++) // for all trigrams
       {
         Map.Entry entry = (Map.Entry) keyValuePairs.next();
-        Object key = entry.getKey(); // get a trigram
-        String [] t = ((String)key).split("_");
-        String tt = t[0]+"_"+t[1];
+        NGram key = (NGram) entry.getKey(); // get a trigram
+        double freq1 =  bigrams.get(new NGram(key.tag1,key.tag2)); // get a count of a preceding bigram
+
+        
+        //String [] t = ((String)key).split("_");
+        //String tt = t[0]+"_"+t[1];
 
-        Double freq1 =  bigrams.get(tt); // get a count of a preceding bigram
+        // Double freq1 =  bigrams.get(tt); // get a count of a preceding bigram
 
         Object freq2 = entry.getValue(); // get  a count of a trigram
         
         double prob3 = (Double)freq2/freq1; // Prob(t3|(t1_t2)) = freq(t1_t2_t3)/freq(t1_t2)
-        probs3.put((String)key, prob3); // save probability as a log2: Math.log(prob3)
+        probs3.put((NGram)key, prob3); // save probability as a log2: Math.log(prob3)
      }
     } else{ 
       throw new IllegalArgumentException ("only uni-, bi-, and trigramms are supported at the moment");
@@ -502,7 +486,7 @@
     return ((N == 2) ? probs2 : (N==3)? probs3 : probs1);
   }
   
-  // TODO: ADD SMOOTHING, both for ngrams and evtl unknown suffixes (?)
+
   /**
    * Computes alphas for linear interpolation smoothing of unknown n-grams
    * @param N N-gram
@@ -523,9 +507,9 @@
         for (int i = 0; i < bigrams.size(); i++) // for all bigrams
         {
           Map.Entry entry = (Map.Entry) keyValuePairs.next();
-          Object key = entry.getKey(); // get a bigram
-          String [] t = ((String)key).split("_");
-          double freq1 =  unigrams.get(t[0]); // get a count of a preceding unigram
+          NGram key = (NGram)entry.getKey(); // get a bigram
+        
+          double freq1 =  unigrams.get(new NGram(key.tag1)); // get a count of a preceding unigram
           Double freq2 = (Double) entry.getValue(); // get  a count of a bigram
           
           double f2 = (freq2-1)/(freq1-1);
@@ -546,13 +530,13 @@
         for (int i = 0; i < trigrams.size(); i++) // for all trigrams
         {
           Map.Entry entry = (Map.Entry) keyValuePairs.next();
-          Object key = entry.getKey(); // get a trigram
-          String [] t = ((String)key).split("_");
-          String tt = t[0]+"_"+t[1];
-
-          Double freq2 = bigrams.containsKey(tt) ?  bigrams.get(tt) : 0.0; // get a count of a preceding bigram
+          NGram key = (NGram) entry.getKey(); // get a trigram
+      //    String [] t = ((String)key).split("_");
+       //   String tt = t[0]+"_"+t[1];
+          NGram preceding_bigram = new NGram(key.tag1, key.tag2);
+          Double freq2 = bigrams.containsKey(preceding_bigram) ?  bigrams.get(preceding_bigram) : 0.0; // get a count of a preceding bigram
           Double freq3 = (Double) entry.getValue(); // get  a count of a trigram
-          Double freq1 = unigrams.containsKey(t[0]) ? unigrams.get(t[0]) : 0.0;
+          Double freq1 = unigrams.containsKey(new NGram(key.tag1)) ? unigrams.get(new NGram(key.tag1)) : 0.0;
         
           double f3 = (freq3-1)/(freq2-1);
           double f2 = (freq2-1)/(freq1-1);
@@ -563,7 +547,7 @@
           else {lambda1+=freq3;count3+=freq3;}
         }  this.lambdas3[0] = lambda1/count3; this.lambdas3[1]=lambda2/count3; this.lambdas3[2]=lambda3/count3;   
     }
-   System.out.println("lambdas for 2-grams = "+this.lambdas2[0]+" "+this.lambdas2[1]+"\n"+"lambdas for 3-grams"+this.lambdas3[0]+" "+this.lambdas3[1]+" "+this.lambdas3[2]);
+   System.out.println("lambdas: "+this.lambdas3[0]+" "+this.lambdas3[1]+" "+this.lambdas3[2]);
       return ((N == 2) ? this.lambdas2 : this.lambdas3);
   }
   
@@ -647,17 +631,14 @@
       in.close();
 
       file = defaultProps.getProperty("FILE");
-      fileOutput = defaultProps.getProperty("FILE_OUTPUT");
+      fileOutput = defaultProps.getProperty("MODEL_FILE");
 
       boolean DO_MAPPING;
       String b =  defaultProps.getProperty("DO_MAPPING");
       DO_MAPPING = Boolean.valueOf(b);
-      System.out.println("HALLO");
       if (DO_MAPPING){
         String m = defaultProps.getProperty("MAPPING");
          MAPPING = (MappingInterface)(Class.forName(m)).newInstance();
-         System.out.println("HALLO2");
-
       } else {
         MAPPING = null;
       }
@@ -666,9 +647,8 @@
       String r = defaultProps.getProperty("CORPUS_READER");
       reader = (CorpusReader)(Class.forName(r)).newInstance();
     
-  //  md = new ModelGeneration(new BrownReader().read_corpus("../brown",m),fileOutput);
-
-     System.out.println(fileOutput);
+      System.out.println("Input file: "+file);
+      System.out.println("Output model file: "+fileOutput);
       md = new ModelGeneration(reader.read_corpus(file, MAPPING),fileOutput);
       md.init();
     } catch (Exception e) {

Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/SuffixTree.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/SuffixTree.java?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/SuffixTree.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/SuffixTree.java Fri Feb 29 05:25:15 2008
@@ -139,13 +139,37 @@
    */
   
   public class EDGE_KEY {
-    int suffix_begin;
-    char suffix;
+    private int suffix_begin;
+    private char suffix;
 
     public EDGE_KEY(int i, char s) {
      suffix_begin = i;
      suffix = s;
     }
+    @Override
+    public boolean equals(Object o) {
+      if (o instanceof EDGE_KEY) {
+        EDGE_KEY ek = (EDGE_KEY) o;
+        if ((this.suffix_begin == ek.suffix_begin) && (this.suffix == ek.suffix)) {
+          return true;
+        }
+      }
+      return false;
+    }
+
+    public int hashCode(){
+      int hash = 0;
+      hash <<= 1;
+      if ( hash < 0 )
+      {
+        hash |= 1;
+      }
+      hash ^= suffix_begin;
+      hash ^= suffix;
+        
+      return hash;
+    }
+    
   }
   
   /**
@@ -187,6 +211,8 @@
     // the new leaves I create.
     // Each time I create a new edge, I also create a suffix pointer
     // from the parent node of the last leaf edge I created to the current parent edge.
+    // (Mark Nelson. Fast String Searching With Suffix Trees.
+    // http://marknelson.us/1996/08/01/suffix-trees/)
 
     public int split_edge(Suffix suffix) {
 
@@ -256,17 +282,17 @@
       } else if (active_point.isImplicit()) { // if suffix is implicit, i.e. it does not end in a
         // leaf node $
         
-        EDGE_KEY keys2 = new EDGE_KEY(active_point.origin_node, chars[active_point.first_char_index]); 
+          EDGE_KEY keys2 = new EDGE_KEY(active_point.origin_node, chars[active_point.first_char_index]); 
         
       //  List keys2 = new ArrayList();
       //  keys2.add(active_point.origin_node);
       //  keys2.add(chars[active_point.first_char_index]);
-        edge = (Edge) edges.get(keys2);
+          edge = (Edge) edges.get(keys2);
 
-        int span = active_point.last_char_index - active_point.first_char_index;
+          int span = active_point.last_char_index - active_point.first_char_index;
         // if the given prefix is already in the tree, do nothing
-        if (chars[edge.first_char_index + span + 1] == chars[last_char]) {
-          break;
+          if (chars[edge.first_char_index + span + 1] == chars[last_char]) {
+            break;
         } else {
           parent_node = edge.split_edge(active_point);
         }

Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TT_FormatReader.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TT_FormatReader.java?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TT_FormatReader.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TT_FormatReader.java Fri Feb 29 05:25:15 2008
@@ -32,25 +32,13 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.uima.examples.tagger.MappingInterface;
-
+/**
+ * TreeTagger Format Reader, i.e. "word-per-line with tags separated with tabs"
+ *
+ */
 
 public class TT_FormatReader implements CorpusReader{
-
-  /*
-  List<Token> corpus;
-
-  public TT_FormatReader(String InputFile, MappingInterface Mapping) {
-    corpus = read_corpus(InputFile);
-    if (Mapping == null) {
-      this.corpus = read_corpus(InputFile);
-    } else {
-      this.corpus = Mapping.map_tags(read_corpus(InputFile)); // in case we need to map
-    }
-  }
-*/
  
-
   public List<Token> read_corpus(String file, MappingInterface Mapping) {
 
     // Text is already tokenized

Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TagMappingBrown.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TagMappingBrown.java?rev=632315&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TagMappingBrown.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TagMappingBrown.java Fri Feb 29 05:25:15 2008
@@ -0,0 +1,81 @@
+/*
+ *Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * 
+ */
+
+package org.apache.uima.examples.tagger.trainAndTest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.uima.examples.tagger.trainAndTest.Token;
+
+public class TagMappingBrown implements MappingInterface{
+
+  /**
+   * Defines mapping for List<{@link Token}>
+   * E.g. if we need to map tags, given a list of {@code Tokens}, we need to map the 
+   * {@code pos} field of every {@code Token} to a different {@code pos}.  
+   * Basically the mapping performed in this class is just a case of simple "normalization",
+   * we just discard compound tags of the Brown corpus here.
+   */
+  @SuppressWarnings("unchecked")
+  public List map_tags(List tokens) {
+
+    // for (int i=0; i<sentences.size(); i++){ // iterate over sentences
+
+    List<Token> tokens2 = new ArrayList<Token>(tokens.size());
+
+    for (int x = 0; x < tokens.size(); x++) { // iterate over tokens of the sentence with their
+      // corresponding POS
+      Token current_token = (Token)tokens.get(x);
+      String[] z = new String[2];
+      if (current_token.pos.contains("+")) {
+        z = current_token.pos.split("[+]");
+        current_token.pos = z[0];
+      }
+      // for cases like : BEZ*
+      if (current_token.pos.contains("*") && !(current_token.pos.startsWith("*"))) {
+        z[0] = current_token.pos.replace("*", "");
+        current_token.pos = z[0];
+      }
+
+      // for: *-h1
+      if (current_token.pos.startsWith("*")) {
+        z[0] = "*";
+        current_token.pos = z[0];
+      }
+      if (current_token.pos.contains("-") && !(current_token.pos.startsWith("--"))) {
+        z = current_token.pos.split("[-]");
+        current_token.pos = z[0];
+      }
+      if (current_token.pos.startsWith("--")) {
+        z[0] = "--";
+        current_token.pos = z[0];
+      }
+
+      Token zwischen = new Token(current_token.word, current_token.pos);
+
+      tokens2.add(zwischen);
+    }
+    return tokens2;
+
+  }
+
+
+}

Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TaggerEvaluation.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TaggerEvaluation.java?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TaggerEvaluation.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TaggerEvaluation.java Fri Feb 29 05:25:15 2008
@@ -29,12 +29,14 @@
 import java.util.Properties;
 
 import org.apache.uima.examples.tagger.HMMTagger;
-import org.apache.uima.examples.tagger.MappingInterface;
 import org.apache.uima.examples.tagger.Viterbi;
 
+
 /**
- * Evaluation of Tagger NB. As it is implemented at hte moment, to be used just for small tests with
- * small files .. (very naive and takes quite a long time with big files..)
+ * Evaluation of Tagger 
+ * NB. As it is implemented at the moment, to be used just for small tests with
+ * small files. Takes quite a long time with big files. If one needs to test big files,
+ * it is better to implement iteration over sentences and not over the complete list of {@code Tokens}.
  */
 public class TaggerEvaluation {
 
@@ -45,7 +47,7 @@
     int wrong_tag = 0;
     int right_tags = 0;
     List unknown_list = new ArrayList<String>();
-    int unknown_count = 0; // counter for erroronously tagged unknown words
+    int unknown_count = 0; // counter for erroneously tagged unknown words
     int unknown_all = 0; // counter for all unknown words
     List wrong_tags = new ArrayList<String>();
     Map wrong_tag_counts = new HashMap();
@@ -54,8 +56,6 @@
     for (int u = 0; u < posList.size(); u++) {
 
       if (!posList.get(u).equalsIgnoreCase((String) TagList.get(u))) {
-        String test = TagList.get(u).toString().toLowerCase();
-
         wrong_tags.add(posList.get(u));
 
         Integer freq = (Integer) wrong_tag_counts.get(posList.get(u));
@@ -126,6 +126,7 @@
   /**
    * @param args
    */
+  @SuppressWarnings("unchecked")
   public static void main(String[] args) {
 
     ModelGeneration my_model;
@@ -143,8 +144,9 @@
       defaultProps.load(in);
       in.close();
 
-      String MODEL = defaultProps.getProperty("MODEL");
       String n = defaultProps.getProperty("N");
+      String MODEL = defaultProps.getProperty("MODEL_FILE");
+      
 
       my_model = HMMTagger.get_model(MODEL);
       String t = defaultProps.getProperty("DO_MAPPING");
@@ -163,11 +165,9 @@
       String r = defaultProps.getProperty("CORPUS_READER");
       reader = (CorpusReader) (Class.forName(r)).newInstance();
       file = defaultProps.getProperty("GOLD_STANDARD");
-
       List<Token> corpus = reader.read_corpus(file, MAPPING);
       List<String> wordList = new ArrayList<String>();
-      // List<String> posSent = new ArrayList<String>(); // for pos-s on the level of sentence
-
+     
       List<String> posList = new ArrayList<String>();
 
       for (int x = 0; x < corpus.size(); x++) { // iterate over tokens with their corresponding POS

Modified: incubator/uima/sandbox/trunk/Tagger/tagger.properties
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/tagger.properties?rev=632315&r1=632314&r2=632315&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/tagger.properties (original)
+++ incubator/uima/sandbox/trunk/Tagger/tagger.properties Fri Feb 29 05:25:15 2008
@@ -1,34 +1,58 @@
-## This is the default tagger.properties file
-## This file is used for tagging, training and testing
+######## This is the default tagger.properties file
+######## This file is used for training and testing only,
+######## The configuration for tagging is directly tuned in the descriptor "HmmTagger.xml"
 
-## uncomment for English
-#MODEL = resources/english/BrownModel.dat
+###########  ONLY FOR TRAINING #####################################################
 
-## uncomment for German
-MODEL = resources/german/TuebaModel.dat
 
-## N =2 or N= 3 are supported, default N=3 
+####### FILE OR DIRECTORY CONTAINING TRAINING CORPUS:
+####### can be in specified either as an absolute or as a relative path
+####### e.g. FILE = ../../tueba_tigerFormat.txt or FILE = C:/Data/tueba.txt
+FILE = 
 
-N = 3
 
-## If mapping of tags is desired, uncomment the following
-#DO_MAPPING = true
+##########################  BOTH FOR TRAINING AND EVALUATION  ################################
 
-## Basic mapping for the Brown corpus (nltk distribution) tagset: to get 93 tags out of 473
-MAPPING = org.apache.uima.examples.tagger.TagMapping
+######## THESE ARE THE DEFAULT MODEL FILES FOR GERMAN AND ENGLISH
+######## You can either uncomment one of them, if you want to replace given models with your own one,
+
+#MODEL_FILE = resources/german/TuebaModel.dat
+MODEL_FILE = resources/english/BrownModel.dat
+
+######## or specify a completely different name
+# MODEL_FILE = 
+
+######## If mapping of tags is desired, uncomment the following
+DO_MAPPING = true
 
 
+#######	EXAMPLES OF MAPPING CLASSES 
+
+## Basic mapping for the Brown corpus (nltk distribution) tagset: to get 93 tags out of 473
+MAPPING = org.apache.uima.examples.tagger.trainAndTest.TagMappingBrown
+
 ## Basic mapping for STTS tagset: from 54 tags onto the basic ca. 15 classes plus punctuation
-#MAPPING = org.apache.uima.examples.tagger.GrobMapping
+#MAPPING = org.apache.uima.examples.tagger.trainAndTest.GrobMappingTueba
+
+## If you implement your own mapping, you should specify here in the same manner as above a java-path to the class  
+#MAPPING = 
+
+######## If corpus is in a different format and cannot be read with the provided READERS,
+######## you should specify here a java-path to the class (s. examples below) 
+
+#CORPUS_READER = org.apache.uima.examples.tagger.trainAndTest.TT_FormatReader
+CORPUS_READER = org.apache.uima.examples.tagger.trainAndTest.BrownReader
+
+#CORPUS_READER = 
+
+#################      ONLY FOR EVALUATION   ###############################
+
+######### GOLD STANDARD CORPUS FILE: 
+######### can be specified as an absolute or as a relative path 
+##e.g. GOLD_STANDARD = ../../tueba_tigerFormat.txt or GOLD_STANDARD = C:/Data/tueba.txt
 
-### THIS SECTION SHOULD BE MODIFIED ONLY IN CASE OF OWN MODEL TRAINING
-#FILE = D:/Jane/IBM/Tagger_komplett/tueba_tigerFormat.txt
-#FILE = ../brown/
-#FILE_OUTPUT = resources/german/TuebaModel.dat
-#FILE_OUTPUT = resources/english/BrownModel.dat
-CORPUS_READER = org.apache.uima.examples.tagger.trainAndTest.TT_FormatReader
-#CORPUS_READER = org.apache.uima.examples.tagger.trainAndTest.BrownReader
+GOLD_STANDARD = 
 
+######### Here we specify whether one intends to test a bi- or a trigram model (default is a trigram model)  
+N=3
 
-### FOR EVALUATION IF GOLD STANDARD CORPUS IS PRESENT
-GOLD_STANDARD = C:/TreeTagger/src/corpora/WacGold_NormalizedLength30012008
\ No newline at end of file