You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by tw...@apache.org on 2008/02/08 18:08:16 UTC
svn commit: r619943 [1/2] - in /incubator/uima/sandbox/trunk/Tagger: ./
desc/ docbook/hmmTaggerUsersGuide/ metadata/ resources/english/
resources/german/ src/main/java/org/apache/uima/
src/main/java/org/apache/uima/examples/tagger/ src/main/java/org/ap...
Author: twgoetz
Date: Fri Feb 8 09:07:58 2008
New Revision: 619943
URL: http://svn.apache.org/viewvc?rev=619943&view=rev
Log:
Jira UIMA-753: commit Eugenie's latest patch, with some modifications.
See Jira issue for details.
https://issues.apache.org/jira/browse/UIMA-753
Added:
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/GrobMapping.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/MappingInterface.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/BrownReader.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/CorpusReader.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/ModelGeneration.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/SuffixTree.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TT_FormatReader.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/TaggerEvaluation.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/trainAndTest/Token.java
incubator/uima/sandbox/trunk/Tagger/src/test/
incubator/uima/sandbox/trunk/Tagger/src/test/java/
incubator/uima/sandbox/trunk/Tagger/src/test/java/org/
incubator/uima/sandbox/trunk/Tagger/src/test/java/org/apache/
incubator/uima/sandbox/trunk/Tagger/src/test/java/org/apache/uima/
incubator/uima/sandbox/trunk/Tagger/src/test/java/org/apache/uima/examples/
incubator/uima/sandbox/trunk/Tagger/src/test/java/org/apache/uima/examples/tagger/
incubator/uima/sandbox/trunk/Tagger/src/test/java/org/apache/uima/examples/tagger/test/
incubator/uima/sandbox/trunk/Tagger/src/test/java/org/apache/uima/examples/tagger/test/TaggerTest.java
Removed:
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/BrownReader.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/MapBrownToPenn.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/ModelGeneration.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/ModelGenerationBytes.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/SuffixTree.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/TT_FormatReader.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Token.java
Modified:
incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml
incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml
incubator/uima/sandbox/trunk/Tagger/docbook/hmmTaggerUsersGuide/hmmTaggerUsersGuide.xml
incubator/uima/sandbox/trunk/Tagger/metadata/install.xml
incubator/uima/sandbox/trunk/Tagger/pom.xml
incubator/uima/sandbox/trunk/Tagger/resources/english/BrownModel.dat
incubator/uima/sandbox/trunk/Tagger/resources/german/TuebaModel.dat
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation_Type.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation_Type.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/TagMapping.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Tagger.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Viterbi.java
incubator/uima/sandbox/trunk/Tagger/tagger.properties
Modified: incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml (original)
+++ incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml Fri Feb 8 09:07:58 2008
@@ -1,71 +1,101 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<analysisEngineDescription
- xmlns="http://uima.apache.org/resourceSpecifier">
- <frameworkImplementation>
- org.apache.uima.java
- </frameworkImplementation>
- <primitive>true</primitive>
- <annotatorImplementationName>
- org.apache.uima.examples.tagger.HMMTagger
- </annotatorImplementationName>
- <analysisEngineMetaData>
- <name>Hidden Markov Model - Part of Speech Tagger</name>
- <description>
- A configuration of the HmmTaggerAnnotator that looks for
- parts of speech of identified tokens within existing
- Sentence and Token annotations. See also
- WhitespaceTokenizer.xml.
- </description>
- <version>1.0</version>
- <vendor>The Apache Software Foundation</vendor>
- <configurationParameters>
- <configurationParameter>
- <name>PARAM_FILE</name>
- <type>String</type>
- <multiValued>false</multiValued>
- <mandatory>true</mandatory>
- </configurationParameter>
- </configurationParameters>
- <configurationParameterSettings>
- <nameValuePair>
- <name>PARAM_FILE</name>
- <value>
- <string>tagger.properties</string>
- </value>
- </nameValuePair>
- </configurationParameterSettings>
- <typeSystemDescription>
- <typeDescription>
- <name>org.apache.uima.TokenAnnotation</name>
- <description>Single token annotation</description>
- <supertypeName>uima.tcas.Annotation</supertypeName>
- <features>
- <featureDescription>
- <name>posTag</name>
- <description>
- contains part-of-speech of a corresponding
- token
- </description>
- <rangeTypeName>uima.cas.String</rangeTypeName>
- </featureDescription>
- </features>
- </typeDescription>
- <typeDescription>
- <name>org.apache.uima.SentenceAnnotation</name>
- <description>sentence annotation</description>
- <supertypeName>uima.tcas.Annotation</supertypeName>
- <features>
-
- </features>
- </typeDescription>
- </typeSystemDescription>
- <typePriorities />
- <fsIndexCollection />
- <operationalProperties>
- <modifiesCas>true</modifiesCas>
- <multipleDeploymentAllowed>true</multipleDeploymentAllowed>
- <outputsNewCASes>false</outputsNewCASes>
- </operationalProperties>
- </analysisEngineMetaData>
- <resourceManagerConfiguration />
-</analysisEngineDescription>
+<?xml version="1.0" encoding="UTF-8"?>
+<analysisEngineDescription
+ xmlns="http://uima.apache.org/resourceSpecifier">
+ <frameworkImplementation>
+ org.apache.uima.java
+ </frameworkImplementation>
+ <primitive>true</primitive>
+ <annotatorImplementationName>
+ org.apache.uima.examples.tagger.HMMTagger
+ </annotatorImplementationName>
+ <analysisEngineMetaData>
+ <name>Hidden Markov Model - Part of Speech Tagger</name>
+ <description>
+ A configuration of the HmmTaggerAnnotator that looks for
+ parts of speech of identified tokens within existing
+ Sentence and Token annotations. See also
+ WhitespaceTokenizer.xml.
+ </description>
+ <version>1.0</version>
+ <vendor>The Apache Software Foundation</vendor>
+ <configurationParameters>
+ <configurationParameter>
+ <name>PARAM_FILE</name>
+ <type>String</type>
+ <multiValued>false</multiValued>
+ <mandatory>true</mandatory>
+ </configurationParameter>
+ </configurationParameters>
+ <configurationParameterSettings>
+ <nameValuePair>
+ <name>PARAM_FILE</name>
+ <value>
+ <string>tagger.properties</string>
+ </value>
+ </nameValuePair>
+ </configurationParameterSettings>
+ <typeSystemDescription>
+ <types>
+ <typeDescription>
+ <name>org.apache.uima.TokenAnnotation</name>
+ <description>Single token annotation</description>
+ <supertypeName>uima.tcas.Annotation</supertypeName>
+ <features>
+ <featureDescription>
+ <name>posTag</name>
+ <description>
+ contains part-of-speech of a
+ corresponding token
+ </description>
+ <rangeTypeName>
+ uima.cas.String
+ </rangeTypeName>
+ </featureDescription>
+ </features>
+ </typeDescription>
+ <typeDescription>
+ <name>org.apache.uima.SentenceAnnotation</name>
+ <description>sentence annotation</description>
+ <supertypeName>uima.tcas.Annotation</supertypeName>
+ </typeDescription>
+ </types>
+ </typeSystemDescription>
+ <typePriorities />
+ <fsIndexCollection />
+ <capabilities>
+ <capability>
+ <inputs>
+ <type>org.apache.uima.TokenAnnotation</type>
+ <type allAnnotatorFeatures="true">
+ org.apache.uima.SentenceAnnotation
+ </type>
+ <feature>
+ org.apache.uima.TokenAnnotation:end
+ </feature>
+ <feature>
+ org.apache.uima.TokenAnnotation:begin
+ </feature>
+ </inputs>
+ <outputs>
+ <type>org.apache.uima.TokenAnnotation</type>
+ <feature>
+ org.apache.uima.TokenAnnotation:posTag
+ </feature>
+ <feature>
+ org.apache.uima.TokenAnnotation:end
+ </feature>
+ <feature>
+ org.apache.uima.TokenAnnotation:begin
+ </feature>
+ </outputs>
+ <languagesSupported />
+ </capability>
+ </capabilities>
+ <operationalProperties>
+ <modifiesCas>true</modifiesCas>
+ <multipleDeploymentAllowed>true</multipleDeploymentAllowed>
+ <outputsNewCASes>false</outputsNewCASes>
+ </operationalProperties>
+ </analysisEngineMetaData>
+ <resourceManagerConfiguration />
+</analysisEngineDescription>
Modified: incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml (original)
+++ incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml Fri Feb 8 09:07:58 2008
@@ -1,66 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
-<analysisEngineDescription xmlns="http://uima.apache.org/resourceSpecifier">
- <frameworkImplementation>org.apache.uima.java</frameworkImplementation>
- <primitive>false</primitive>
- <delegateAnalysisEngineSpecifiers>
- <delegateAnalysisEngine key="SimpleTokenAndSentenceAnnotator">
- <import name="WhitespaceTokenizer"/>
- </delegateAnalysisEngine>
- <delegateAnalysisEngine key="HmmTagger">
- <import name="HmmTagger"/>
- </delegateAnalysisEngine>
- </delegateAnalysisEngineSpecifiers>
- <analysisEngineMetaData>
- <name>HmmTaggerTAE</name>
- <description/>
- <version/>
- <vendor/>
- <configurationParameters searchStrategy="language_fallback">
- <configurationParameter>
- <name>PARAM_FILE</name>
- <type>String</type>
- <multiValued>false</multiValued>
- <mandatory>true</mandatory>
- <overrides>
- <parameter>HmmTagger/PARAM_FILE</parameter>
- </overrides>
- </configurationParameter>
- </configurationParameters>
- <configurationParameterSettings>
- <nameValuePair>
- <name>PARAM_FILE</name>
- <value>
- <string>tagger.properties</string>
- </value>
- </nameValuePair>
- </configurationParameterSettings>
- <flowConstraints>
- <fixedFlow>
- <node>SimpleTokenAndSentenceAnnotator</node>
- <node>HmmTagger</node>
- </fixedFlow>
- </flowConstraints>
- <typePriorities/>
- <fsIndexCollection/>
- <capabilities>
- <capability>
- <inputs>
- <type allAnnotatorFeatures="true">org.apache.uima.TokenAnnotation</type>
- <type allAnnotatorFeatures="true">org.apache.uima.SentenceAnnotation</type>
- </inputs>
- <outputs>
- <type allAnnotatorFeatures="true">org.apache.uima.TokenAnnotation</type>
- </outputs>
- <languagesSupported>
- <language>en</language>
- </languagesSupported>
- </capability>
- </capabilities>
- <operationalProperties>
- <modifiesCas>true</modifiesCas>
- <multipleDeploymentAllowed>true</multipleDeploymentAllowed>
- <outputsNewCASes>false</outputsNewCASes>
- </operationalProperties>
- </analysisEngineMetaData>
- <resourceManagerConfiguration/>
+<analysisEngineDescription
+ xmlns="http://uima.apache.org/resourceSpecifier">
+ <frameworkImplementation>
+ org.apache.uima.java
+ </frameworkImplementation>
+ <primitive>false</primitive>
+ <delegateAnalysisEngineSpecifiers>
+ <delegateAnalysisEngine key="SimpleTokenAndSentenceAnnotator">
+ <import name="WhitespaceTokenizer" />
+ </delegateAnalysisEngine>
+ <delegateAnalysisEngine key="HmmTagger">
+ <import name="HmmTagger" />
+ </delegateAnalysisEngine>
+ </delegateAnalysisEngineSpecifiers>
+ <analysisEngineMetaData>
+ <name>HmmTaggerTAE</name>
+ <description />
+ <version />
+ <vendor />
+ <configurationParameters searchStrategy="language_fallback" />
+ <configurationParameterSettings />
+ <flowConstraints>
+ <fixedFlow>
+ <node>SimpleTokenAndSentenceAnnotator</node>
+ <node>HmmTagger</node>
+ </fixedFlow>
+ </flowConstraints>
+ <typePriorities />
+ <fsIndexCollection />
+ <capabilities />
+ <operationalProperties>
+ <modifiesCas>true</modifiesCas>
+ <multipleDeploymentAllowed>true</multipleDeploymentAllowed>
+ <outputsNewCASes>false</outputsNewCASes>
+ </operationalProperties>
+ </analysisEngineMetaData>
+ <resourceManagerConfiguration />
</analysisEngineDescription>
Modified: incubator/uima/sandbox/trunk/Tagger/docbook/hmmTaggerUsersGuide/hmmTaggerUsersGuide.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/docbook/hmmTaggerUsersGuide/hmmTaggerUsersGuide.xml?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/docbook/hmmTaggerUsersGuide/hmmTaggerUsersGuide.xml (original)
+++ incubator/uima/sandbox/trunk/Tagger/docbook/hmmTaggerUsersGuide/hmmTaggerUsersGuide.xml Fri Feb 8 09:07:58 2008
@@ -1,50 +1,783 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
<!ENTITY imgroot "./images/" >
<!ENTITY % xinclude SYSTEM "../../../uima-docbook-tool/xinclude.mod">
%xinclude;
]>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<book lang="en">
-
- <title>
- Tagger Annotator Documentation
- </title>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<book lang="en">
+
+ <title>Tagger Annotator Documentation</title>
+ <subtitle>.. work in progress..</subtitle>
+
+ <xi:include xmlns:xi="http://www.w3.org/2001/XInclude"
+ href="../../../SandboxDocs/src/docbook/book_info.xml" />
+
+ <preface id="sandbox.tagger.introduction">
+ <title>Introduction</title>
+ <para>
+ Tagger Annotator is an Apache UIMA statistical analysis
+ engine that annotates tokens with corresponding grammatical
+ types (parts of speech, or just POS). The tagger is a
+ standard hidden Markov model (HMM) tagger.
+ </para>
+ </preface>
+
+ <chapter id="sandbox.tagger.prerequisites">
+ <title>Prerequisites</title>
+ <para>
+ The UIMA HMM Tagger annotator assumes that sentences and
+ tokens have already been annotated in the CAS with Sentence
+ and Token annotations respectively (see e.g.
+ <code>Whitespace Tokenizer Annotator</code>
+ ).
+
+ Further, the tagger requires a parameter file which
+ specifies a number of necessary parameters for tagging
+ procedure (see
+ <xref
+ linkend="sandbox.tagger.annotatorDescriptor.configParam" />
+ ).
+
+ Two trained models for English and German are included in
+ the package (in the
+ <code>resources</code>
+ folder). Other models can be trained outside of the UIMA
+ framework (see
+ <xref linkend="sandbox.tagger.training" />
+ ).
+ </para>
+ </chapter>
+
+ <chapter id="sandbox.tagger.processingOverview">
+ <title>Processing Overview</title>
+ <para>
+ The algorithm iterates over sentences and tokens in turn to
+ accumulate a list of words. These are then sent to a
+ processing engine of HMM tagger. For each
+ <code>Token</code>
+ , the
+ <code>posTag</code>
+ field is updated with the corresponding part of speech (e.g.
+ <code>posTag = "NN"</code>
+ where
+ <code>NN</code>
+ stands for
+ <emphasis>common noun</emphasis>
+ ).
+ </para>
+ </chapter>
+
+ <chapter id="sandbox.tagger.annotatorDescriptor">
+ <title>Annotator Descriptor</title>
+ <para>
+ Two descriptors are employed to configure tagger's
+ functionality:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <code>HmmTagger.xml</code>
+ - is a primitive analysis engine descriptor,
+ which defines the tagger basic functionality and
+ can be combined in an aggregate analysis engine
+ with an arbitrary tokenizer. This descriptor
+ cannot be used on itself as the tagger alone
+ does not perfom tokenization.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <code>HmmTaggerTAE.xml</code>
+ - is an aggregate analysis engine whose only
+ function is to combine UIMA
+ <code>Whitespace Tokenizer Annotator</code>
+ with
+ <code>HMM Tagger Annotator</code>
+ and is thereby a "ready to use" tagging
+ descriptor.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+ <section id="sandbox.tagger.annotatorDescriptor.configParam">
+ <title>Configuration Parameters</title>
+ <para>
+ The HMM tagger annotator requires the following
+ configuration parameter:
+ </para>
+ <para>
+ <itemizedlist>
+ <listitem>
+ <para>
+ <code>PARAM_FILE</code>
+ - This parameter is a String defining the
+ name of a tagger's parameter file (default
+ is
+ <code>tagger.properties</code>
+ ), which is in Java properties file format.
+ The parameter file must be specified using a
+ relative path that is available in the UIMA
+ datapath or in the classpath.
+
+ <programlisting><emphasis><![CDATA[<configurationParameterSettings>
+ <nameValuePair>
+ <name>PARAM_FILE</name>
+ <value>
+ <string>tagger.properties</string>
+ </value>
+ </nameValuePair>
+</configurationParameterSettings>]]></emphasis></programlisting>
+
+ </para>
+ </listitem>
+ </itemizedlist>
+ An example of a tagger parameter file is shown in the listing below:
+
+ <programlisting><emphasis><![CDATA[## This is the default tagger.properties file
+
+## uncomment for English
+#MODEL = resources/english/BrownModel.dat
+
+## uncomment for German
+MODEL = resources/german/TuebaModel.dat
+
+
+## N =2 or N= 3 are supported, default N=3
+
+N = 3
+
+## if mapping of tags is desired
+#DO_MAPPING = true
+
+## Basic mapping for the Brown corpus (nltk distribution) tagset: to get 93 tags out of ca. 460
+#MAPPING = org.apache.uima.examples.tagger.TagMapping
- <xi:include xmlns:xi="http://www.w3.org/2001/XInclude"
- href="../../../SandboxDocs/src/docbook/book_info.xml" />
- <preface>
- <title>Introduction</title>
+## Basic mapping for STTS tagset: from 54 tags onto the basic ca. 15 classes plus punctuation
+#MAPPING = org.apache.uima.examples.tagger.GrobMapping]]></emphasis>
+ </programlisting>
+
+ Thus, parameter file defines the following values:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <code>MODEL</code>
+ - defines the relative path to the file containing the trained model
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <code>N</code>
+ - sets up whether we want to use a bi- (N=2) or a trigram (N=3) model for tagging (default is trigram)
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <code>MAPPING</code>
+ - this parameter is useful in two scenarios: either if mapping of tags is desired after tagging
+ is performed with a given model, or if one is willing to train and test a model (s. <xref linkend="sandbox.tagger.training"/>)
+ for a modified tagset than the one used by a manually annotated corpus.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+ </section>
+ <section id="sandbox.tagger.annotatorDescriptor.capabilities">
+ <title>Capabilities</title>
+ <para>
+ As the tagger inherits tokenization indexes from the CAS,
+ <code>uima.SentenceAnnotation</code> and <code>uima.TokenAnnotation</code> with their
+ <code>begin</code> and <code>end</code> features respectively have to be defined as
+ input capabilities in the HMM Tagger annotator descriptor. <code>Token</code> receives
+ also an additional <code>posTag</code> feature as an output capability.
+ </para>
+ <para>
+ <programlisting><emphasis><![CDATA[<capabilities>
+ <capability>
+ <inputs>
+ <type>org.apache.uima.TokenAnnotation</type>
+ <type allAnnotatorFeatures="true">org.apache.uima.SentenceAnnotation</type>
+ <feature>org.apache.uima.TokenAnnotation:end</feature>
+ <feature>org.apache.uima.TokenAnnotation:begin</feature>
+ </inputs>
+ <outputs>
+ <type>org.apache.uima.TokenAnnotation</type>
+ <feature>org.apache.uima.TokenAnnotation:posTag</feature>
+ <feature>org.apache.uima.TokenAnnotation:end</feature>
+ <feature>org.apache.uima.TokenAnnotation:begin</feature>
+ </outputs>
+ </capability>
+ </capabilities>]]></emphasis></programlisting>
+ </para>
+ </section>
+ </chapter>
+
+ <chapter id="sandbox.tagger.unittest">
+ <title>Functionality Test</title>
<para>
- Introduction content
+ The <code>TaggerTest</code> is a JUnit test file (available in the folder <code>unittests</code>),
+ which provides an opportunity to test provided models for English and German,
+ as well as the basic functionality of the tagger. In order to check whether
+ the tagger's configuration is correct, just run this file as JUnit Test and you should get the following output:
+
+ <programlisting><![CDATA[Tesing German Model...
+The used model is:resources/german/TuebaModel.dat
+61646 distinct words in the model
+Number of part-of-speech tags used: 54
+These are: [$(, $,, $., ADJA, ADJD, ADV, APPO, APPR, APPRART, APZR, ART, CARD, ... ]
+Testing German trigram tagger..
+[Jerry, liebt, Wansley, .]
+expected: [NE, VVFIN, NE, $.]
+tagger output: [NE, VVFIN, NE, $.]
+Very Good!
+==========================================================
+Tesing English Model...
+The used model is:resources/english/BrownModel.dat
+56012 distinct words in the model
+Number of part-of-speech tags used: 473
+These are: [', '', (, ), *, ,, --, ., :, ``, abl, abn, abx, ap, ap$, at, be, bed, ...]
+Testing English trigram tagger...
+[Jerry, loves, Wansley, .]
+expected: [np, vbz, np, .]
+tagger output: [np, vbz, np, .]
+Very Good!]]></programlisting>
+
</para>
- </preface>
+ </chapter>
+
+ <chapter id="sandbox.tagger.tagger">
+ <title>Overview of the Tagger Package</title>
+ <para>
+ The package <code>org.apache.uima.examples.tagger</code> contains:
+ <itemizedlist>
+ <listitem>
+ <para>
+ two interfaces:
+ <orderedlist>
+ <listitem>
+ <para>
+ <code>MappingInterface</code>
+ - defines mapping for a tagset. For example, one may wish to map a more detailed tagset
+ to a less distinctive one (i.e. tell a program to tag all verbs as just <code>VERB</code>
+ instead of differentiating between <code>verb infinitive</code>, <code>verb imperative</code>, etc.
+
+ Two sample implementations for <code>MappingInterface</code> are included,
+ namely <code>TagMapping</code> (mapping reducing Brown corpus tagset from more than 400 tags to ca. 93) and
+ <code>GrobMapping</code>(mapping German STTS tagset from 54 tags to basic
+ 11 categories plus special symbols and punctuation)
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <code>Tagger</code>
+ - general tagger interface in case one would want to integrate further tagger types.
+ </para>
+ </listitem>
+ </orderedlist>
+
+ </para>
+ </listitem>
+ <listitem>
+ <para> two basic classes:
+ <orderedlist>
+ <listitem>
+ <para>
+ <code>HMMTagger</code>
+ - hidden Markov model tagger for UIMA, that is using Viterbi algorithm to compute the most
+ probable part-of-speech sequence for a given list of tokens.
+ </para>
+ </listitem>
+
+ <listitem>
+ <para>
+ <code>Viterbi</code>
+ - implementation of the Viterbi Algorithm. This class makes up the core of the tagger.
+ </para>
+ </listitem>
- <chapter id="sandbox.tagger.processingOverview">
- <title>Processing Overview</title>
+ </orderedlist>
+
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+ </chapter>
+
+
+ <chapter id="sandbox.tagger.training">
+ <title>Training Own Models</title>
<para>
- Processing overview.
+ Though we decide not to include training directly into UIMA framework, one can easily
+ train other models for different pre-annotated corpora outside of the UIMA using <code>ModelGeneration</code> class,
+ available in the subpackage <code>org.apache.uima.examples.tagger.trainAndTest</code>.
+ This subpackage includes some further files needed for training of own models:
+
+
+ <itemizedlist>
+
+
+ <listitem>
+ <para>
+ <code>ModelGeneration</code>
+ - trains an N-gram model for the tagger, iterating over the files from some predefined
+ training directory. Writes the resulting model to a binary file. At the moment,
+ only bi-and trigram models are supported. Further N-grams can be easily integrated.
+
+ </para>
+ </listitem>
+ <listitem>
+
+ <para>
+ Interface <code>CorpusReader</code>
+ - should be used to implement corpus readers for own corpora; the objective
+ of the reader is to take charge of the preprocessing and transform tokenized units
+ (usually <emphasis>words</emphasis>) into a List of <code>Token</code> objects.
+ Two sample implementations of <code>CorpusReader</code> are included:
+
+ <orderedlist>
+
+ <listitem>
+ <para>
+ <code>BrownReader</code>
+ - for the Brown corpus from the nltk distribution (nltk.sourceforge.net)
+ </para>
+ </listitem>
+
+ <listitem>
+ <para>
+ <code>TT_FormatReader</code>
+ - for the corpora in TreeTagger format, i.e. one word per line
+ with tags separated from the words by tab.
+ </para>
+ </listitem>
+
+
+ </orderedlist>
+
+ </para>
+ </listitem>
+
+
+ <listitem>
+ <para>
+ <code>SuffixTree</code>
+ - Java implementation of the Ukkonen's suffix tree, used to gain suffix statistics for unknown words.
+ </para>
+ </listitem>
+
+ <listitem>
+ <para>
+ <code>Token</code>
+ - is used as a kind of an interface between
+ the text and the annotator. Every tokenized unit of text should be defined as a <code>Token</code>
+ .
+ </para>
+ </listitem>
+
+ <listitem>
+ <para>
+ <code>TaggerEvaluation</code>
+ - can be used to evaluate the tagger and/or new models on a small manually annotated corpus. </para>
+ </listitem>
+
+ </itemizedlist>
+ </para>
+ <para>
+ To train a new model, one should specify ....
+ </para>
+ </chapter>
+
+ <chapter id="sandbox.tagger.evaluation">
+ <title>Evaluation</title>
+ <para>
+ To evaluate performance if a "gold standard" corpus is given, ....
+
</para>
</chapter>
+
+ <appendix id="sandbox.tagger.theory">
+ <title>Theory Behind</title>
+
+ <para>
+ This chapter is just a sketch of the statistical model
+ undelying the tagger.
+
+ Hidden Markov Models (HMMs) are the mainstay of the
+ applications employing statistical modeling in any form,
+ like speech recognition and production systems, signal
+ processing, part of speech tagging.
+
+ A Hidden Markov Model is a probabilistic function of a
+ Markov process. A Markov process is a process that fulfills
+ Markov assumptions.
+
+
+ Markov assumptions are:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <code>limited horizon</code>
+ - Markov processes are states without memory,
+ except for condition of the current state.
+ Though we usually consider sequences of
+ variables that are not independent of each
+ other, it often suffices to know the value of
+ the current situation without going deep into
+ the past happenings. As [
+ <biblioref linkend="schuetze" />
+ ] put it, we do not really need to know, how
+ many books were in the library last week or last
+ year in order to predict how many books there
+ will be tomorrow. It is often enough to know the
+ current situation. Thereby, future states in the
+ Markov process are independent of the past, they
+ only depend on the present. Let
+ <inlineequation>
+ <mathphrase>
+ X = (X
+ <subscript>1</subscript>
+ , ..., X
+ <subscript>T</subscript>
+ )
+ </mathphrase>
+ </inlineequation>
+ be a sequence of random variables taking the
+ values from the finite state space
+ <inlineequation>
+ <mathphrase>
+ S = (s
+ <subscript>1</subscript>
+ , ..., s
+ <subscript>N</subscript>
+ )
+ </mathphrase>
+ </inlineequation>
+ , then a limited horizon property could be
+ formalized by:
+ <informalequation>
+ <mathphrase>
+ P(X
+ <subscript>t+1</subscript>
+ = s
+ <subscript>k</subscript>
+ |X
+ <subscript>1</subscript>
+ , ..., X
+ <subscript>t</subscript>
+ ) = P(X
+ <subscript>t+1</subscript>
+ = s
+ <subscript>k</subscript>
+ |X
+ <subscript>t</subscript>
+ )
+ </mathphrase>
+ </informalequation>
+
+
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <code>time invariance</code>
+ </para>
+
+ <para>
+ The probabilities do not change over time, i.e.
+ if we know that the probability of observing a
+ rainbow after the rain is equal to 90\%, we know
+ that it should be true for today as well as for
+ tomorrow.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+ <para>
+
+ If
+ <code>X</code>
+ conforms to these two properties, then it is said to be a
+ Markov chain.
+
+ One can describe a Markov chain by a transition matrix:
+ <informalequation>
+ <mathphrase>
+ A = a
+ <subscript>i,j</subscript>
+ = P(X
+ <subscript>t+1</subscript>
+ = s
+ <subscript>j</subscript>
+ |X
+ <subscript>t</subscript>
+ =s
+ <subscript>i</subscript>
+ )
+ </mathphrase>
+ </informalequation>
+
+
+ <informalequation>
+ <mathphrase>
+ - with a
+ <subscript>i,j</subscript>
+ >= 0 (for all
+ <emphasis>i,j</emphasis>
+ ) and the sum of all transition probabilities from
+ state
+ <emphasis>i</emphasis>
+ (a
+ <subscript>i,j</subscript>
+ ) should be equal to 1 (for all
+ <emphasis>i</emphasis>
+ )
+ </mathphrase>
+ </informalequation>
+
+ </para>
+
+ <para>
+
+ Markov models can be used whenever one needs to model the
+ probability of a linear sequence of variables.
+
+ One distinguishes Visible Markov Models (VMM) vs. Hidden
+ Markov Models. The difference is that when we work with
+ "visible" events, we can directly estimate the corresponding
+ probabilities (which is the case if training corpus is
+ available to train own models for HMM taggers).
+
+ Finding a sequence of part of speech tags (i.e. Viterbi part
+ of the tagger) in contrast is a hidden Markov model as the
+ states (tags) are not directly observable.
+ </para>
+
+ <para>
+ <emphasis>The goal of HMM - based tagger</emphasis>
+ is to find part of speech tags ( = hidden states) that
+ generate a sequence of words ( = observable states). Most of
+ the known implementations of POS taggers are viewing text as
+ being produced by a hidden Markov model, so that tagging can
+ be regarded as a Markov process deciding which states the
+ system went through to generate a given text.
+
+ </para>
+ <para>
+ <emphasis>General Form of HMM</emphasis>
+ </para>
+ <para>
+ A HMM is a five-tuple:
+ <inlineequation>
+ <mathphrase>(S, O, &pgr;, A, B)</mathphrase>
+ </inlineequation>
+ <informalexample>
+ <para>where:</para>
+ <para>
+ <itemizedlist>
+ <listitem>
+ <para>
+ <code>S</code>
+ - the set of states (here: parts of
+ speech)
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <code>K</code>
+ - the set of observations (here: words)
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <code>&pgr;</code>
+ - initial state probabilities
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <code>A</code>
+ - state transitions probabilities
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <code>B</code>
+ - symbol emissions probabilities
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ </informalexample>
+ </para>
+
+ <para>
+ Further,
+ <code>
+ X
+ <subscript>t</subscript>
+ </code>
+ (state sequence) and
+ <code>
+ O
+ <subscript>t</subscript>
+ </code>
+ (output sequence) are given.
+
+ Tagging procedure is then the following:
+ <informalexample>
+ <orderedlist>
+ <listitem>
+ <para>
+ <code>t := 1</code>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <code>
+ Start in state s
+ <subscript>i</subscript>
+ with probability &pgr;
+ <subscript>i</subscript>
+ (i.e., X
+ <subscript>1</subscript>
+ = i)
+ </code>
+ </para>
+ </listitem>
+
+ <listitem>
+ <para>
+ <code>forever do:</code>
+ </para>
+
+ <itemizedlist>
+ <listitem>
+ <para>
+ <code>
+ Move from s
+ <subscript>i</subscript>
+ to s
+ <subscript>j</subscript>
+ with probability a
+ <subscript>i,j</subscript>
+ (i.e. X
+ <subscript>t+1</subscript>
+ = j)
+ </code>
+ </para>
+ </listitem>
+
+ <listitem>
+ <para>
+ <code>
+ Emit observation symbol o
+ <subscript>t</subscript>
+ = k with probability b
+ <subscript>i,j,k</subscript>
+ </code>
+ </para>
+ </listitem>
+
+ <listitem>
+ <para>
+ <code>t := T+1</code>
+ </para>
+
+ </listitem>
+ </itemizedlist>
+ </listitem>
+
+ <listitem>
+ <para>
+ <code>end</code>
+ </para>
+ </listitem>
+
+ </orderedlist>
+ </informalexample>
+</para>
+
+ <para>
+ Despite their limitations, HMM-s are one of the most
+ successful techniques in natural language processing and are
+ widely used, especially in sequence tagging applications.
+ The best statistical taggers all perform at about the same
+ level of accuracy.
+ </para>
+ </appendix>
+
+ <!-- ... -->
+ <glossary>
+ <title>Glossary</title>
+
+ <glossdiv>
+ <title>HMM</title>
+
+ <glossentry id="hmm">
+ <glossterm>Hidden Markov Model</glossterm>
+ <acronym>HMM</acronym>
+ <glossdef>
+ <para></para>
+ </glossdef>
+ </glossentry>
+ </glossdiv>
+
+ <glossdiv>
+ <title>POS</title>
+ <glossentry id="pos">
+ <glossterm>Part of Speech</glossterm>
+ <acronym>POS</acronym>
+ <glossdef>
+ <para></para>
+ </glossdef>
+ </glossentry>
+ </glossdiv>
+
+ </glossary>
+
+ <bibliography>
+ <biblioentry xreflabel="ManningSchuetze99" id="schuetze">
+ <authorgroup>
+ <author>
+ <firstname>Christopher</firstname>
+ <surname>Manning</surname>
+ </author>
+ <author>
+ <firstname>Hinrich</firstname>
+ <surname>Schuetze</surname>
+ </author>
+ </authorgroup>
+ <copyright>
+ <year>1999</year>
+ </copyright>
+ <publisher>
+ <publishername>MIT Press</publishername>
+ </publisher>
+ <title>
+ Foundations of Statistical Natural Language Processing
+ </title>
+ </biblioentry>
+ </bibliography>
+
</book>
Modified: incubator/uima/sandbox/trunk/Tagger/metadata/install.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/metadata/install.xml?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/metadata/install.xml (original)
+++ incubator/uima/sandbox/trunk/Tagger/metadata/install.xml Fri Feb 8 09:07:58 2008
@@ -1,22 +1,22 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
-->
<COMPONENT_INSTALLATION_DESCRIPTOR>
@@ -26,8 +26,7 @@
<TOOLKITS>
<JDK_VERSION>1.5</JDK_VERSION>
</TOOLKITS>
- <UIMA_FRAMEWORK>
- </UIMA_FRAMEWORK>
+ <UIMA_FRAMEWORK></UIMA_FRAMEWORK>
<SUBMITTED_COMPONENT>
<ID>Tagger</ID>
<NAME></NAME>
Modified: incubator/uima/sandbox/trunk/Tagger/pom.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/pom.xml?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/pom.xml (original)
+++ incubator/uima/sandbox/trunk/Tagger/pom.xml Fri Feb 8 09:07:58 2008
@@ -1,83 +1,92 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.uima</groupId>
- <artifactId>Tagger</artifactId>
- <packaging>jar</packaging>
- <version>2.3.0-incubating-SNAPSHOT</version>
- <name>Apache UIMA Sandbox -- HMM Tagger</name>
- <url>http://incubator.apache.org/uima</url>
- <dependencies>
- <dependency>
- <groupId>org.apache.uima</groupId>
- <artifactId>uimaj-core</artifactId>
- <version>2.2.1-incubating</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.uima</groupId>
- <artifactId>uimaj-test-util</artifactId>
- <version>2.2.1-incubating</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.uima</groupId>
- <artifactId>uimaj-component-test-util</artifactId>
- <version>2.2.1-incubating</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.uima</groupId>
- <artifactId>WhitespaceTokenizer</artifactId>
- <version>2.3.0-incubating-SNAPSHOT</version>
- <scope>compile</scope>
- </dependency>
-
- </dependencies>
- <build>
- <finalName>uimaj-tagger</finalName>
- <resources>
- <resource>
- <directory>desc</directory>
- </resource>
- </resources>
-
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.5</source>
- <target>1.5</target>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <repositories>
- <repository>
- <id>ApacheIncubatorRepository</id>
- <url>http://people.apache.org/repo/m2-incubating-repository/</url>
- </repository>
- </repositories>
-</project>
-
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.uima</groupId>
+ <artifactId>Tagger</artifactId>
+ <packaging>jar</packaging>
+ <version>2.3.0-incubating-SNAPSHOT</version>
+ <name>Apache UIMA Sandbox -- HMM Tagger</name>
+ <url>http://incubator.apache.org/uima</url>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.uima</groupId>
+ <artifactId>uimaj-core</artifactId>
+ <version>2.2.1-incubating</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uima</groupId>
+ <artifactId>uimaj-test-util</artifactId>
+ <version>2.2.1-incubating</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uima</groupId>
+ <artifactId>uimaj-component-test-util</artifactId>
+ <version>2.2.1-incubating</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uima</groupId>
+ <artifactId>WhitespaceTokenizer</artifactId>
+ <version>2.3.0-incubating-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+
+ </dependencies>
+ <build>
+ <finalName>uimaj-tagger</finalName>
+ <resources>
+ <resource>
+ <directory>desc</directory>
+ </resource>
+ </resources>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.5</source>
+ <target>1.5</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <argLine>-Xmx500M</argLine>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <repositories>
+ <repository>
+ <id>ApacheIncubatorRepository</id>
+ <url>
+ http://people.apache.org/repo/m2-incubating-repository/
+ </url>
+ </repository>
+ </repositories>
+</project>
+
Modified: incubator/uima/sandbox/trunk/Tagger/resources/english/BrownModel.dat
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/resources/english/BrownModel.dat?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
Binary files - no diff available.
Modified: incubator/uima/sandbox/trunk/Tagger/resources/german/TuebaModel.dat
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/resources/german/TuebaModel.dat?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
Binary files - no diff available.
Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation.java?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation.java Fri Feb 8 09:07:58 2008
@@ -1,62 +1,75 @@
-
/* First created by JCasGen Thu Oct 25 11:28:37 CEST 2007 */
package org.apache.uima;
-import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.JCas;
import org.apache.uima.jcas.JCasRegistry;
import org.apache.uima.jcas.cas.TOP_Type;
import org.apache.uima.jcas.tcas.Annotation;
-
-/** sentence annotation
- * Updated by JCasGen Thu Oct 25 11:28:37 CEST 2007
- * XML source: C:/code/ApacheUIMA/Tagger/desc/HmmTaggerTAE.xml
- * @generated */
+/**
+ * sentence annotation Updated by JCasGen Thu Oct 25 11:28:37 CEST 2007 XML source:
+ * C:/code/ApacheUIMA/Tagger/desc/HmmTaggerTAE.xml
+ *
+ * @generated
+ */
public class SentenceAnnotation extends Annotation {
- /** @generated
- * @ordered
+ /**
+ * @generated
+ * @ordered
*/
public final static int typeIndexID = JCasRegistry.register(SentenceAnnotation.class);
- /** @generated
- * @ordered
+
+ /**
+ * @generated
+ * @ordered
*/
public final static int type = typeIndexID;
- /** @generated */
- public int getTypeIndexID() {return typeIndexID;}
-
- /** Never called. Disable default constructor
- * @generated */
- protected SentenceAnnotation() {}
-
- /** Internal - constructor used by generator
- * @generated */
+
+ /** @generated */
+ public int getTypeIndexID() {
+ return typeIndexID;
+ }
+
+ /**
+ * Never called. Disable default constructor
+ *
+ * @generated
+ */
+ protected SentenceAnnotation() {
+ }
+
+ /**
+ * Internal - constructor used by generator
+ *
+ * @generated
+ */
public SentenceAnnotation(int addr, TOP_Type type) {
super(addr, type);
readObject();
}
-
+
/** @generated */
public SentenceAnnotation(JCas jcas) {
super(jcas);
- readObject();
- }
+ readObject();
+ }
- /** @generated */
+ /** @generated */
public SentenceAnnotation(JCas jcas, int begin, int end) {
super(jcas);
setBegin(begin);
setEnd(end);
readObject();
- }
+ }
- /** <!-- begin-user-doc -->
- * Write your own initialization here
- * <!-- end-user-doc -->
- @generated modifiable */
- private void readObject() {}
-
-}
+ /**
+ * <!-- begin-user-doc --> Write your own initialization here <!-- end-user-doc -->
+ *
+ * @generated modifiable
+ */
+ private void readObject() {
+ }
-
\ No newline at end of file
+}
Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation_Type.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation_Type.java?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation_Type.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation_Type.java Fri Feb 8 09:07:58 2008
@@ -1,4 +1,3 @@
-
/* First created by JCasGen Thu Oct 25 11:28:37 CEST 2007 */
package org.apache.uima;
@@ -11,45 +10,52 @@
import org.apache.uima.cas.Type;
import org.apache.uima.jcas.tcas.Annotation_Type;
-/** sentence annotation
- * Updated by JCasGen Thu Oct 25 11:28:37 CEST 2007
- * @generated */
+/**
+ * sentence annotation Updated by JCasGen Thu Oct 25 11:28:37 CEST 2007
+ *
+ * @generated
+ */
public class SentenceAnnotation_Type extends Annotation_Type {
/** @generated */
- protected FSGenerator getFSGenerator() {return fsGenerator;}
+ protected FSGenerator getFSGenerator() {
+ return fsGenerator;
+ }
+
/** @generated */
- private final FSGenerator fsGenerator =
- new FSGenerator() {
- public FeatureStructure createFS(int addr, CASImpl cas) {
- if (SentenceAnnotation_Type.this.useExistingInstance) {
- // Return eq fs instance if already created
- FeatureStructure fs = SentenceAnnotation_Type.this.jcas.getJfsFromCaddr(addr);
- if (null == fs) {
- fs = new SentenceAnnotation(addr, SentenceAnnotation_Type.this);
- SentenceAnnotation_Type.this.jcas.putJfsFromCaddr(addr, fs);
- return fs;
- }
- return fs;
- } else return new SentenceAnnotation(addr, SentenceAnnotation_Type.this);
- }
- };
+ private final FSGenerator fsGenerator = new FSGenerator() {
+ public FeatureStructure createFS(int addr, CASImpl cas) {
+ if (SentenceAnnotation_Type.this.useExistingInstance) {
+ // Return eq fs instance if already created
+ FeatureStructure fs = SentenceAnnotation_Type.this.jcas.getJfsFromCaddr(addr);
+ if (null == fs) {
+ fs = new SentenceAnnotation(addr, SentenceAnnotation_Type.this);
+ SentenceAnnotation_Type.this.jcas.putJfsFromCaddr(addr, fs);
+ return fs;
+ }
+ return fs;
+ } else
+ return new SentenceAnnotation(addr, SentenceAnnotation_Type.this);
+ }
+ };
+
/** @generated */
public final static int typeIndexID = SentenceAnnotation.typeIndexID;
- /** @generated
- @modifiable */
- public final static boolean featOkTst = JCasRegistry.getFeatOkTst("org.apache.uima.SentenceAnnotation");
-
-
- /** initialize variables to correspond with Cas Type and Features
- * @generated */
+ /**
+ * @generated
+ * @modifiable
+ */
+ public final static boolean featOkTst = JCasRegistry
+ .getFeatOkTst("org.apache.uima.SentenceAnnotation");
+
+ /**
+ * initialize variables to correspond with Cas Type and Features
+ *
+ * @generated
+ */
public SentenceAnnotation_Type(JCas jcas, Type casType) {
super(jcas, casType);
- casImpl.getFSClassRegistry().addGeneratorForType((TypeImpl)this.casType, getFSGenerator());
+ casImpl.getFSClassRegistry().addGeneratorForType((TypeImpl) this.casType, getFSGenerator());
}
}
-
-
-
-
\ No newline at end of file
Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation.java?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation.java Fri Feb 8 09:07:58 2008
@@ -1,98 +1,130 @@
-
/* First created by JCasGen Thu Oct 25 11:28:37 CEST 2007 */
package org.apache.uima;
-import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.JCas;
import org.apache.uima.jcas.JCasRegistry;
import org.apache.uima.jcas.cas.TOP_Type;
import org.apache.uima.jcas.tcas.Annotation;
-
-/** Single token annotation
- * Updated by JCasGen Thu Oct 25 11:28:37 CEST 2007
- * XML source: C:/code/ApacheUIMA/Tagger/desc/HmmTaggerTAE.xml
- * @generated */
+/**
+ * Single token annotation Updated by JCasGen Thu Oct 25 11:28:37 CEST 2007 XML source:
+ * C:/code/ApacheUIMA/Tagger/desc/HmmTaggerTAE.xml
+ *
+ * @generated
+ */
public class TokenAnnotation extends Annotation {
- /** @generated
- * @ordered
+ /**
+ * @generated
+ * @ordered
*/
public final static int typeIndexID = JCasRegistry.register(TokenAnnotation.class);
- /** @generated
- * @ordered
+
+ /**
+ * @generated
+ * @ordered
*/
public final static int type = typeIndexID;
- /** @generated */
- public int getTypeIndexID() {return typeIndexID;}
-
- /** Never called. Disable default constructor
- * @generated */
- protected TokenAnnotation() {}
-
- /** Internal - constructor used by generator
- * @generated */
+
+ /** @generated */
+ public int getTypeIndexID() {
+ return typeIndexID;
+ }
+
+ /**
+ * Never called. Disable default constructor
+ *
+ * @generated
+ */
+ protected TokenAnnotation() {
+ }
+
+ /**
+ * Internal - constructor used by generator
+ *
+ * @generated
+ */
public TokenAnnotation(int addr, TOP_Type type) {
super(addr, type);
readObject();
}
-
+
/** @generated */
public TokenAnnotation(JCas jcas) {
super(jcas);
- readObject();
- }
+ readObject();
+ }
- /** @generated */
+ /** @generated */
public TokenAnnotation(JCas jcas, int begin, int end) {
super(jcas);
setBegin(begin);
setEnd(end);
readObject();
- }
+ }
+
+ /**
+ * <!-- begin-user-doc --> Write your own initialization here <!-- end-user-doc -->
+ *
+ * @generated modifiable
+ */
+ private void readObject() {
+ }
- /** <!-- begin-user-doc -->
- * Write your own initialization here
- * <!-- end-user-doc -->
- @generated modifiable */
- private void readObject() {}
-
-
-
- //*--------------*
- //* Feature: tokenType
+ // *--------------*
+ // * Feature: tokenType
- /** getter for tokenType - gets token type
- * @generated */
+ /**
+ * getter for tokenType - gets token type
+ *
+ * @generated
+ */
public String getTokenType() {
- if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_tokenType == null)
+ if (TokenAnnotation_Type.featOkTst
+ && ((TokenAnnotation_Type) jcasType).casFeat_tokenType == null)
jcasType.jcas.throwFeatMissing("tokenType", "org.apache.uima.TokenAnnotation");
- return jcasType.ll_cas.ll_getStringValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_tokenType);}
-
- /** setter for tokenType - sets token type
- * @generated */
+ return jcasType.ll_cas.ll_getStringValue(addr,
+ ((TokenAnnotation_Type) jcasType).casFeatCode_tokenType);
+ }
+
+ /**
+ * setter for tokenType - sets token type
+ *
+ * @generated
+ */
public void setTokenType(String v) {
- if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_tokenType == null)
+ if (TokenAnnotation_Type.featOkTst
+ && ((TokenAnnotation_Type) jcasType).casFeat_tokenType == null)
jcasType.jcas.throwFeatMissing("tokenType", "org.apache.uima.TokenAnnotation");
- jcasType.ll_cas.ll_setStringValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_tokenType, v);}
-
-
- //*--------------*
- //* Feature: posTag
+ jcasType.ll_cas.ll_setStringValue(addr,
+ ((TokenAnnotation_Type) jcasType).casFeatCode_tokenType, v);
+ }
+
+ // *--------------*
+ // * Feature: posTag
- /** getter for posTag - gets contains part-of-speech of a corresponding token
- * @generated */
+ /**
+ * getter for posTag - gets contains part-of-speech of a corresponding token
+ *
+ * @generated
+ */
public String getPosTag() {
- if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_posTag == null)
+ if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type) jcasType).casFeat_posTag == null)
jcasType.jcas.throwFeatMissing("posTag", "org.apache.uima.TokenAnnotation");
- return jcasType.ll_cas.ll_getStringValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_posTag);}
-
- /** setter for posTag - sets contains part-of-speech of a corresponding token
- * @generated */
+ return jcasType.ll_cas.ll_getStringValue(addr,
+ ((TokenAnnotation_Type) jcasType).casFeatCode_posTag);
+ }
+
+ /**
+ * setter for posTag - sets contains part-of-speech of a corresponding token
+ *
+ * @generated
+ */
public void setPosTag(String v) {
- if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_posTag == null)
+ if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type) jcasType).casFeat_posTag == null)
jcasType.jcas.throwFeatMissing("posTag", "org.apache.uima.TokenAnnotation");
- jcasType.ll_cas.ll_setStringValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_posTag, v);}
+ jcasType.ll_cas
+ .ll_setStringValue(addr, ((TokenAnnotation_Type) jcasType).casFeatCode_posTag, v);
}
-
-
\ No newline at end of file
+}
Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation_Type.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation_Type.java?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation_Type.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation_Type.java Fri Feb 8 09:07:58 2008
@@ -1,4 +1,3 @@
-
/* First created by JCasGen Thu Oct 25 11:28:37 CEST 2007 */
package org.apache.uima;
@@ -13,89 +12,101 @@
import org.apache.uima.cas.Feature;
import org.apache.uima.jcas.tcas.Annotation_Type;
-/** Single token annotation
- * Updated by JCasGen Thu Oct 25 11:28:37 CEST 2007
- * @generated */
+/**
+ * Single token annotation Updated by JCasGen Thu Oct 25 11:28:37 CEST 2007
+ *
+ * @generated
+ */
public class TokenAnnotation_Type extends Annotation_Type {
/** @generated */
- protected FSGenerator getFSGenerator() {return fsGenerator;}
+ protected FSGenerator getFSGenerator() {
+ return fsGenerator;
+ }
+
/** @generated */
- private final FSGenerator fsGenerator =
- new FSGenerator() {
- public FeatureStructure createFS(int addr, CASImpl cas) {
- if (TokenAnnotation_Type.this.useExistingInstance) {
- // Return eq fs instance if already created
- FeatureStructure fs = TokenAnnotation_Type.this.jcas.getJfsFromCaddr(addr);
- if (null == fs) {
- fs = new TokenAnnotation(addr, TokenAnnotation_Type.this);
- TokenAnnotation_Type.this.jcas.putJfsFromCaddr(addr, fs);
- return fs;
- }
- return fs;
- } else return new TokenAnnotation(addr, TokenAnnotation_Type.this);
- }
- };
+ private final FSGenerator fsGenerator = new FSGenerator() {
+ public FeatureStructure createFS(int addr, CASImpl cas) {
+ if (TokenAnnotation_Type.this.useExistingInstance) {
+ // Return eq fs instance if already created
+ FeatureStructure fs = TokenAnnotation_Type.this.jcas.getJfsFromCaddr(addr);
+ if (null == fs) {
+ fs = new TokenAnnotation(addr, TokenAnnotation_Type.this);
+ TokenAnnotation_Type.this.jcas.putJfsFromCaddr(addr, fs);
+ return fs;
+ }
+ return fs;
+ } else
+ return new TokenAnnotation(addr, TokenAnnotation_Type.this);
+ }
+ };
+
/** @generated */
public final static int typeIndexID = TokenAnnotation.typeIndexID;
- /** @generated
- @modifiable */
- public final static boolean featOkTst = JCasRegistry.getFeatOkTst("org.apache.uima.TokenAnnotation");
-
+
+ /**
+ * @generated
+ * @modifiable
+ */
+ public final static boolean featOkTst = JCasRegistry
+ .getFeatOkTst("org.apache.uima.TokenAnnotation");
+
/** @generated */
final Feature casFeat_tokenType;
+
+ /** @generated */
+ final int casFeatCode_tokenType;
+
/** @generated */
- final int casFeatCode_tokenType;
- /** @generated */
public String getTokenType(int addr) {
- if (featOkTst && casFeat_tokenType == null)
+ if (featOkTst && casFeat_tokenType == null)
jcas.throwFeatMissing("tokenType", "org.apache.uima.TokenAnnotation");
return ll_cas.ll_getStringValue(addr, casFeatCode_tokenType);
}
- /** @generated */
+
+ /** @generated */
public void setTokenType(int addr, String v) {
- if (featOkTst && casFeat_tokenType == null)
+ if (featOkTst && casFeat_tokenType == null)
jcas.throwFeatMissing("tokenType", "org.apache.uima.TokenAnnotation");
- ll_cas.ll_setStringValue(addr, casFeatCode_tokenType, v);}
-
-
-
+ ll_cas.ll_setStringValue(addr, casFeatCode_tokenType, v);
+ }
+
/** @generated */
final Feature casFeat_posTag;
+
+ /** @generated */
+ final int casFeatCode_posTag;
+
/** @generated */
- final int casFeatCode_posTag;
- /** @generated */
public String getPosTag(int addr) {
- if (featOkTst && casFeat_posTag == null)
+ if (featOkTst && casFeat_posTag == null)
jcas.throwFeatMissing("posTag", "org.apache.uima.TokenAnnotation");
return ll_cas.ll_getStringValue(addr, casFeatCode_posTag);
}
- /** @generated */
+
+ /** @generated */
public void setPosTag(int addr, String v) {
- if (featOkTst && casFeat_posTag == null)
+ if (featOkTst && casFeat_posTag == null)
jcas.throwFeatMissing("posTag", "org.apache.uima.TokenAnnotation");
- ll_cas.ll_setStringValue(addr, casFeatCode_posTag, v);}
-
-
-
-
+ ll_cas.ll_setStringValue(addr, casFeatCode_posTag, v);
+ }
- /** initialize variables to correspond with Cas Type and Features
- * @generated */
+ /**
+ * initialize variables to correspond with Cas Type and Features
+ *
+ * @generated
+ */
public TokenAnnotation_Type(JCas jcas, Type casType) {
super(jcas, casType);
- casImpl.getFSClassRegistry().addGeneratorForType((TypeImpl)this.casType, getFSGenerator());
+ casImpl.getFSClassRegistry().addGeneratorForType((TypeImpl) this.casType, getFSGenerator());
-
- casFeat_tokenType = jcas.getRequiredFeatureDE(casType, "tokenType", "uima.cas.String", featOkTst);
- casFeatCode_tokenType = (null == casFeat_tokenType) ? JCas.INVALID_FEATURE_CODE : ((FeatureImpl)casFeat_tokenType).getCode();
+ casFeat_tokenType = jcas.getRequiredFeatureDE(casType, "tokenType", "uima.cas.String",
+ featOkTst);
+ casFeatCode_tokenType = (null == casFeat_tokenType) ? JCas.INVALID_FEATURE_CODE
+ : ((FeatureImpl) casFeat_tokenType).getCode();
-
casFeat_posTag = jcas.getRequiredFeatureDE(casType, "posTag", "uima.cas.String", featOkTst);
- casFeatCode_posTag = (null == casFeat_posTag) ? JCas.INVALID_FEATURE_CODE : ((FeatureImpl)casFeat_posTag).getCode();
+ casFeatCode_posTag = (null == casFeat_posTag) ? JCas.INVALID_FEATURE_CODE
+ : ((FeatureImpl) casFeat_posTag).getCode();
}
}
-
-
-
-
\ No newline at end of file
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/GrobMapping.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/GrobMapping.java?rev=619943&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/GrobMapping.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/GrobMapping.java Fri Feb 8 09:07:58 2008
@@ -0,0 +1,146 @@
+/*
+ *Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.uima.examples.tagger;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.uima.examples.tagger.trainAndTest.Token;
+
+/**
+ *
+ */
+public class GrobMapping implements MappingInterface{
+
+
+ /**
+ * Defines mapping for List<{@link Token}>
+ * E.g. if we need to map tags, given a list of {@code Tokens}, we need to map the
+ * {@code pos} field of every {@code Token} to a different {@code pos}.
+ */
+ public List map_tags(List tokens){
+
+
+ // for (int i=0; i<sentences.size(); i++){ // iterate over sentences
+
+ List<Token> tokens2 = new ArrayList<Token>(tokens.size());
+
+ for (int x=0; x<tokens.size(); x++){ // iterate over tokens of the sentence with their corresponding POS
+ Token current_token = (Token)tokens.get(x);
+
+ if(current_token.pos.startsWith("N")){
+ current_token.pos="Noun";
+ }
+ if(current_token.pos.startsWith("V")){
+ current_token.pos="Verb";
+ }
+ if(current_token.pos.startsWith("ADJ")){
+ current_token.pos="Adjective";
+ }
+ if(current_token.pos.startsWith("P")){
+ current_token.pos="Pronoun";
+ }
+ if(current_token.pos.startsWith("KO")){
+ current_token.pos="Conjunction";
+ }
+ if(current_token.pos.startsWith("AP")){
+ current_token.pos="Preposition";
+ }
+ if(current_token.pos.startsWith("PTK")){
+ current_token.pos="PTK";
+ }
+ if(current_token.pos.startsWith("ADV")){
+ current_token.pos="Adverb";
+ }
+
+ if(current_token.pos.startsWith("ART")){
+ current_token.pos="Article";
+ }
+
+ if(current_token.pos.startsWith("ITJ")){
+ current_token.pos="Interjection";
+ }
+
+ Token zwischen = new Token(current_token.word, current_token.pos);
+
+ tokens2.add(zwischen);
+ }
+ return tokens2;
+
+ }
+
+ /**
+ * Defines mapping for {@code List<String>}.
+ * E.g. if we need to map pos-tags given simply as {@code Strings} in a {@code List}.
+
+
+ public List<String> map_pos(List<String> pos){
+
+
+ // for (int i=0; i<sentences.size(); i++){ // iterate over sentences
+
+ List<String> pos2 = new ArrayList<String>(pos.size());
+
+ for (int x=0; x<pos.size(); x++){
+ String current_pos = pos.get(x);
+
+ if(current_pos.startsWith("N")){
+ current_pos="Noun";
+ }
+ if(current_pos.startsWith("V")){
+ current_pos="Verb";
+ }
+ if(current_pos.startsWith("ADJ")){
+ current_pos="Adjective";
+ }
+ if(current_pos.startsWith("P")){
+ current_pos="Pronoun";
+ }
+ if(current_pos.startsWith("KO")){
+ current_pos="Conjunction";
+ }
+ if(current_pos.startsWith("AP")){
+ current_pos="Preposition";
+ }
+ if(current_pos.startsWith("PTK")){
+ current_pos="Particle";
+ }
+ if(current_pos.startsWith("ADV")){
+ current_pos="Adverb";
+ }
+
+ if(current_pos.startsWith("ART")){
+ current_pos="Article";
+ }
+
+ if(current_pos.startsWith("ITJ")){
+ current_pos="Interjection";
+ }
+
+ // Token zwischen = new Token(current_token.word, current_token.pos);
+
+ pos2.add(current_pos);
+ }
+ return pos2;
+
+ }
+ */
+}
Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java Fri Feb 8 09:07:58 2008
@@ -17,7 +17,6 @@
* under the License.
*/
-
package org.apache.uima.examples.tagger;
import java.io.FileInputStream;
@@ -35,45 +34,39 @@
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.FSIterator;
import org.apache.uima.cas.text.AnnotationIndex;
+import org.apache.uima.examples.tagger.trainAndTest.ModelGeneration;
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.ResourceInitializationException;
/**
- * UIMA Analysis Engine that invokes HMM POS tagger. HMM POS tagger generates a
- * Brown Corpus style POS tags. This annotator assumes that sentences and tokens have already been
- * annotated in the CAS with Sentence and Token annotations, respectively. We iterate over
- * sentences, then iterate over tokens in the current sentence to accumlate a list of words, then
- * invoke the HMM POS tagger on the list of words. For each Token we then update the posTag
- * field with the POS tag. The model file for the HMM POS tagger is specified as a parameter
- * (MODEL_FILE_PARAM).
- * Implements {@link Tagger}
- *
- */
+ * UIMA Analysis Engine that invokes HMM POS tagger. HMM POS tagger generates part-of-speech tags
+ * for every token. This annotator assumes that sentences and tokens have already been annotated in the CAS
+ * with Sentence and Token annotations, respectively. We iterate over sentences, then iterate over
+ * tokens in the current sentence to accumulate a list of words, then invoke the HMM POS tagger on
+ * the list of words. For each Token we then update the posTag field with the POS tag. The model
+ * file for the HMM POS tagger is specified as a parameter (MODEL_FILE_PARAM).
+ *
+ */
-public class HMMTagger extends JCasAnnotator_ImplBase{
+public class HMMTagger extends JCasAnnotator_ImplBase implements Tagger{
-
/**
* Model file name
*/
- String MODEL;
-
+ String MODEL;
/**
- * for a bigram model: N = 2, for a trigram model N=3
- * N is defined in parameter file
+ * for a bigram model: N = 2, for a trigram model N=3 N is defined in parameter file
*/
- int N;
-
- String END_OF_SENT_TAG;
+ public int N;
+
+ // public String END_OF_SENT_TAG;
+
+ public ModelGeneration my_model;
- /**
- * Model used for current tagging
- */
+ MappingInterface MAPPING;
+ boolean DO_MAPPING;
- ModelGeneration my_model;
- HMMTagger tagger;
-
/**
* Initialize the Annotator.
*
@@ -85,22 +78,30 @@
try {
// Get configuration parameter values
String paramFile = (String) aContext.getConfigParameterValue("PARAM_FILE");
-
-
- // create and load default properties
+
+ // create and load default properties
Properties defaultProps = new Properties();
FileInputStream in = new FileInputStream(paramFile);
defaultProps.load(in);
in.close();
-
+
MODEL = defaultProps.getProperty("MODEL");
-
+
String n = defaultProps.getProperty("N");
N = Integer.parseInt(n);
+
+ String b = defaultProps.getProperty("DO_MAPPING");
+ DO_MAPPING = Boolean.valueOf(b);
+
+ if (DO_MAPPING){
+ String m = defaultProps.getProperty("MAPPING");
+ MappingInterface klasse = (MappingInterface)(Class.forName(m)).newInstance();
+ MAPPING = klasse;
+ } else {
+ MAPPING = null;
+ }
+ my_model = get_model(MODEL);
- END_OF_SENT_TAG = defaultProps.getProperty("END_OF_SENT_TAG");
- my_model = get_model(MODEL);
-
} catch (Exception e) {
throw new ResourceInitializationException(e);
}
@@ -108,63 +109,76 @@
/**
* Reads a saved {@code MODEL} object from a file
- * @param filename model file
+ *
+ * @param filename
+ * model file
* @return {@link ModelGeneration}
*/
- public static ModelGeneration get_model(String filename){
-
+ public static ModelGeneration get_model(String filename) {
+
System.out.println("The used model is:" + filename);
- InputStream model = null;
+ InputStream model = null;
ModelGeneration oRead = null;
-
- try
- {
- model = new FileInputStream( filename );
- ObjectInputStream p = new ObjectInputStream( model);
+
+ try {
+ model = new FileInputStream(filename);
+ ObjectInputStream p = new ObjectInputStream(model);
oRead = (ModelGeneration) p.readObject();
- }
-
- catch ( IOException e ) { System.err.println( e ); }
- catch ( ClassNotFoundException e ) { System.err.println( e ); }
- finally { try {model.close(); } catch ( Exception e ) { } }
+ }
+
+ catch (IOException e) {
+ System.err.println(e);
+ } catch (ClassNotFoundException e) {
+ System.err.println(e);
+ } finally {
+ try {
+ model.close();
+ } catch (Exception e) {
+ }
+ }
return oRead;
}
-
-
+
/**
* Process a CAS.
*
* @see JCasAnnotator_ImplBase#process(JCas)
*/
+ @SuppressWarnings("unchecked")
public void process(JCas aJCas) throws AnalysisEngineProcessException {
ArrayList<TokenAnnotation> tokenList = new ArrayList<TokenAnnotation>();
ArrayList<String> wordList = new ArrayList<String>();
-
AnnotationIndex sentenceIndex = aJCas.getAnnotationIndex(SentenceAnnotation.type);
AnnotationIndex tokenIndex = aJCas.getAnnotationIndex(TokenAnnotation.type);
-
-
+
// iterate over Sentences
FSIterator sentenceIterator = sentenceIndex.iterator();
-
+
while (sentenceIterator.hasNext()) {
SentenceAnnotation sentence = (SentenceAnnotation) sentenceIterator.next();
-
+
tokenList.clear();
wordList.clear();
-
+
FSIterator tokenIterator = tokenIndex.subiterator(sentence);
while (tokenIterator.hasNext()) {
TokenAnnotation token = (TokenAnnotation) tokenIterator.next();
-
+
tokenList.add(token);
wordList.add(token.getCoveredText());
}
-
- List<String> wordTagList = Viterbi.process(N, wordList, END_OF_SENT_TAG, my_model.suffix_tree , my_model.suffix_tree_capitalized, my_model.transition_probs, my_model.word_probs, my_model.lambdas2, my_model.lambdas3, my_model.theta);
-
+
+ List<String> wordTagList = Viterbi.process(N, wordList, ".",
+ my_model.suffix_tree, my_model.suffix_tree_capitalized, my_model.transition_probs,
+ my_model.word_probs, my_model.lambdas2, my_model.lambdas3, my_model.theta);
+
+
+ if (MAPPING != null){
+ wordTagList = MAPPING.map_tags(wordTagList);
+ }
+
try {
for (int i = 0; i < tokenList.size(); i++) {
TokenAnnotation token = (TokenAnnotation) tokenList.get(i);
@@ -173,8 +187,8 @@
}
} catch (IndexOutOfBoundsException e) {
System.err.println("POS tagger error - list of tags shorter than list of words");
- }
- }
+ }
+ }
}
}
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/MappingInterface.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/MappingInterface.java?rev=619943&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/MappingInterface.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/MappingInterface.java Fri Feb 8 09:07:58 2008
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.examples.tagger;
+
+import java.util.List;
+
+/**
+ * Defines mapping for a tagset. For example, one may wish to map a more detailed
+ * tagset to a less distinctive one (i.e. tell a program to tag all verbs as just
+ * {@code VERB} instead of differentiating between {@code verb infinitive}, {@code verb imperative}, etc ...
+ *
+ */
+
+public interface MappingInterface {
+
+
+ public List map_tags(List pos);
+
+}
Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/TagMapping.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/TagMapping.java?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/TagMapping.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/TagMapping.java Fri Feb 8 09:07:58 2008
@@ -23,19 +23,27 @@
import java.util.ArrayList;
import java.util.List;
-/**
- */
-public class TagMapping {
+import org.apache.uima.examples.tagger.trainAndTest.Token;
- public static List<Token> map_tags(List<Token> tokens) {
+public class TagMapping implements MappingInterface{
+
+ /**
+ * Defines mapping for List<{@link Token}>
+ * E.g. if we need to map tags, given a list of {@code Tokens}, we need to map the
+ * {@code pos} field of every {@code Token} to a different {@code pos}.
+ * Basically the mapping performed in this class is just a case of simple "normalization",
+ * we just discard compound tags of the Brown corpus here.
+ */
+ @SuppressWarnings("unchecked")
+ public List map_tags(List tokens) {
// for (int i=0; i<sentences.size(); i++){ // iterate over sentences
List<Token> tokens2 = new ArrayList<Token>(tokens.size());
for (int x = 0; x < tokens.size(); x++) { // iterate over tokens of the sentence with their
- // corresponding POS
- Token current_token = tokens.get(x);
+ // corresponding POS
+ Token current_token = (Token)tokens.get(x);
String[] z = new String[2];
if (current_token.pos.contains("+")) {
z = current_token.pos.split("[+]");
@@ -68,5 +76,6 @@
return tokens2;
}
+
}
Modified: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Tagger.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Tagger.java?rev=619943&r1=619942&r2=619943&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Tagger.java (original)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Tagger.java Fri Feb 8 09:07:58 2008
@@ -23,47 +23,35 @@
package org.apache.uima.examples.tagger;
-import java.util.List;
+import org.apache.uima.UimaContext;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ResourceInitializationException;
/**
- * known implementations: - HMMTagger using Viterbi algorithm
+ * General tagger interface in case one would want to define further types of taggers.
+ *
+ * Known implementations: {@link HMMTagger} using {@link Viterbi} algorithm
* to compute the most probable path of parts of speech for a given sequence of tokens
- * @see Viterbi
+ * @see Viterbi, HMMTagger
*/
-// AT THE MOMENT IS USELESS TODO: check if we need it at all..and integrate if :)
public interface Tagger {
- /**
- * Initiates smoothing procedure for unknown words and N-grams
- * @return true or false, depending on whether {@code smoothing} is set in a {@code param.txt} file
- */
- public boolean set_smoothing();
/**
* Instantiates {@code MODEL} for current tagger
*/
- public void init();
+ public void initialize(UimaContext aContext) throws ResourceInitializationException;
/**
* Trains a new model for tagger, if a training is defined in {@code tagger.properties} file
* @see ModelGenerator
*/
- public void train();
-
- /**
- * Tags a sequence
- * @param wordList
- */
-
- public List process(List wordList);
-
- /**
- * Tests tagging accuracy if reference corpus is available
- */
- public void test();
+
+ public void process(JCas aJCas) throws AnalysisEngineProcessException;
}