You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by tw...@apache.org on 2007/10/25 11:35:07 UTC
svn commit: r588179 - in /incubator/uima/sandbox/trunk/Tagger: ./ desc/
src/main/ src/main/java/ src/main/java/org/ src/main/java/org/apache/
src/main/java/org/apache/uima/ src/main/java/org/apache/uima/examples/
src/main/java/org/apache/uima/examples/...
Author: twgoetz
Date: Thu Oct 25 02:35:06 2007
New Revision: 588179
URL: http://svn.apache.org/viewvc?rev=588179&view=rev
Log:
Jira UIMA-605: first code drop of HMM tagger.
https://issues.apache.org/jira/browse/UIMA-605
Added:
incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml
incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml
incubator/uima/sandbox/trunk/Tagger/params.txt
incubator/uima/sandbox/trunk/Tagger/src/main/
incubator/uima/sandbox/trunk/Tagger/src/main/java/
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation_Type.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation_Type.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HmmTaggerTest.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/ModelGeneration.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/ModelGenerator.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Tagger.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/UnknownHandler.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/UnknownWordHandler.java
incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Viterbi.java
incubator/uima/sandbox/trunk/Tagger/tagger.properties
Modified:
incubator/uima/sandbox/trunk/Tagger/ (props changed)
incubator/uima/sandbox/trunk/Tagger/pom.xml
Propchange: incubator/uima/sandbox/trunk/Tagger/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Thu Oct 25 02:35:06 2007
@@ -1,2 +1,3 @@
.classpath
.project
+target
Added: incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml (added)
+++ incubator/uima/sandbox/trunk/Tagger/desc/HmmTagger.xml Thu Oct 25 02:35:06 2007
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<analysisEngineDescription xmlns="http://uima.apache.org/resourceSpecifier">
+ <frameworkImplementation>org.apache.uima.java</frameworkImplementation>
+ <primitive>true</primitive>
+ <annotatorImplementationName>org.apache.uima.examples.tagger.HMMTagger</annotatorImplementationName>
+ <analysisEngineMetaData>
+ <name>Hidden Markov Model - Part of Speech Tagger</name>
+ <description>A configuration of the HmmTaggerAnnotator that looks for parts of speech of identified tokens within existing Sentence and Token annotations. See also WhitespaceTokenizer.xml.</description>
+ <version>1.0</version>
+ <vendor>The Apache Software Foundation</vendor>
+ <configurationParameters>
+ <configurationParameter>
+ <name>PARAM_FILE</name>
+ <type>String</type>
+ <multiValued>false</multiValued>
+ <mandatory>true</mandatory>
+ </configurationParameter>
+ </configurationParameters>
+ <configurationParameterSettings>
+ <nameValuePair>
+ <name>PARAM_FILE</name>
+ <value>
+ <string>tagger.properties</string>
+ </value>
+ </nameValuePair>
+ </configurationParameterSettings>
+ <typeSystemDescription>
+ <typeDescription>
+ <name>org.apache.uima.TokenAnnotation</name>
+ <description>Single token annotation</description>
+ <supertypeName>uima.tcas.Annotation</supertypeName>
+ <features>
+ <featureDescription>
+ <name>posTag</name>
+ <description>contains part-of-speech of a corresponding token</description>
+ <rangeTypeName>uima.cas.String</rangeTypeName>
+ </featureDescription>
+ </features>
+ </typeDescription>
+ <typeDescription>
+ <name>org.apache.uima.SentenceAnnotation</name>
+ <description>sentence annotation</description>
+ <supertypeName>uima.tcas.Annotation</supertypeName>
+ <features>
+
+ </features>
+ </typeDescription>
+ </typeSystemDescription>
+ <typePriorities/>
+ <fsIndexCollection/>
+ <operationalProperties>
+ <modifiesCas>true</modifiesCas>
+ <multipleDeploymentAllowed>true</multipleDeploymentAllowed>
+ <outputsNewCASes>false</outputsNewCASes>
+ </operationalProperties>
+ </analysisEngineMetaData>
+ <resourceManagerConfiguration/>
+</analysisEngineDescription>
Added: incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml (added)
+++ incubator/uima/sandbox/trunk/Tagger/desc/HmmTaggerTAE.xml Thu Oct 25 02:35:06 2007
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<analysisEngineDescription xmlns="http://uima.apache.org/resourceSpecifier">
+ <frameworkImplementation>org.apache.uima.java</frameworkImplementation>
+ <primitive>false</primitive>
+ <delegateAnalysisEngineSpecifiers>
+ <delegateAnalysisEngine key="SimpleTokenAndSentenceAnnotator">
+ <import name="WhitespaceTokenizer"/>
+ </delegateAnalysisEngine>
+ <delegateAnalysisEngine key="HmmTagger">
+ <import name="HmmTagger"/>
+ </delegateAnalysisEngine>
+ </delegateAnalysisEngineSpecifiers>
+ <analysisEngineMetaData>
+ <name>HmmTaggerTAE</name>
+ <description/>
+ <version/>
+ <vendor/>
+ <configurationParameters searchStrategy="language_fallback">
+ <configurationParameter>
+ <name>PARAM_FILE</name>
+ <type>String</type>
+ <multiValued>false</multiValued>
+ <mandatory>true</mandatory>
+ <overrides>
+ <parameter>HmmTagger/PARAM_FILE</parameter>
+ </overrides>
+ </configurationParameter>
+ </configurationParameters>
+ <configurationParameterSettings>
+ <nameValuePair>
+ <name>PARAM_FILE</name>
+ <value>
+ <string>tagger.properties</string>
+ </value>
+ </nameValuePair>
+ </configurationParameterSettings>
+ <flowConstraints>
+ <fixedFlow>
+ <node>SimpleTokenAndSentenceAnnotator</node>
+ <node>HmmTagger</node>
+ </fixedFlow>
+ </flowConstraints>
+ <typePriorities/>
+ <fsIndexCollection/>
+ <capabilities>
+ <capability>
+ <inputs>
+ <type allAnnotatorFeatures="true">org.apache.uima.examples.tokenizer.Token</type>
+ <type allAnnotatorFeatures="true">org.apache.uima.examples.tokenizer.Sentence</type>
+ </inputs>
+ <outputs>
+ <type allAnnotatorFeatures="true">org.apache.uima.examples.tokenizer.Token</type>
+ </outputs>
+ <languagesSupported>
+ <language>en</language>
+ </languagesSupported>
+ </capability>
+ </capabilities>
+ <operationalProperties>
+ <modifiesCas>true</modifiesCas>
+ <multipleDeploymentAllowed>true</multipleDeploymentAllowed>
+ <outputsNewCASes>false</outputsNewCASes>
+ </operationalProperties>
+ </analysisEngineMetaData>
+ <resourceManagerConfiguration/>
+</analysisEngineDescription>
Added: incubator/uima/sandbox/trunk/Tagger/params.txt
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/params.txt?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/params.txt (added)
+++ incubator/uima/sandbox/trunk/Tagger/params.txt Thu Oct 25 02:35:06 2007
@@ -0,0 +1,16 @@
+%% PARAMETERS FOR TAGGER %%
+%
+MODEL = 'brown_model_trigramm.dat'
+
+%% uncomment the following two lines if training is desirable, and change the corpus directory destination path,
+%% e.g. T_CORPUS='brown' for all the files of the brown corpus which are in the 'brown/' -directory
+%TRAINING = 'true'
+%T_CORPUS = 'brown'
+
+%% uncomment the following line for smoothing
+
+%SMOOTHING = 'true'
+
+%% N =2 or N= 3 are supported, default N=3
+
+N='3'
\ No newline at end of file
Modified: incubator/uima/sandbox/trunk/Tagger/pom.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/pom.xml?rev=588179&r1=588178&r2=588179&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/pom.xml (original)
+++ incubator/uima/sandbox/trunk/Tagger/pom.xml Thu Oct 25 02:35:06 2007
@@ -46,6 +46,12 @@
<version>2.2.0-incubating</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.uima</groupId>
+ <artifactId>WhitespaceTokenizer</artifactId>
+ <version>2.2-incubating-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation.java?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation.java Thu Oct 25 02:35:06 2007
@@ -0,0 +1,62 @@
+
+
+/* First created by JCasGen Thu Oct 25 11:28:37 CEST 2007 */
+package org.apache.uima;
+
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.JCasRegistry;
+import org.apache.uima.jcas.cas.TOP_Type;
+
+import org.apache.uima.jcas.tcas.Annotation;
+
+
+/** sentence annotation
+ * Updated by JCasGen Thu Oct 25 11:28:37 CEST 2007
+ * XML source: C:/code/ApacheUIMA/Tagger/desc/HmmTaggerTAE.xml
+ * @generated */
+public class SentenceAnnotation extends Annotation {
+ /** @generated
+ * @ordered
+ */
+ public final static int typeIndexID = JCasRegistry.register(SentenceAnnotation.class);
+ /** @generated
+ * @ordered
+ */
+ public final static int type = typeIndexID;
+ /** @generated */
+ public int getTypeIndexID() {return typeIndexID;}
+
+ /** Never called. Disable default constructor
+ * @generated */
+ protected SentenceAnnotation() {}
+
+ /** Internal - constructor used by generator
+ * @generated */
+ public SentenceAnnotation(int addr, TOP_Type type) {
+ super(addr, type);
+ readObject();
+ }
+
+ /** @generated */
+ public SentenceAnnotation(JCas jcas) {
+ super(jcas);
+ readObject();
+ }
+
+ /** @generated */
+ public SentenceAnnotation(JCas jcas, int begin, int end) {
+ super(jcas);
+ setBegin(begin);
+ setEnd(end);
+ readObject();
+ }
+
+ /** <!-- begin-user-doc -->
+ * Write your own initialization here
+ * <!-- end-user-doc -->
+ @generated modifiable */
+ private void readObject() {}
+
+}
+
+
\ No newline at end of file
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation_Type.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation_Type.java?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation_Type.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/SentenceAnnotation_Type.java Thu Oct 25 02:35:06 2007
@@ -0,0 +1,55 @@
+
+/* First created by JCasGen Thu Oct 25 11:28:37 CEST 2007 */
+package org.apache.uima;
+
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.JCasRegistry;
+import org.apache.uima.cas.impl.CASImpl;
+import org.apache.uima.cas.impl.FSGenerator;
+import org.apache.uima.cas.FeatureStructure;
+import org.apache.uima.cas.impl.TypeImpl;
+import org.apache.uima.cas.Type;
+import org.apache.uima.jcas.tcas.Annotation_Type;
+
+/** sentence annotation
+ * Updated by JCasGen Thu Oct 25 11:28:37 CEST 2007
+ * @generated */
+public class SentenceAnnotation_Type extends Annotation_Type {
+ /** @generated */
+ protected FSGenerator getFSGenerator() {return fsGenerator;}
+ /** @generated */
+ private final FSGenerator fsGenerator =
+ new FSGenerator() {
+ public FeatureStructure createFS(int addr, CASImpl cas) {
+ if (SentenceAnnotation_Type.this.useExistingInstance) {
+ // Return eq fs instance if already created
+ FeatureStructure fs = SentenceAnnotation_Type.this.jcas.getJfsFromCaddr(addr);
+ if (null == fs) {
+ fs = new SentenceAnnotation(addr, SentenceAnnotation_Type.this);
+ SentenceAnnotation_Type.this.jcas.putJfsFromCaddr(addr, fs);
+ return fs;
+ }
+ return fs;
+ } else return new SentenceAnnotation(addr, SentenceAnnotation_Type.this);
+ }
+ };
+ /** @generated */
+ public final static int typeIndexID = SentenceAnnotation.typeIndexID;
+ /** @generated
+ @modifiable */
+ public final static boolean featOkTst = JCasRegistry.getFeatOkTst("org.apache.uima.SentenceAnnotation");
+
+
+
+ /** initialize variables to correspond with Cas Type and Features
+ * @generated */
+ public SentenceAnnotation_Type(JCas jcas, Type casType) {
+ super(jcas, casType);
+ casImpl.getFSClassRegistry().addGeneratorForType((TypeImpl)this.casType, getFSGenerator());
+
+ }
+}
+
+
+
+
\ No newline at end of file
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation.java?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation.java Thu Oct 25 02:35:06 2007
@@ -0,0 +1,98 @@
+
+
+/* First created by JCasGen Thu Oct 25 11:28:37 CEST 2007 */
+package org.apache.uima;
+
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.JCasRegistry;
+import org.apache.uima.jcas.cas.TOP_Type;
+
+import org.apache.uima.jcas.tcas.Annotation;
+
+
+/** Single token annotation
+ * Updated by JCasGen Thu Oct 25 11:28:37 CEST 2007
+ * XML source: C:/code/ApacheUIMA/Tagger/desc/HmmTaggerTAE.xml
+ * @generated */
+public class TokenAnnotation extends Annotation {
+ /** @generated
+ * @ordered
+ */
+ public final static int typeIndexID = JCasRegistry.register(TokenAnnotation.class);
+ /** @generated
+ * @ordered
+ */
+ public final static int type = typeIndexID;
+ /** @generated */
+ public int getTypeIndexID() {return typeIndexID;}
+
+ /** Never called. Disable default constructor
+ * @generated */
+ protected TokenAnnotation() {}
+
+ /** Internal - constructor used by generator
+ * @generated */
+ public TokenAnnotation(int addr, TOP_Type type) {
+ super(addr, type);
+ readObject();
+ }
+
+ /** @generated */
+ public TokenAnnotation(JCas jcas) {
+ super(jcas);
+ readObject();
+ }
+
+ /** @generated */
+ public TokenAnnotation(JCas jcas, int begin, int end) {
+ super(jcas);
+ setBegin(begin);
+ setEnd(end);
+ readObject();
+ }
+
+ /** <!-- begin-user-doc -->
+ * Write your own initialization here
+ * <!-- end-user-doc -->
+ @generated modifiable */
+ private void readObject() {}
+
+
+
+ //*--------------*
+ //* Feature: tokenType
+
+ /** getter for tokenType - gets token type
+ * @generated */
+ public String getTokenType() {
+ if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_tokenType == null)
+ jcasType.jcas.throwFeatMissing("tokenType", "org.apache.uima.TokenAnnotation");
+ return jcasType.ll_cas.ll_getStringValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_tokenType);}
+
+ /** setter for tokenType - sets token type
+ * @generated */
+ public void setTokenType(String v) {
+ if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_tokenType == null)
+ jcasType.jcas.throwFeatMissing("tokenType", "org.apache.uima.TokenAnnotation");
+ jcasType.ll_cas.ll_setStringValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_tokenType, v);}
+
+
+ //*--------------*
+ //* Feature: posTag
+
+ /** getter for posTag - gets contains part-of-speech of a corresponding token
+ * @generated */
+ public String getPosTag() {
+ if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_posTag == null)
+ jcasType.jcas.throwFeatMissing("posTag", "org.apache.uima.TokenAnnotation");
+ return jcasType.ll_cas.ll_getStringValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_posTag);}
+
+ /** setter for posTag - sets contains part-of-speech of a corresponding token
+ * @generated */
+ public void setPosTag(String v) {
+ if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_posTag == null)
+ jcasType.jcas.throwFeatMissing("posTag", "org.apache.uima.TokenAnnotation");
+ jcasType.ll_cas.ll_setStringValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_posTag, v);}
+ }
+
+
\ No newline at end of file
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation_Type.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation_Type.java?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation_Type.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/TokenAnnotation_Type.java Thu Oct 25 02:35:06 2007
@@ -0,0 +1,101 @@
+
+/* First created by JCasGen Thu Oct 25 11:28:37 CEST 2007 */
+package org.apache.uima;
+
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.JCasRegistry;
+import org.apache.uima.cas.impl.CASImpl;
+import org.apache.uima.cas.impl.FSGenerator;
+import org.apache.uima.cas.FeatureStructure;
+import org.apache.uima.cas.impl.TypeImpl;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.impl.FeatureImpl;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.jcas.tcas.Annotation_Type;
+
+/** Single token annotation
+ * Updated by JCasGen Thu Oct 25 11:28:37 CEST 2007
+ * @generated */
+public class TokenAnnotation_Type extends Annotation_Type {
+ /** @generated */
+ protected FSGenerator getFSGenerator() {return fsGenerator;}
+ /** @generated */
+ private final FSGenerator fsGenerator =
+ new FSGenerator() {
+ public FeatureStructure createFS(int addr, CASImpl cas) {
+ if (TokenAnnotation_Type.this.useExistingInstance) {
+ // Return eq fs instance if already created
+ FeatureStructure fs = TokenAnnotation_Type.this.jcas.getJfsFromCaddr(addr);
+ if (null == fs) {
+ fs = new TokenAnnotation(addr, TokenAnnotation_Type.this);
+ TokenAnnotation_Type.this.jcas.putJfsFromCaddr(addr, fs);
+ return fs;
+ }
+ return fs;
+ } else return new TokenAnnotation(addr, TokenAnnotation_Type.this);
+ }
+ };
+ /** @generated */
+ public final static int typeIndexID = TokenAnnotation.typeIndexID;
+ /** @generated
+ @modifiable */
+ public final static boolean featOkTst = JCasRegistry.getFeatOkTst("org.apache.uima.TokenAnnotation");
+
+ /** @generated */
+ final Feature casFeat_tokenType;
+ /** @generated */
+ final int casFeatCode_tokenType;
+ /** @generated */
+ public String getTokenType(int addr) {
+ if (featOkTst && casFeat_tokenType == null)
+ jcas.throwFeatMissing("tokenType", "org.apache.uima.TokenAnnotation");
+ return ll_cas.ll_getStringValue(addr, casFeatCode_tokenType);
+ }
+ /** @generated */
+ public void setTokenType(int addr, String v) {
+ if (featOkTst && casFeat_tokenType == null)
+ jcas.throwFeatMissing("tokenType", "org.apache.uima.TokenAnnotation");
+ ll_cas.ll_setStringValue(addr, casFeatCode_tokenType, v);}
+
+
+
+ /** @generated */
+ final Feature casFeat_posTag;
+ /** @generated */
+ final int casFeatCode_posTag;
+ /** @generated */
+ public String getPosTag(int addr) {
+ if (featOkTst && casFeat_posTag == null)
+ jcas.throwFeatMissing("posTag", "org.apache.uima.TokenAnnotation");
+ return ll_cas.ll_getStringValue(addr, casFeatCode_posTag);
+ }
+ /** @generated */
+ public void setPosTag(int addr, String v) {
+ if (featOkTst && casFeat_posTag == null)
+ jcas.throwFeatMissing("posTag", "org.apache.uima.TokenAnnotation");
+ ll_cas.ll_setStringValue(addr, casFeatCode_posTag, v);}
+
+
+
+
+
+ /** initialize variables to correspond with Cas Type and Features
+ * @generated */
+ public TokenAnnotation_Type(JCas jcas, Type casType) {
+ super(jcas, casType);
+ casImpl.getFSClassRegistry().addGeneratorForType((TypeImpl)this.casType, getFSGenerator());
+
+
+ casFeat_tokenType = jcas.getRequiredFeatureDE(casType, "tokenType", "uima.cas.String", featOkTst);
+ casFeatCode_tokenType = (null == casFeat_tokenType) ? JCas.INVALID_FEATURE_CODE : ((FeatureImpl)casFeat_tokenType).getCode();
+
+
+ casFeat_posTag = jcas.getRequiredFeatureDE(casType, "posTag", "uima.cas.String", featOkTst);
+ casFeatCode_posTag = (null == casFeat_posTag) ? JCas.INVALID_FEATURE_CODE : ((FeatureImpl)casFeat_posTag).getCode();
+
+ }
+}
+
+
+
+
\ No newline at end of file
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HMMTagger.java Thu Oct 25 02:35:06 2007
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.uima.examples.tagger;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.uima.SentenceAnnotation;
+import org.apache.uima.TokenAnnotation;
+import org.apache.uima.UimaContext;
+import org.apache.uima.analysis_component.JCasAnnotator_ImplBase;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.cas.FSIterator;
+import org.apache.uima.cas.text.AnnotationIndex;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ResourceInitializationException;
+
+/**
+ * UIMA Analysis Engine that invokes HMM POS tagger. HMM POS tagger generates a
+ * Brown Corpus style POS tags. This annotator assumes that sentences and tokens have already been
+ * annotated in the CAS with Sentence and Token annotations, respectively. We iterate over
+ * sentences, then iterate over tokens in the current sentence to accumlate a list of words, then
+ * invoke the HMM POS tagger on the list of words. For each Token we then update the posTag
+ * field with the POS tag. The model file for the HMM POS tagger is specified as a parameter
+ * (MODEL_FILE_PARAM).
+ * Implements {@link Tagger}
+ *
+ */
+
+public class HMMTagger extends JCasAnnotator_ImplBase{
+
+
+ //public static final String PARAM_FILE = "tagger.properties";
+
+ /**
+ * Defines whether {@code TRAINING} of a new model should be done, or whether we apply a ready pre-trained model from a parameter file
+ */
+ boolean TRAINING=false;
+
+ /**
+ * Defines whether smoothing should be applied to UNKNOWN
+ */
+ boolean SMOOTHING=false;
+ /**
+ * Model file name
+ */
+ String MODEL;
+
+ /**
+ * Training corpus file name
+ */
+ String T_CORPUS;
+
+ /**
+ * for a bigram model: N = 2, for a trigram model N=3
+ * N is defined in parameter file
+ */
+ int N;
+
+ String END_OF_SENT_TAG;
+
+ /**
+ * Model used for current tagging
+ */
+
+ ModelGeneration my_model;
+ HMMTagger tagger;
+
+ /**
+ * Initialize the Annotator.
+ *
+ * @see JCasAnnotator_ImplBase#initialize(UimaContext)
+ */
+ public void initialize(UimaContext aContext) throws ResourceInitializationException {
+ super.initialize(aContext);
+
+ try {
+ // Get configuration parameter values
+ String paramFile = (String) aContext.getConfigParameterValue("PARAM_FILE");
+
+
+ // create and load default properties
+ Properties defaultProps = new Properties();
+ FileInputStream in = new FileInputStream(paramFile);
+ defaultProps.load(in);
+ in.close();
+
+ MODEL = defaultProps.getProperty("MODEL");
+ String train = defaultProps.getProperty("TRAINING");
+
+ if (train == "true") {
+ TRAINING = true ;
+ } else {TRAINING = false; }
+
+ T_CORPUS = defaultProps.getProperty("T_CORPUS");
+
+ String smooth = defaultProps.getProperty("SMOOTHING");
+ if (smooth == "true") {
+ SMOOTHING = true ;
+ } else {SMOOTHING = false;}
+
+ String n = defaultProps.getProperty("N");
+ N = Integer.parseInt(n);
+
+ END_OF_SENT_TAG = defaultProps.getProperty("END_OF_SENT_TAG");
+ my_model = get_model(MODEL);
+
+ } catch (Exception e) {
+ throw new ResourceInitializationException(e);
+ }
+ }
+
+ /**
+ * Reads a saved {@code MODEL} object from a file
+ * @param filename model file
+ * @return {@link ModelGeneration}
+ */
+ public ModelGeneration get_model(String filename){
+ // filename = "brown_model_trigramm.dat"; // model - trigram file
+ System.out.println("The used model is:" + filename);
+ InputStream model = null;
+ ModelGeneration oRead = null;
+
+ try
+ {
+ model = new FileInputStream( filename );
+ ObjectInputStream p = new ObjectInputStream( model);
+ oRead = (ModelGeneration) p.readObject();
+ }
+
+ catch ( IOException e ) { System.err.println( e ); }
+ catch ( ClassNotFoundException e ) { System.err.println( e ); }
+ finally { try {model.close(); } catch ( Exception e ) { } }
+ return oRead;
+ }
+
+ /**
+ * Initiates {@code MODEL} used for current tagging.
+ * If {@link #TRAINING} is defined, then trains a new Model from a reference corpus;
+ * otherwise reads a ready model from {@link #MODEL} defined in parameter file
+ */
+
+ public void init(){
+
+ }
+
+
+ /**
+ * Trains a new model {@link #N} - gram probabilistic model from a predefined training corpus ({@code #T_CORPUS}),
+ * write a new model to the {@link #MODEL} - file
+ */
+
+ public void train() {
+ my_model = new ModelGeneration(N, T_CORPUS, MODEL);
+ }
+
+ /**
+ * TODO Tests tagging results
+ */
+
+ public void test(){
+ }
+
+ public boolean set_smoothing() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ /*
+ public static double log2(double d){
+ return Math.log(d)/Math.log(2.0);
+ }
+
+
+ public List process(List wordList) {
+ // Viterbi v = new Viterbi();
+
+ List sent_tagging = Viterbi.process(N, wordList, END_OF_SENT_TAG, null , my_model.transition_probs, my_model.word_probs);
+ System.out.println(sent_tagging);
+ return sent_tagging;
+ }
+ */
+
+
+ /**
+ * Process a CAS.
+ *
+ * @see JCasAnnotator_ImplBase#process(JCas)
+ */
+ public void process(JCas aJCas) throws AnalysisEngineProcessException {
+
+ ArrayList tokenList = new ArrayList();
+ ArrayList wordList = new ArrayList();
+
+
+ AnnotationIndex sentenceIndex = aJCas.getAnnotationIndex(SentenceAnnotation.type);
+ AnnotationIndex tokenIndex = aJCas.getAnnotationIndex(TokenAnnotation.type);
+
+
+ // iterate over Sentences
+ FSIterator sentenceIterator = sentenceIndex.iterator();
+
+ while (sentenceIterator.hasNext()) {
+ SentenceAnnotation sentence = (SentenceAnnotation) sentenceIterator.next();
+
+ tokenList.clear();
+ wordList.clear();
+
+ FSIterator tokenIterator = tokenIndex.subiterator(sentence);
+ while (tokenIterator.hasNext()) {
+ TokenAnnotation token = (TokenAnnotation) tokenIterator.next();
+
+ tokenList.add(token);
+ wordList.add(token.getCoveredText());
+ }
+
+ List wordTagList = Viterbi.process(N, wordList, END_OF_SENT_TAG, null , my_model.transition_probs, my_model.word_probs);
+ System.out.println(wordTagList);
+
+ try {
+ for (int i = 0; i < tokenList.size(); i++) {
+ TokenAnnotation token = (TokenAnnotation) tokenList.get(i);
+ String posTag = (String) wordTagList.get(i);
+ token.setPosTag(posTag);
+ }
+ } catch (IndexOutOfBoundsException e) {
+ System.err.println("POS tagger error - list of tags shorter than list of words");
+ }
+ }
+ }
+
+}
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HmmTaggerTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HmmTaggerTest.java?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HmmTaggerTest.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/HmmTaggerTest.java Thu Oct 25 02:35:06 2007
@@ -0,0 +1,272 @@
+/*
+ *Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.uima.examples.tagger;
+
+import java.io.*;
+import java.util.*;
+
+
+/**
+ * UIMA Analysis Engine that invokes HMM POS tagger. HMM POS tagger generates a
+ * Brown Corpus style POS tags. This annotator assumes that sentences and tokens have already been
+ * annotated in the CAS with Sentence and Token annotations, respectively. We iterate over
+ * sentences, then iterate over tokens in the current sentence to accumlate a list of words, then
+ * invoke the HMM POS tagger on the list of words. For each Token we then update the posTag
+ * field with the POS tag. The model file for the HMM POS tagger is specified as a parameter
+ * (MODEL_FILE_PARAM).
+ * Implements {@link Tagger}
+ *
+ */
+
+public class HmmTaggerTest{
+
+
+ /**
+ * Defines whether {@code TRAINING} of a new model should be done, or whether we apply a ready pre-trained model from a parameter file
+ */
+ boolean TRAINING=false;
+
+ /**
+ * Defines whether smoothing should be applied to UNKNOWN
+ */
+ boolean SMOOTHING=false;
+ /**
+ * Model file name
+ */
+ String MODEL;
+
+ /**
+ * Training corpus file name
+ */
+ String T_CORPUS;
+
+ /**
+ * for a bigram model: N = 2, for a trigram model N=3
+ * N is defined in parameter file
+ */
+ int N;
+
+ /**
+ * Model used for current tagging
+ */
+
+ ModelGeneration my_model;
+
+ /**
+ * Construct a HMM tagger with default parameter settings.
+ * The defaults are ... TO_DO
+ */
+
+ public HmmTaggerTest() {
+ }
+
+ /**
+ * @param text
+ *
+ */
+ public HmmTaggerTest(List<String> text) {
+ this.read_params();
+ System.out.println(MODEL);
+ System.out.println(TRAINING);
+ System.out.println(T_CORPUS);
+ System.out.println(SMOOTHING);
+ System.out.println(N);
+ this.init();
+ }
+
+
+
+ /**
+ * Reads parameters for tagging from a parameter file
+ */
+ public void read_params(){
+ System.out.println("Reading parameters from the file");
+ // InputStream params = null;
+ String line = null;
+
+ try
+ {
+ BufferedReader in = new BufferedReader(new FileReader("params.txt"));
+
+ while ((line=in.readLine()) != null){
+ if (line.trim().length() > 0){
+
+ if (line.startsWith("MODEL")) {
+ int s = line.indexOf("'");
+ int e = line.lastIndexOf("'");
+ MODEL = line.substring(s+1, e);
+ continue;
+ }
+
+ if (line.startsWith("TRAINING")) {
+
+ int s = line.indexOf("'");
+ int e = line.lastIndexOf("'");
+ String t =line.substring(s+1, e);
+ if (String.valueOf(TRAINING) !=t) {
+ TRAINING = true ;
+ continue;
+ }
+ }
+ if (line.startsWith("T_CORPUS")) {
+ int s = line.indexOf("'");
+ int e = line.lastIndexOf("'");
+ T_CORPUS = line.substring(s+1, e);
+ continue;
+ }
+
+ if (line.startsWith("N")) {
+ System.out.println(line);
+ int s = line.indexOf("'");
+ int e = line.lastIndexOf("'");
+ String n = line.substring(s+1, e);
+ N = Integer.parseInt(n);
+ }
+ }
+ else {
+ continue;
+ }
+ // }
+ }
+ in.close();
+ }
+ catch (IOException e)
+ {
+ System.out.println(e);
+ }
+ }
+
+
+ /**
+ * @param filename model file
+ * @return {@link ModelGeneration}
+ */
+ public ModelGeneration get_model(String filename){
+ // filename = "brown_model_trigramm.dat"; // model - trigram file
+ System.out.println("The used model is:" + filename);
+ InputStream model = null;
+ ModelGeneration oRead = null;
+
+ try
+ {
+ model = new FileInputStream( filename );
+ ObjectInputStream p = new ObjectInputStream( model);
+ oRead = (ModelGeneration) p.readObject();
+ // System.out.println(oRead.InputDir);
+ // System.out.println(oRead.OutputFile);
+ // System.out.println(oRead.N);
+ // System.out.println(oRead.word_probs);
+ // System.out.println(oRead.transition_probs);
+
+ }
+
+ catch ( IOException e ) { System.err.println( e ); }
+ catch ( ClassNotFoundException e ) { System.err.println( e ); }
+ finally { try {model.close(); } catch ( Exception e ) { } }
+ return oRead;
+ }
+
+ /**
+ * Initiates the model used for current tagging.
+ * If {@link #TRAINING} is defined, then trains a new Model from a reference corpus;
+ * otherwise reads a ready model from {@link #MODEL} defined in parameter file
+ */
+
+ public void init(){
+ if (TRAINING) {train();}
+ else my_model = get_model(MODEL);
+ }
+
+
+ /**
+ * Trains a new model {@link #N} - gram probabilistic model from a predefined training corpus ({@code #T_CORPUS}),
+ * write a new model to the {@link #MODEL} - file
+ */
+
+ public void train() {
+ my_model = new ModelGeneration(N, T_CORPUS, MODEL);
+ }
+
+ /**
+ * Tests tagging results
+ */
+
+ public void test(){
+
+ }
+
+ public void process() {
+ // TODO Auto-generated method stub
+
+ }
+
+ public boolean set_smoothing() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ public static double log2(double d){
+ return Math.log(d)/Math.log(2.0);
+ }
+
+
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+
+ List<String> sent = new ArrayList<String>();
+
+ sent.add("Clinton");
+ sent.add("loves");
+ sent.add("Wansley");
+ sent.add(".");
+
+ System.out.println(sent);
+ HmmTaggerTest hmm = new HmmTaggerTest(sent);
+
+ System.out.println("going over to Viterbi");
+
+ List sent_tagging = Viterbi.process(hmm.N, sent,"." , null , hmm.my_model.transition_probs, hmm.my_model.word_probs);
+ System.out.println(sent_tagging);
+
+
+ /**
+ * TEST
+
+
+ System.out.println("IN_DT_IN = "+hmm.my_model.transition_probs.get("in_dt_in"));
+ System.out.println(-4/2*log2(2048/2)-2*log2(1024));
+
+
+ double log_test = -86/143*log2(86/143)-10/143*log2(10/143)-45/143*log2(45/143)-2/143*log2(2/143);
+ System.out.println(log_test); */
+ }
+
+
+
+
+
+}
+
+
+
+
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/ModelGeneration.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/ModelGeneration.java?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/ModelGeneration.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/ModelGeneration.java Thu Oct 25 02:35:06 2007
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.uima.examples.tagger;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.uima.TokenAnnotation;
+
+/**
+ * Trains an N-gram model for the tagger, iterating over the files from some predefined training directory
+ * Writes the resulting model to a binary file
+ *
+ */
+
+
+public class ModelGeneration implements java.io.Serializable, ModelGenerator{
+
+ /**
+ * @serial
+ */
+ private static final long serialVersionUID = 1L;
+
+
+ /**
+ * Map containing {@code <word,tag>} probabilities, that is probability of a certain word given a certain tag at a time t: P(word<sub>t</sub>|tag<sub>t</sub>))
+ *
+ */
+ public Map<String, Map<String, Double>> word_probs = new HashMap<String, Map<String,Double>>();
+
+ /**
+ * Map containing N-gram probabilities
+ */
+
+ public Map<String, Double> transition_probs = new HashMap<String, Double>() ;
+
+
+ private List<String> posList = new ArrayList<String>();
+
+ int N; // for the N-gram model
+
+ String InputDir;
+ String OutputFile;
+
+ /*
+ public ModelGeneration() {
+ this(3, InputDir, OutputFile);
+ }
+ */
+
+ /**
+ * @param N N=1, 2 or 3
+ * @param InputDir input directory name
+ * @param OutputFile output file name
+ *
+ */
+ public ModelGeneration(int N, String InputDir, String OutputFile) {
+ this.N = N;
+ this.InputDir = InputDir;
+ this.OutputFile = OutputFile;
+ word_probs = get_word_probs(InputDir);
+ transition_probs = get_transition_probs(2);
+ transition_probs.putAll(get_transition_probs(3));
+ write_to_file(OutputFile);
+ }
+
+ /**
+ * Reads file names from Directory
+ * @param directory name
+ * @return an array of file names in the directory
+ */
+
+ public static String [] read_dir(String directory){
+ File dir = new File(directory);
+ String[] list = dir.list();
+ String[] new_list = dir.list();
+ for (int i = 0; i < list.length; i++) {
+ String dir_list = directory+"/"+list[i];
+ new_list[i]=dir_list;
+ System.out.println(new_list[i]);
+ }
+
+ return new_list;
+ }
+
+ /**
+ * Reads Brown Corpus from NLTK Distribution Format. Iterates over all files in the directory, which are in a sentence per line format,
+ * and returns all sentences in the collection in a list
+ * @param files an array of file names
+ * @return a list of sentences from all files
+ *
+ */
+
+
+ private List<String> read_corpus(String [] files){
+
+ String line;
+ List<String> text = new ArrayList<String>();
+
+ for (int i = 0; i < files.length; i++) {
+ System.out.println(files[i]);
+ String file = files[i];
+ try
+ {
+ BufferedReader in = new BufferedReader(new FileReader(file));
+
+ while ((line=in.readLine()) != null){
+ if (line.trim().length() > 0){
+ text.add(line);
+ }
+ }
+ in.close();
+ }
+ catch (IOException e)
+ {
+ System.out.println(e);
+ return null;
+ }
+ }
+ return text;
+ }
+
+ /**
+ * Reads sentences, extracts {@code <word, possible parts-of-speech>} frequency patterns
+ * @param corpus list containing all sentences of the training corpus
+ * @return map containing frequency counts for {@code <word, its pos>}
+ */
+ private Map<String,Map<String, Double>> get_lexicon(List<String> corpus){
+
+ //simple tokenizer: match one or more spaces
+ String delimiters = " +";
+ //
+ Map<String,Map<String, Double>> lexicon= new HashMap<String,Map<String, Double>>();
+
+
+ for (int i=0; i<corpus.size(); i++){ // iterate over sentences
+
+ String [] tokens = corpus.get(i).split(delimiters);
+
+ for (int x=0; x<tokens.length; x++){ // iterate over tokens with their corresponding POS
+
+ tokens[x] = tokens[x].replaceAll("[\\n$\\t^I]+","");
+ String [] t = tokens[x].split("/");
+ posList.add(t[1]); // Filling of POS tags list with available in the training corpus POSs.. ZU UEBERLEGEN: OB DIE NICHT UEBERGEBEN WERDEN ALS PARAMETER
+
+ if (lexicon.containsKey(t[0])){
+ // if the token is already in a dictionary, then get its POS-s
+ Map<String, Double> pos = lexicon.get(t[0]);
+ Double freq=pos.get(t[1]);
+ // if a given POS is already in its values, then add its corresponding count, otherwise add a POS value with a count of 1
+ pos.put(t[1], (freq == null) ? 1 : freq + 1);
+ } else {
+ Map<String, Double> pos = new HashMap<String, Double>();
+ pos.put(t[1],new Double(1));
+ lexicon.put(t[0], pos);
+ }
+ }
+
+ } System.out.println(corpus.size() + " sentences in the corpus");
+ System.out.println(lexicon.get("baby"));
+ return lexicon;
+ }
+
+ /**
+ * Computes {@code word_probs} using {@link #get_lexicon(List)} frequency counts for known words..
+ * TO_DO: ADD SMOOTHING FOR UNKNOWNS?? OR add smoothing directly when come across unknown..
+ */
+
+ @SuppressWarnings("unchecked")
+ public Map<String,Map<String, Double>> get_word_probs(String filename){
+
+
+ Map<String,Map<String, Double>> word_counts=get_lexicon(read_corpus(read_dir(filename)));
+ Map<String, Double> pos_counts = get_ngrams(1);
+
+ Map<String,Map<String, Double>> word_probs = new HashMap<String, Map<String,Double>>();
+
+ int mapsize = word_counts.size();
+ System.out.println(mapsize + " words in the corpus");
+
+ Iterator<Entry<String, Map<String, Double>>> keyValuePairs = word_counts.entrySet().iterator(); // iterate over words
+
+ for (int i = 0; i < mapsize; i++)
+ {
+ Map.Entry<String, Map<String, Double>> entry = (Map.Entry<String, Map<String, Double>>) keyValuePairs.next();
+ Object key = entry.getKey();
+
+ Map<String, Double> pos= word_counts.get(key); // map of possible pos-s of the word
+ Object [] pos_s = pos.entrySet().toArray(); // for iteration over possible pos_s
+
+
+ for (int u = 0; u < pos_s.length; u++)
+ {
+
+ Map.Entry<String, Map<String, Double>> entry2 = (Map.Entry<String, Map<String, Double>>) pos_s[u];
+ // System.out.println(entry);
+ Object key2 = entry2.getKey(); // pos of a word
+ Object value2 = entry2.getValue(); // its count
+ double freq_pos=pos_counts.get(key2);
+ Double val2 = (Double)value2 / freq_pos; // Prob(w|t) = freq(w,t)/freq(t)
+ pos.put((String)key2, val2);
+ System.out.println(value2+":"+freq_pos+"="+
+ val2);
+ }
+ word_probs.put((String)key, pos);
+ System.out.println(key+":"+pos);
+
+ }
+ return word_probs;
+ }
+
+ /**
+ * Computes N-gram frequencies
+ * @param N
+ * @return Map<String, Double> N-grams of parts-of-speech, where {@code N = 1, 2 or 3}
+ * @throws IllegalArgumentException
+ */
+ private Map<String, Double> get_ngrams(int N) throws IllegalArgumentException{
+
+ Map<String, Double> ngrams1= new HashMap<String,Double>();
+ Map<String, Double> ngrams2= new HashMap<String,Double>();
+ Map<String, Double> ngrams3= new HashMap<String,Double>();
+
+ if (N==1){
+ for (int y=0; y<posList.size(); y++){
+ Double freq=ngrams1.get(posList.get(y));
+ // if a given POS is already in the Map, then add its corresponding count, otherwise add a POS value with a count of 1
+ ngrams1.put(posList.get(y), (freq == null) ? 1 : freq + 1);
+ }
+ }
+ else if (N==2){
+ for (int y=0; y<(posList.size()-1); y++){
+ String s2 = posList.get(y)+"_"+posList.get(y+1);
+ Double freq=ngrams2.get(s2);
+ ngrams2.put(s2, (freq == null) ? 1 : freq + 1);
+ }
+ }
+ else if (N==3){
+ for (int y=0; y<(posList.size()-2); y++){
+ String s3 = posList.get(y)+"_"+posList.get(y+1)+"_"+posList.get(y+2);
+ Double freq=ngrams3.get(s3);
+ ngrams3.put(s3, (freq == null) ? 1 : freq + 1);
+ }
+ } else{
+ throw new IllegalArgumentException ("N=1, N=2 or N=3, no further N-grams are supported at the moment");
+ }
+ return ((N==1)? ngrams1: (N == 2) ? ngrams2 : ngrams3);
+ }
+
+ // @SuppressWarnings("unchecked")
+ /**
+ * Computes {@code transition_probs} using {@link #get_ngrams(int)} frequency counts for N-grams..
+ */
+
+ public Map<String, Double> get_transition_probs(int N) throws IllegalArgumentException{
+
+ Map<String, Double> probs2= new HashMap<String, Double>();
+ Map<String, Double> probs3= new HashMap<String, Double>();
+
+ if (N==2){
+
+ Map<String, Double> unigrams = get_ngrams(1);
+ Map<String, Double> bigrams = get_ngrams(2);
+
+ Iterator keyValuePairs = bigrams.entrySet().iterator();
+ for (int i = 0; i < bigrams.size(); i++) // for all bigrams
+ {
+ Map.Entry entry = (Map.Entry) keyValuePairs.next();
+ Object key = entry.getKey(); // get a bigram
+ String [] t = ((String)key).split("_");
+ double freq1 = unigrams.get(t[0]); // get a count of a preceding unigram
+
+ Object freq2 = entry.getValue(); // get a count of a bigram
+
+ double prob2 = (Double)freq2 / freq1; // Prob(t2|t1) = freq(t1,t2)/freq(t1)
+ probs2.put((String)key, prob2);
+ }
+ } else if (N==3){ // for trigram models
+
+ Map<String, Double> bigrams = get_ngrams(2);
+ Map<String, Double> trigrams = get_ngrams(3);
+
+ Iterator keyValuePairs = trigrams.entrySet().iterator();
+ for (int i = 0; i < trigrams.size(); i++) // for all trigrams
+ {
+ Map.Entry entry = (Map.Entry) keyValuePairs.next();
+ Object key = entry.getKey(); // get a trigram
+ String [] t = ((String)key).split("_");
+ String tt = t[0]+"_"+t[1];
+
+ Double freq1 = bigrams.get(tt); // get a count of a preceding bigram
+
+ Object freq2 = entry.getValue(); // get a count of a trigram
+
+ double prob3 = (Double)freq2/freq1; // Prob(t3|(t1_t2)) = freq(t1_t2_t3)/freq(t1_t2)
+ probs3.put((String)key, prob3);
+ }
+ } else{
+ throw new IllegalArgumentException ("only uni-, bi-, and trigramms are supported at the moment");
+ }
+ return ((N == 2) ? probs2 : probs3);
+ }
+
+ /**
+ * Writes the model to a binary file
+ * @param filename output file name
+ */
+
+ public void write_to_file(String filename){
+
+ File file = null;
+ if (filename!=null) { file = new File (filename);}
+ // or use a default file name
+ if (file == null) {
+ System.out.println ("Default: params.dat");
+ file = new File ("params.dat");
+ }
+
+ try {
+ // Create an output stream to the file.
+ FileOutputStream file_output = new FileOutputStream (file);
+ ObjectOutputStream o = new ObjectOutputStream( file_output );
+ o.writeObject(this);
+
+ file_output.close ();
+ }
+ catch (IOException e) {
+ System.err.println ("IO exception = " + e );
+ }
+
+}
+ public Map<String, Map<String, Double>> get_word_probs(List<TokenAnnotation> tokens) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ public List<TokenAnnotation> read_directory(String dir) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+
+ /**
+ * ONLY FOR TEST
+
+ public static void main(String[] args) {
+
+ new ModelGeneration(3, "brown", "brown_model_trigramm.dat");
+ }
+*/
+
+}
+
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/ModelGenerator.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/ModelGenerator.java?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/ModelGenerator.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/ModelGenerator.java Thu Oct 25 02:35:06 2007
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.examples.tagger;
+
+import java.util.*;
+
+import org.apache.uima.TokenAnnotation;
+
+/**
+ * Generates model for tagging (= <i>training procedure</i>)
+ * Model contains probabilities of tokens and corresponding parts of speech:{@code P(word<sub>i</sub>|tag<sub>y</sub>) }
+ * as well as probabilities of n-grams: P(tag<sub>i</sub>|tag<sub>i-1</sub>,tag<sub>i-2</sub>
+ * Supported n-grams at the moment are: bigrams and trigrams
+ *
+ */
+public interface ModelGenerator {
+
+ /**
+ * Reads file names from directory {@code dir}
+ * @param directory name
+ * @return a list of {@code Token}-s
+ */
+
+
+ public List<TokenAnnotation> read_directory(String dir);
+
+ /**
+ * Computes P(word<sub>i</sub>|tag<sub>y</sub>) and saves it in a {@code word_probs} map
+ * @param tokens List of objects of type {@code Token}, containing
+ * @return Map(word<sub>i</sub>, Map((tag<sub>y</sub>, P(word<sub>i</sub>|tag<sub>y</sub>)), (tag<sub>u</sub>, P(word<sub>i</sub>|tag<sub>y</sub>)), etc) }
+ * */
+ public Map<String, Map<String, Double>> get_word_probs(List<TokenAnnotation> tokens);
+
+ /**
+ * Computes {@code transition_probs} map
+ * @param N N=2 for bigram model, N=3 for trigram model
+ * @return Map(N-gram, P(tag<sub>i</sub>|tag<sub>i-1</sub>,tag<sub>i-2</sub>)
+ */
+ public Map<String, Double> get_transition_probs(int N);
+
+ /**
+ * Writes the resulting model to a binary file
+ * @param Outputfilename file where the model should be written to
+ *
+ */
+
+ public void write_to_file(String Outputfilename);
+
+}
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Tagger.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Tagger.java?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Tagger.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Tagger.java Thu Oct 25 02:35:06 2007
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.examples.tagger;
+
+import java.util.List;
+
+/**
+ * known implementations: - HMMTagger using Viterbi algorithm
+ * to compute the most probable path of parts of speech for a given sequence of tokens
+ * @see Viterbi
+ */
+
+public interface Tagger {
+
+ /**
+ * Initiates smoothing procedure for unknown words and N-grams
+ * @return true or false, depending on whether {@code smoothing} is set in a {@code param.txt} file
+ */
+ public boolean set_smoothing();
+
+ /**
+ * Instantiates {@code MODEL} for current tagger
+ */
+
+ public void init();
+
+ /**
+ * Trains a new model for tagger, if a training is defined in {@code tagger.properties} file
+ * @see ModelGenerator
+ */
+
+ public void train();
+
+ /**
+ * Tags a sequence
+ * @param wordList
+ */
+
+ public List process(List wordList);
+
+ /**
+ * Tests tagging accuracy if reference corpus is available
+ */
+ public void test();
+
+}
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/UnknownHandler.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/UnknownHandler.java?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/UnknownHandler.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/UnknownHandler.java Thu Oct 25 02:35:06 2007
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.uima.examples.tagger;
+
+/**
+ * Defines an interface for handling of unknown words or unknown N-grams
+ * @version
+ * known implementations: {@link}UnknownWordHandler
+ */
+
+public interface UnknownHandler {
+
+ /**
+ *
+ */
+
+}
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/UnknownWordHandler.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/UnknownWordHandler.java?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/UnknownWordHandler.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/UnknownWordHandler.java Thu Oct 25 02:35:06 2007
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.examples.tagger;
+
+/**
+ *
+ *
+ */
+public class UnknownWordHandler {
+
+ /**
+ *
+ */
+ public UnknownWordHandler() {
+ // TODO Auto-generated constructor stub
+ }
+
+}
Added: incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Viterbi.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Viterbi.java?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Viterbi.java (added)
+++ incubator/uima/sandbox/trunk/Tagger/src/main/java/org/apache/uima/examples/tagger/Viterbi.java Thu Oct 25 02:35:06 2007
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.uima.examples.tagger;
+
+import java.util.*;
+import java.lang.String;
+
+
+
+/**
+ * Viterbi Algorithm: Given a model and a sequence of observations,
+ * what is the most likely state sequence in the model that produces the observations?
+ */
+public class Viterbi {
+
+
+ public Viterbi() {
+ }
+
+
+ @SuppressWarnings("unchecked")
+ public static Map<String, List> init_probs(String END_OF_SENT_TAG, Map<String, Double> pos_s) {
+ Map<String, List> init_probs = new HashMap<String, List>();
+ List init_list = new ArrayList(3);
+ double p_init = 1.0; // symbolic initial probability, at the moment equal for all tags
+
+ // if (state==END_OF_SENT_TAG){
+ Iterator a_pos= pos_s.entrySet().iterator();
+ double p; // P(tag|preceding_tag, here it is just a symbolic initial state ".")
+ List vpath_list = new ArrayList<String>();
+
+ for (int h= 0; h<pos_s.size(); h++){ // for each of them
+ Map.Entry entry_init = (Map.Entry) a_pos.next();
+ String key_init = (String) entry_init.getKey(); // get the tag (state)
+ Object value_init = entry_init.getValue(); // get its P(w|t)
+ Double p_local = p_init; // get the 'initial' probability of this tag
+
+ vpath_list.add(key_init);
+
+ p = p_local*(Double)value_init; // Prob(this tag|beginning of sequence) x P(w|t)
+
+ init_list.add(p);
+ init_list.add(vpath_list);
+ init_list.add(p);
+ init_probs.put(key_init, init_list);
+ }
+ // }
+ // else {
+ // TODO
+
+ // }
+ return init_probs;
+ }
+
+
+ /**
+ * @param N
+ * @param sentence
+ * @param END_OF_SENT_TAG
+ * @param pos_s
+ * @param transition_probs
+ * @param word_probs
+ * @return
+ */
+ @SuppressWarnings("unchecked")
+ public static List process(int N, List<String> sentence, String END_OF_SENT_TAG, List<String> pos_s, Map<String, Double> transition_probs, Map<String, Map<String, Double>> word_probs)
+ {
+
+ sentence.add(0, END_OF_SENT_TAG);
+
+ /***********************************************************/
+
+ Map<String,List> all = new HashMap<String, List>(); // saves paths to the current token
+
+ /************************************************************/
+ /** **/
+ /************************************************************/
+
+ for (int i=0; i<sentence.size()-1; i++){ // for every token (observation) in a sentence
+
+
+ /*
+ * Get initial probabilities for the first token in a sentence, at the moment we just make them all equal 1.0 through the init_probs() function
+ */
+ Map<String, Double> available_pos = new HashMap<String, Double>();
+ if (i==0) {
+
+ if (word_probs.containsKey(sentence.get(1))){
+ available_pos = word_probs.get(sentence.get(1)); // here we get available states of the first observation
+ }
+ else {
+ // TODO ADD UNKNOWN WORDS HANDLER
+ System.out.println("UNKNOWN");
+ available_pos = word_probs.get("Apache"); // just for test purposes
+ }
+ all.putAll(init_probs(END_OF_SENT_TAG, available_pos));
+ continue; // go over to the next token
+ }
+
+ String token = sentence.get(i);
+ Map<String,Double> possible_pos = new HashMap<String, Double>();
+ if (word_probs.containsKey(sentence.get(i))){
+ possible_pos= word_probs.get(sentence.get(i)); // get possible POS of the current word, otherwise iterate over all, or limited set
+ } else {
+ possible_pos = word_probs.get("Apache"); // just for testing
+ }
+
+ System.out.println("current word: "+token);
+ System.out.println("available POSs of the current token: " +possible_pos);
+ // Iterator keyValuePairs = possible_pos.entrySet().iterator();
+
+ Map<String,List> m = new HashMap<String, List>(); // saves paths to the next token
+
+ /************************************************************/
+ /** INDUCTION (~forward algorithm) **/
+ /************************************************************/
+ /**
+ * At INDUCTION step the algorithm looks at the total and local possibilities of the next state(=tag), given a current observation (=token)
+ * If a token is known, we limit the coming tags by only possible ones for this token
+ */
+
+ Map<String,Double> possible_pos_next = new HashMap<String, Double>(); // possible pos of the next token
+ if (word_probs.containsKey(sentence.get(i+1))){ // if the next token is known
+ System.out.println("next token is: "+sentence.get(i+1)+" and it is in our vocabulary");
+ possible_pos_next = word_probs.get(sentence.get(i+1)); // get possible POS of the next word, otherwise iterate over all, or limited set
+ }
+ else {
+ // TODO ADD UNKNOWN WORDS HANDLER
+ System.err.println("UNKNOWN NEXT WORD");
+ possible_pos_next = word_probs.get("Apache"); // just for test purposes
+ }
+
+ Iterator keyValuePairs_next = possible_pos_next.entrySet().iterator();
+ for (int u = 0; u < possible_pos_next.size(); u++) // for every possible tag of the next token, if the token is known.. otherwise - go through all available POSs
+ {
+
+ double total_prob = 0.0; // just for fun, for forward algorithm
+ List max_viterbi_path = new ArrayList<String>(); // viterbi path
+ double max_viterbi_prob = 0.0; // viterbi probability
+
+ Map.Entry entry_next = (Map.Entry) keyValuePairs_next.next();
+ String key_next = (String) entry_next.getKey();
+ Double value_next = (Double)entry_next.getValue(); // get P(next_token|this_next_state) or P(w_i+1|t_i+i)
+
+ // System.out.println("next possible pos of the next word: "+key_next);
+
+
+ /*
+ * calculates for the given next token which of the possible paths to the current token are more possible to end in the next token
+ */
+ Iterator keyValuePairs = possible_pos.entrySet().iterator();
+ for (int y = 0; y < possible_pos.size(); y++) // for every possible tag of this token (path till now)
+ {
+ Map.Entry entry = (Map.Entry) keyValuePairs.next();
+ Object key = entry.getKey(); // get possible tag for the current token
+ System.out.println("key current: "+key);
+ System.out.println("key next : "+key_next);
+ Double value = (Double)entry.getValue(); // get its P(w|t)
+
+ //
+ List probs = all.get(key);
+ Double prob_local = (Double) probs.get(0); // total_probability for forward algorihtm
+ List path_local = (List) probs.get(1); // path to that token
+
+
+ // int s = path_local.size();
+ Double vprob_local = (Double)probs.get(2); // get viterbi probability for this token
+ String ngram = null;
+
+ /*
+ * The following 4 lines can be modified to get any other number of N-grams
+ */
+
+ /* -- from here -- viterbi_path=viterbi_path.subList(0, path_local.size() */
+
+
+ if (N == 3 && i==1) {ngram = path_local.get(path_local.size()-1)+"_"+key_next; System.out.println("ngram"+ngram); } // um den ersten trigramm abzufangen
+ else if (N==2){ngram = path_local.get(path_local.size()-1)+"_"+key_next; System.out.println("ngram"+ngram);}
+ else if (N==3 && i !=1) { ngram = path_local.get(path_local.size()-2)+"_"+path_local.get(path_local.size()-1)+"_"+key_next;}
+ else { System.err.println("at the moment only bi-and trigramms are supported"); }
+
+ /* -- till here -- */
+
+ double pp=0;
+ // System.out.println(transition_probs.containsKey(ngram));
+ if (transition_probs.containsKey(ngram)) {
+ pp = value_next*transition_probs.get(ngram); //// P(t2|t1) || use logs because of small numbers: log(pq) = log(p) + log(q); if model parameters are stored logged then only addition is performed at runtime
+ // System.out.println("pp: "+pp);
+ } else {
+ System.out.println("unknown ngram");
+
+ // TODO add unknown ngram handler
+ pp = 0.001; // vorläufig, zum testen
+ // System.out.println("pp: "+pp);
+ }
+ prob_local *= pp;
+ vprob_local *=pp;
+ total_prob += prob_local;
+ // System.out.println("P("+sentence.get(i)+"|"+key_next+")*P("+ngram+")="+vprob_local);
+ if(vprob_local>max_viterbi_prob){ // HIER ENTSCHEIDET WELCHE von den möglichen states wird übernehmen
+ max_viterbi_prob = vprob_local;
+
+ max_viterbi_path=new ArrayList(path_local);
+ max_viterbi_path.add(key_next);
+ System.out.println("max viterbi path : "+max_viterbi_path);
+ }
+ }
+
+ List v = new ArrayList(3);
+ v.add(total_prob);
+ v.add(max_viterbi_path);
+ v.add(max_viterbi_prob);
+ m.put(key_next, v); // m contains total of the probabilities ending in this step,
+ // i.e. for every possible pos of the next token we store all the possible paths to it
+
+ }
+ System.out.println("m: "+m);
+ System.out.println("all: "+all);
+ all = m;
+ System.out.println("all: "+all);
+ }
+
+
+
+ /*********************************************************/
+ /* Best sequence identification / Termination / Backtracking
+ /*********************************************************/
+ double total_prob = 0.0; // the overall probability of this sequence of observations, answered by a forward algorithm (braucht man eigentlich nich unbedingt?)
+ List max_viterbi_path = new ArrayList<String>(); // viterbi path
+ double max_viterbi_prob = 0.0; // viterbi probability
+
+ Iterator keyValuePairs_all = all.entrySet().iterator(); // iterate over them
+
+ for (int j = 0; j < all.size(); j++) // for all computed possible paths, chosse the one with the maximum local probability
+ {
+ Map.Entry entry = (Map.Entry) keyValuePairs_all.next();
+ // Object key = entry.getKey();
+ ArrayList value = (ArrayList)entry.getValue();
+
+ Double total_local = (Double) value.get(0);
+ List path_local = (List)value.get(1);
+ Double vprob_local = (Double)value.get(2);
+
+ total_prob += total_local;
+ // System.out.println("P("+token+"|"+key+")*P("+bigram+")="+p);
+ if(vprob_local>max_viterbi_prob){
+ max_viterbi_path=path_local;
+ max_viterbi_prob = vprob_local;
+ }
+
+ }
+ List result = new ArrayList(3);
+ result.add(total_prob);
+ System.out.println("total_prob of this path"+total_prob);
+ result.add(max_viterbi_path);
+
+ result.add(max_viterbi_prob);
+ System.out.println("max viterbi probability of this path"+max_viterbi_prob);
+ System.out.println("path: "+max_viterbi_path);
+ return max_viterbi_path;
+ }
+ }
+
+
+
+
+
Added: incubator/uima/sandbox/trunk/Tagger/tagger.properties
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Tagger/tagger.properties?rev=588179&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Tagger/tagger.properties (added)
+++ incubator/uima/sandbox/trunk/Tagger/tagger.properties Thu Oct 25 02:35:06 2007
@@ -0,0 +1,18 @@
+## This is the default tagger.properties file
+MODEL = brown_model_trigramm.dat
+
+## uncomment the following two lines if training is desirable, and change the corpus directory destination path,
+## e.g. T_CORPUS='brown' for all the files of the brown corpus which are in the 'brown/' -directory
+
+TRAINING = false
+# T_CORPUS = brown
+
+## change the following line to "true" for smoothing
+
+SMOOTHING = false
+
+## N =2 or N= 3 are supported, default N=3
+
+N = 3
+
+END_OF_SENT_TAG = .
\ No newline at end of file