You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by mb...@apache.org on 2007/01/04 12:55:18 UTC
svn commit: r492533 - in /incubator/uima/sandbox/trunk/SnowballAnnotator: ./
desc/ doc/ lib/ metadata/ src/ src/org/ src/org/apache/
src/org/apache/uima/ src/org/apache/uima/annotator/
Author: mbaessler
Date: Thu Jan 4 03:55:17 2007
New Revision: 492533
URL: http://svn.apache.org/viewvc?view=rev&rev=492533
Log:
JIRA ticket UIMA-154 (https://issues.apache.org/jira/browse/UIMA-154)
add new SnowballAnnotator project
Added:
incubator/uima/sandbox/trunk/SnowballAnnotator/desc/
incubator/uima/sandbox/trunk/SnowballAnnotator/desc/SnowballAnnotator.xml
incubator/uima/sandbox/trunk/SnowballAnnotator/desc/aggregate.xml
incubator/uima/sandbox/trunk/SnowballAnnotator/doc/
incubator/uima/sandbox/trunk/SnowballAnnotator/lib/
incubator/uima/sandbox/trunk/SnowballAnnotator/metadata/
incubator/uima/sandbox/trunk/SnowballAnnotator/metadata/install.xml
incubator/uima/sandbox/trunk/SnowballAnnotator/pom.xml
incubator/uima/sandbox/trunk/SnowballAnnotator/src/
incubator/uima/sandbox/trunk/SnowballAnnotator/src/org/
incubator/uima/sandbox/trunk/SnowballAnnotator/src/org/apache/
incubator/uima/sandbox/trunk/SnowballAnnotator/src/org/apache/uima/
incubator/uima/sandbox/trunk/SnowballAnnotator/src/org/apache/uima/annotator/
incubator/uima/sandbox/trunk/SnowballAnnotator/src/org/apache/uima/annotator/SnowballAnnotator.java
Added: incubator/uima/sandbox/trunk/SnowballAnnotator/desc/SnowballAnnotator.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/SnowballAnnotator/desc/SnowballAnnotator.xml?view=auto&rev=492533
==============================================================================
--- incubator/uima/sandbox/trunk/SnowballAnnotator/desc/SnowballAnnotator.xml (added)
+++ incubator/uima/sandbox/trunk/SnowballAnnotator/desc/SnowballAnnotator.xml Thu Jan 4 03:55:17 2007
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ***************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ ***************************************************************
+-->
+
+<analysisEngineDescription
+ xmlns="http://uima.apache.org/resourceSpecifier">
+ <frameworkImplementation>org.apache.uima.java</frameworkImplementation>
+ <primitive>true</primitive>
+ <annotatorImplementationName>org.apache.uima.annotator.SnowballAnnotator</annotatorImplementationName>
+
+ <analysisEngineMetaData>
+ <name>WhitespaceTokenizer</name>
+ <description></description>
+ <version>1.0</version>
+ <vendor></vendor>
+
+ <configurationParameters>
+ </configurationParameters>
+
+ <configurationParameterSettings>
+ </configurationParameterSettings>
+
+ <typeSystemDescription>
+ <typeDescription>
+ <name>org.apache.uima.TokenAnnotation</name>
+ <description>Single token annotation</description>
+ <supertypeName>uima.tcas.Annotation</supertypeName>
+ <features>
+ <featureDescription>
+ <name>stemm</name>
+ <description>stemm</description>
+ <rangeTypeName>uima.cas.String</rangeTypeName>
+ </featureDescription>
+ </features>
+ </typeDescription>
+
+ <typeDescription>
+ <name>org.apache.uima.SentenceAnnotation</name>
+ <description>sentence annotation</description>
+ <supertypeName>uima.tcas.Annotation</supertypeName>
+ <features>
+
+ </features>
+ </typeDescription>
+ </typeSystemDescription>
+
+ <fsIndexes/>
+
+ <capabilities>
+ <capability>
+ <inputs/>
+
+ <outputs>
+ </outputs>
+
+ <languagesSupported>
+ <language>x-unspecified</language>
+ </languagesSupported>
+ </capability>
+ </capabilities>
+
+ </analysisEngineMetaData>
+</analysisEngineDescription>
+
Added: incubator/uima/sandbox/trunk/SnowballAnnotator/desc/aggregate.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/SnowballAnnotator/desc/aggregate.xml?view=auto&rev=492533
==============================================================================
--- incubator/uima/sandbox/trunk/SnowballAnnotator/desc/aggregate.xml (added)
+++ incubator/uima/sandbox/trunk/SnowballAnnotator/desc/aggregate.xml Thu Jan 4 03:55:17 2007
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ***************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ ***************************************************************
+-->
+
+<analysisEngineDescription
+ xmlns="http://uima.apache.org/resourceSpecifier">
+
+ <frameworkImplementation>
+ org.apache.uima.java
+ </frameworkImplementation>
+ <primitive>false</primitive>
+ <delegateAnalysisEngineSpecifiers>
+ <delegateAnalysisEngine key="tokenizer">
+ <import
+ location="../../WhitespaceTokenizer/desc/WhitespaceTokenizer.xml" />
+ </delegateAnalysisEngine>
+ <delegateAnalysisEngine key="snowball">
+ <import location="./SnowballAnnotator.xml" />
+ </delegateAnalysisEngine>
+
+ </delegateAnalysisEngineSpecifiers>
+
+ <analysisEngineMetaData>
+ <name>language detection and tokenization</name>
+ <description>
+ Runs langID annotator and frost or jtok annotator
+ </description>
+ <version>1.0</version>
+ <vendor>IBM Corporation</vendor>
+
+ <configurationParameters />
+
+ <configurationParameterSettings />
+
+ <flowConstraints>
+ <fixedFlow>
+ <node>tokenizer</node>
+ <node>snowball</node>
+ </fixedFlow>
+ </flowConstraints>
+
+ <fsIndexes />
+
+ <capabilities />
+ <operationalProperties>
+ <modifiesCas>true</modifiesCas>
+ <multipleDeploymentAllowed>false</multipleDeploymentAllowed>
+ </operationalProperties>
+
+ </analysisEngineMetaData>
+</analysisEngineDescription>
+
Added: incubator/uima/sandbox/trunk/SnowballAnnotator/metadata/install.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/SnowballAnnotator/metadata/install.xml?view=auto&rev=492533
==============================================================================
--- incubator/uima/sandbox/trunk/SnowballAnnotator/metadata/install.xml (added)
+++ incubator/uima/sandbox/trunk/SnowballAnnotator/metadata/install.xml Thu Jan 4 03:55:17 2007
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<COMPONENT_INSTALLATION_DESCRIPTOR>
+ <OS>
+ </OS>
+ <TOOLKITS>
+ </TOOLKITS>
+ <UIMA_FRAMEWORK>
+ </UIMA_FRAMEWORK>
+ <SUBMITTED_COMPONENT>
+ </SUBMITTED_COMPONENT>
+ <INSTALLATION>
+ </INSTALLATION>
+</COMPONENT_INSTALLATION_DESCRIPTOR>
Added: incubator/uima/sandbox/trunk/SnowballAnnotator/pom.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/SnowballAnnotator/pom.xml?view=auto&rev=492533
==============================================================================
--- incubator/uima/sandbox/trunk/SnowballAnnotator/pom.xml (added)
+++ incubator/uima/sandbox/trunk/SnowballAnnotator/pom.xml Thu Jan 4 03:55:17 2007
@@ -0,0 +1,64 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.uima</groupId>
+ <artifactId>SnowballAnnotator</artifactId>
+ <packaging>jar</packaging>
+ <version>2.1.0-incubating-SNAPSHOT</version>
+ <name>Apache UIMA Java Annotator - Snowball Annotator</name>
+ <url>http://incubator.apache.org/uima</url>
+ <parent>
+ <groupId>org.apache.uima</groupId>
+ <artifactId>uimaj</artifactId>
+ <version>2.1.0-incubating-SNAPSHOT</version>
+ </parent>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.uima</groupId>
+ <artifactId>uimaj-core</artifactId>
+ <version>2.1.0-incubating-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uima</groupId>
+ <artifactId>uimaj-test-util</artifactId>
+ <version>2.1.0-incubating-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uima</groupId>
+ <artifactId>uimaj-component-test-util</artifactId>
+ <version>2.1.0-incubating-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+
+ </dependencies>
+ <build>
+ <finalName>uimaj-an-snowball</finalName>
+ <resources>
+ <resource>
+ <directory>desc</directory>
+ </resource>
+ </resources>
+ </build>
+</project>
Added: incubator/uima/sandbox/trunk/SnowballAnnotator/src/org/apache/uima/annotator/SnowballAnnotator.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/SnowballAnnotator/src/org/apache/uima/annotator/SnowballAnnotator.java?view=auto&rev=492533
==============================================================================
--- incubator/uima/sandbox/trunk/SnowballAnnotator/src/org/apache/uima/annotator/SnowballAnnotator.java (added)
+++ incubator/uima/sandbox/trunk/SnowballAnnotator/src/org/apache/uima/annotator/SnowballAnnotator.java Thu Jan 4 03:55:17 2007
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.uima.annotator;
+
+import java.lang.reflect.Method;
+import java.util.HashMap;
+
+import org.apache.uima.analysis_engine.ResultSpecification;
+import org.apache.uima.analysis_engine.annotator.AnnotatorConfigurationException;
+import org.apache.uima.analysis_engine.annotator.AnnotatorContext;
+import org.apache.uima.analysis_engine.annotator.AnnotatorInitializationException;
+import org.apache.uima.analysis_engine.annotator.AnnotatorProcessException;
+import org.apache.uima.analysis_engine.annotator.JTextAnnotator_ImplBase;
+import org.apache.uima.cas.FSIterator;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.cas.text.Language;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.util.Level;
+import org.apache.uima.util.Logger;
+import org.tartarus.snowball.SnowballProgram;
+import org.tartarus.snowball.ext.danishStemmer;
+import org.tartarus.snowball.ext.dutchStemmer;
+import org.tartarus.snowball.ext.englishStemmer;
+import org.tartarus.snowball.ext.finnishStemmer;
+import org.tartarus.snowball.ext.frenchStemmer;
+import org.tartarus.snowball.ext.germanStemmer;
+import org.tartarus.snowball.ext.hungarianStemmer;
+import org.tartarus.snowball.ext.italianStemmer;
+import org.tartarus.snowball.ext.norwegianStemmer;
+import org.tartarus.snowball.ext.portugueseStemmer;
+import org.tartarus.snowball.ext.russianStemmer;
+import org.tartarus.snowball.ext.spanishStemmer;
+import org.tartarus.snowball.ext.swedishStemmer;
+
+public class SnowballAnnotator extends JTextAnnotator_ImplBase {
+
+ private static final String TOKEN_ANNOTATION_NAME = "org.apache.uima.TokenAnnotation";
+
+ private static final String TOKEN_ANNOTATION_STEMM_FEATURE_NAME = "stemm";
+
+ private Type tokenAnnotation;
+
+ private Feature tokenAnnotationStemmFeature;
+
+ private Logger logger;
+
+ private HashMap stemmers;
+
+ private static final Object[] emptyArgs = new Object[0];
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.uima.analysis_engine.annotator.JTextAnnotator#process(org.apache.uima.jcas.impl.JCas,
+ * org.apache.uima.analysis_engine.ResultSpecification)
+ */
+ public void process(JCas aJCas, ResultSpecification aResultSpec) throws AnnotatorProcessException {
+
+ this.logger.log(Level.INFO, "Snowball annotator starts processing");
+
+ // get get stemmer for the document language
+ String language = new Language(aJCas.getDocumentLanguage()).getLanguagePart();
+ SnowballProgram stemmer = (SnowballProgram) this.stemmers.get(language);
+
+ // create stemms if stemmer for the current document language is available
+ if (stemmer != null) {
+
+ // get stem() method from stemmer
+ Method stemmerStemMethod;
+ try {
+ stemmerStemMethod = stemmer.getClass().getMethod("stem", new Class[0]);
+ } catch (Exception ex) {
+ throw new AnnotatorProcessException(ex);
+ }
+
+ // iterate over all token annotations and add stem if available
+ FSIterator tokenIterator = aJCas.getCas().getAnnotationIndex(this.tokenAnnotation).iterator();
+ while (tokenIterator.hasNext()) {
+ // get token content
+ AnnotationFS annot = (AnnotationFS) tokenIterator.next();
+ String span = annot.getCoveredText();
+
+ // set annotation content and call stemmer
+ try {
+ stemmer.setCurrent(span);
+ stemmerStemMethod.invoke(stemmer, emptyArgs);
+ } catch (Exception ex) {
+ throw new AnnotatorProcessException(ex);
+ }
+
+ // get stemmer result and set annotation feature
+ annot.setStringValue(this.tokenAnnotationStemmFeature, stemmer.getCurrent());
+ }
+ }
+ this.logger.log(Level.INFO, "Snowball annotator processing finished");
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.uima.analysis_engine.annotator.Annotator_ImplBase#initialize(org.apache.uima.analysis_engine.annotator.AnnotatorContext)
+ */
+ public void initialize(AnnotatorContext aContext) throws AnnotatorInitializationException,
+ AnnotatorConfigurationException {
+
+ // initialize logger
+ try {
+ this.logger = aContext.getLogger();
+
+ // initialize stemmers
+ this.stemmers = new HashMap();
+ this.stemmers.put("da", new danishStemmer());
+ this.stemmers.put("nl", new dutchStemmer());
+ this.stemmers.put("en", new englishStemmer());
+ this.stemmers.put("fi", new finnishStemmer());
+ this.stemmers.put("fr", new frenchStemmer());
+ this.stemmers.put("de", new germanStemmer());
+ this.stemmers.put("hu", new hungarianStemmer());
+ this.stemmers.put("it", new italianStemmer());
+ this.stemmers.put("no", new norwegianStemmer());
+ this.stemmers.put("pt", new portugueseStemmer());
+ this.stemmers.put("ru", new russianStemmer());
+ this.stemmers.put("es", new spanishStemmer());
+ this.stemmers.put("sw", new swedishStemmer());
+ } catch (Exception ex) {
+ throw new AnnotatorInitializationException(ex);
+ }
+
+ this.logger.log(Level.INFO, "Snowball annotator successfully initialized");
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.uima.analysis_engine.annotator.Annotator_ImplBase#typeSystemInit(org.apache.uima.cas.TypeSystem)
+ */
+ public void typeSystemInit(TypeSystem aTypeSystem) throws AnnotatorInitializationException,
+ AnnotatorConfigurationException {
+
+ // initialize cas token type
+ this.tokenAnnotation = aTypeSystem.getType(TOKEN_ANNOTATION_NAME);
+
+ this.tokenAnnotationStemmFeature = aTypeSystem.getFeatureByFullName(TOKEN_ANNOTATION_NAME
+ + TypeSystem.FEATURE_SEPARATOR + TOKEN_ANNOTATION_STEMM_FEATURE_NAME);
+
+ this.logger.log(Level.INFO, "Snowball annotator typesystem initialized");
+ }
+
+}