You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2012/09/19 10:48:39 UTC
svn commit: r1387488 [2/5] - in
/incubator/stanbol/branches/stanbol-nlp-processing: ./ data/
data/bundlelists/sentiment/ data/bundlelists/sentiment/src/
data/bundlelists/sentiment/src/main/
data/bundlelists/sentiment/src/main/bundles/ data/opennlp/lang...
Added: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/ChunkingEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/ChunkingEngine.java?rev=1387488&view=auto
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/ChunkingEngine.java (added)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/ChunkingEngine.java Wed Sep 19 08:48:32 2012
@@ -0,0 +1,495 @@
+/*
+ * Copyright (c) 2012 Sebastian Schaffert
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stanbol.enhancer.engines.opennlp.chunker.services;
+
+import static org.apache.stanbol.enhancer.nlp.NlpAnnotations.POSAnnotation;
+import static org.apache.stanbol.enhancer.nlp.NlpAnnotations.phraseAnnotation;
+import static org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper.getAnalysedText;
+import static org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper.getLanguage;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Dictionary;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import opennlp.tools.chunker.ChunkerME;
+import opennlp.tools.chunker.ChunkerModel;
+import opennlp.tools.util.Sequence;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.ConfigurationPolicy;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Properties;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.Service;
+import org.apache.stanbol.commons.opennlp.OpenNLP;
+import org.apache.stanbol.enhancer.engines.opennlp.chunker.model.PhraseTagSetRegistry;
+import org.apache.stanbol.enhancer.nlp.NlpAnnotations;
+import org.apache.stanbol.enhancer.nlp.TagSet;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedText;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedTextUtils;
+import org.apache.stanbol.enhancer.nlp.model.Chunk;
+import org.apache.stanbol.enhancer.nlp.model.Section;
+import org.apache.stanbol.enhancer.nlp.model.Span;
+import org.apache.stanbol.enhancer.nlp.model.Token;
+import org.apache.stanbol.enhancer.nlp.model.Span.SpanTypeEnum;
+import org.apache.stanbol.enhancer.nlp.model.annotation.Value;
+import org.apache.stanbol.enhancer.nlp.phrase.PhraseTag;
+import org.apache.stanbol.enhancer.nlp.pos.PosTag;
+import org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.enhancer.servicesapi.EngineException;
+import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
+import org.apache.stanbol.enhancer.servicesapi.impl.AbstractEnhancementEngine;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A noun phrase detector (chunker) for English and German language base on OpenNLP. Uses the following chunker
+ * models for OpenNLP:
+ * <ul>
+ * <li>English: http://opennlp.sourceforge.net/models-1.5/en-chunker.bin</li>
+ * <li>German: http://gromgull.net/blog/2010/01/noun-phrase-chunking-for-the-awful-german-language/</li>
+ * </ul>
+ * The noun phrase detector requires a {@link org.apache.stanbol.enhancer.engines.opennlp.pos.model.POSContentPart} to
+ * be present in the content item and will extend each {@link org.apache.stanbol.enhancer.engines.opennlp.pos.model.POSSentence}
+ * with an array of chunks.
+ * <p/>
+ * Author: Sebastian Schaffert
+ */
+@Component(immediate = true, metatype = true, configurationFactory = true, policy = ConfigurationPolicy.REQUIRE)
+@Service
+@Properties(value={
+ @Property(name= EnhancementEngine.PROPERTY_NAME,value="chunker")
+})
+public class ChunkingEngine extends AbstractEnhancementEngine<RuntimeException,RuntimeException> {
+
+ /**
+ * Language configuration. Takes a list of ISO language codes of supported languages. Currently supported
+ * are the languages given as default value.
+ */
+ @Property(value = {"de;OpenNLP_1.5.1-German-Chunker-TigerCorps07.zip","*"})
+ public static final String CONFIG_LANGUAGES = "org.apache.stanbol.enhancer.chunker.languages";
+
+ @Property
+ public static final String MIN_CHUNK_SCORE = "org.apache.stanbol.enhancer.chunker.minScore";
+
+ public static final String[] AVAILABLE_LANGUAGES = new String[] {"en","de"};
+
+
+ private static Logger log = LoggerFactory.getLogger(ChunkingEngine.class);
+
+
+ private Double minChunkScore;
+
+ @Reference
+ private OpenNLP openNLP;
+
+ /**
+ * The registry used to lookup predefined {@link TagSet}s
+ */
+ private PhraseTagSetRegistry tagSetRegistry = PhraseTagSetRegistry.getInstance();
+
+ /**
+ * Holds as key explicitly enabled languages and as value the name of the
+ * OpenNLP model used for Chunking. If the value is <code>null</code> this
+ * indicates that the default model (
+ * provided by {@link OpenNLP#getChunkerModel(String)}) will be used.<p>
+ * NOTE: a configured language does not automatically mean that also the
+ * requested model is available.
+ */
+ private Map<String,String> configuredLanguages;
+ /**
+ * Languages that are explicitly excluded
+ */
+ private Set<String> excludedLanguages;
+ /**
+ * if '*' is used as language configuration
+ */
+ private boolean allowAll;
+
+
+ /**
+ * Indicate if this engine can enhance supplied ContentItem, and if it
+ * suggests enhancing it synchronously or asynchronously. The
+ * {@link org.apache.stanbol.enhancer.servicesapi.EnhancementJobManager} can force sync/async mode if desired, it is
+ * just a suggestion from the engine.
+ * <p/>
+ * Returns CANNOT_ENHANCE if the content item does not have a POSContentPart, the language of the content is not
+ * available or no chunker for the language is available.
+ *
+ * @throws org.apache.stanbol.enhancer.servicesapi.EngineException
+ * if the introspecting process of the content item
+ * fails
+ */
+ @Override
+ public int canEnhance(ContentItem ci) throws EngineException {
+ String language = getLanguage(this, ci,false);
+ if(language == null){
+ return CANNOT_ENHANCE;
+ }
+ if(!isLangaugeConfigured(language,false)){
+ return CANNOT_ENHANCE;
+ }
+ if(getAnalysedText(this,ci,false) == null) {
+ return CANNOT_ENHANCE;
+ }
+
+ // default enhancement is synchronous enhancement
+ return ENHANCE_ASYNC;
+
+ }
+
+ /**
+ * Compute enhancements for supplied ContentItem. The results of the process
+ * are expected to be stored in the metadata of the content item.
+ * <p/>
+ * The client (usually an {@link org.apache.stanbol.enhancer.servicesapi.EnhancementJobManager}) should take care of
+ * persistent storage of the enhanced {@link org.apache.stanbol.enhancer.servicesapi.ContentItem}.
+ *
+ * @throws org.apache.stanbol.enhancer.servicesapi.EngineException
+ * if the underlying process failed to work as
+ * expected
+ */
+ @Override
+ public void computeEnhancements(ContentItem ci) throws EngineException {
+ AnalysedText at = getAnalysedText(this, ci, true);
+ String language = getLanguage(this, ci, true);
+ ChunkerME chunker = initChunker(language);
+ if(chunker == null){
+ return;
+ }
+ //init the Phrase TagSet
+ TagSet<PhraseTag> tagSet = tagSetRegistry.getTagSet(language);
+ if(tagSet == null){
+ }
+ if(tagSet == null){
+ log.warn("No Phrase TagSet registered for Language '{}'. Will build an "
+ + "adhoc set based on encountered Tags!",language);
+ //for now only created to avoid checks for tagSet == null
+ //TODO: in future we might want to automatically create posModels based
+ //on tagged texts. However this makes no sense as long we can not
+ //persist TagSets.
+ tagSet = new TagSet<PhraseTag>("dummy", language);
+ }
+ //holds PosTags created for POS tags that where not part of the posModel
+ //(will hold all PosTags in case tagSet is NULL
+ Map<String,PhraseTag> adhocTags = new HashMap<String,PhraseTag>();
+
+ ci.getLock().writeLock().lock();
+ try {
+ Iterator<? extends Section> sentences = at.getSentences();
+ if(!sentences.hasNext()){ //no sentences ... iterate over the whole text
+ sentences = Collections.singleton(at).iterator();
+ }
+ List<String> tokenTextList = new ArrayList<String>(64);
+ List<String> posList = new ArrayList<String>(64);
+ List<Token> tokenList = new ArrayList<Token>(64);
+ //process each sentence seperatly
+ while(sentences.hasNext()){
+ // (1) get Tokens and POS information for the sentence
+ Section sentence = sentences.next();
+ Iterator<Token> tokens = sentence.getTokens();
+ while(tokens.hasNext()){
+ Token token = tokens.next();
+ tokenList.add(token);
+ tokenTextList.add(token.getSpan());
+ Value<PosTag> posValue = token.getAnnotation(POSAnnotation);
+ if(posValue == null){
+ throw new EngineException("Missing POS value for Token '"
+ + token.getSpan()+" of ContentItem "+ci.getUri()
+ + "(Sentence: '"+sentence.getSpan()+"'). This may "
+ + "indicate that a POS tagging Engine is missing in "
+ + "the EnhancementChain or that the used POS tagging "
+ + "does not provide POS tags for each token!");
+ } else {
+ posList.add(posValue.value().getTag());
+ }
+ }
+ String[] tokenStrings = tokenTextList.toArray(new String[tokenTextList.size()]);
+ String[] tokenPos = posList.toArray(new String[tokenTextList.size()]);
+ log.info("Tokens: {}"+Arrays.toString(tokenStrings));
+ tokenTextList.clear(); //free memory
+ posList.clear(); //free memory
+
+ // (2) Chunk the sentence
+
+ String[] chunkTags = chunker.chunk(tokenStrings, tokenPos);
+ double[] chunkProb = chunker.probs();
+ log.info("Chunks: {}"+Arrays.toString(chunkTags));
+ tokenStrings = null; //free memory
+ tokenPos = null; //free memory
+
+ // (3) Process the results and write the Annotations
+ double chunkProps = 0;
+ int chunkTokenCount = 0;
+ PhraseTag tag = null;
+ int i;
+ /*
+ * This assumes:
+ * - 'B-{tag}' ... for start of a new chunk
+ * - '???' ... anything other for continuing the current chunk
+ * - 'O' ... no chunk (ends current chunk)
+ */
+ for(i=0;i<tokenList.size();i++){
+ boolean start = chunkTags[i].charAt(0) == 'B';
+ boolean end = tag != null && (start || chunkTags[i].charAt(0) == 'O');
+ if(end){ //add the current phrase
+ //add at AnalysedText level, because offsets are absolute
+ //NOTE we are already at the next token when we detect the end
+ Chunk chunk = at.addChunk(
+ tokenList.get(i-chunkTokenCount).getStart(),
+ tokenList.get(i-1).getEnd());
+ chunk.addAnnotation(phraseAnnotation,
+ new Value<PhraseTag>(tag,
+ chunkProps/(double)chunkTokenCount));
+ //reset the state
+ tag = null;
+ chunkTokenCount = 0;
+ chunkProps = 0;
+ }
+ if(start){ //create the new tag
+ tag = getPhraseTag(tagSet,adhocTags,
+ chunkTags[i].substring(2)); //skip 'B-'
+
+ }
+ if(tag != null){ //count this token for the current chunk
+ chunkProps = chunkProps + chunkProb[i];
+ chunkTokenCount++;
+ }
+ }
+ if(tag != null){
+ Chunk chunk = at.addChunk(
+ tokenList.get(i-chunkTokenCount).getStart(),
+ tokenList.get(i-1).getEnd());
+ chunk.addAnnotation(phraseAnnotation,
+ new Value<PhraseTag>(tag,
+ chunkProps/(double)chunkTokenCount));
+
+ }
+ // (4) clean up
+ tokenList.clear();
+ }
+
+ } finally {
+ ci.getLock().writeLock().unlock();
+ }
+ logChunks(at);
+ }
+
+ private void logChunks(AnalysedText at){
+ Iterator<Span> it = at.getEnclosed(EnumSet.of(SpanTypeEnum.Sentence, SpanTypeEnum.Chunk));
+ while(it.hasNext()){
+ Span span = it.next();
+ if(span.getType() == SpanTypeEnum.Chunk){
+ log.info(" > {} {}",span,span.getSpan());
+ } else {
+ log.info(" > {}",span);
+ }
+ for(Value<PhraseTag> value : span.getAnnotations(phraseAnnotation)){
+ log.info(" - {}",value);
+ }
+ }
+ }
+
+ private PhraseTag getPhraseTag(TagSet<PhraseTag> model, Map<String,PhraseTag> adhocTags, String tag) {
+ PhraseTag phraseTag = model.getTag(tag);
+ if(phraseTag != null){
+ return phraseTag;
+ }
+ phraseTag = adhocTags.get(tag);
+ if(phraseTag != null){
+ return phraseTag;
+ }
+ phraseTag = new PhraseTag(tag);
+ adhocTags.put(tag, phraseTag);
+ log.warn("Encountered unknown POS tag '{}' for langauge '{}'",tag,model.getLanguages());
+ return phraseTag;
+ }
+
+ /**
+ * Activate and read the properties. Configures and initialises a ChunkerHelper for each language configured in
+ * CONFIG_LANGUAGES.
+ *
+ * @param ce the {@link org.osgi.service.component.ComponentContext}
+ */
+ @Activate
+ protected void activate(ComponentContext ce) throws ConfigurationException {
+ log.info("activating POS tagging engine");
+ super.activate(ce);
+ @SuppressWarnings("unchecked")
+ Dictionary<String, Object> properties = ce.getProperties();
+
+ //read the min chunk score
+ Object value = properties.get(MIN_CHUNK_SCORE);
+ Double minScore;
+ if(value instanceof Number){
+ minScore = ((Number)value).doubleValue();
+ } else if (value != null && !value.toString().isEmpty()){
+ try {
+ minScore = Double.parseDouble(value.toString());
+ } catch (NumberFormatException e) {
+ throw new ConfigurationException(MIN_CHUNK_SCORE,
+ "The configured minumum chunk score MUST BE a floating point"
+ + "number in the range > 0 < 1. Values >= 0 will deactivate "
+ + "this feature.",e);
+ }
+ } else {
+ minScore = null;
+ }
+ if(minScore != null && minScore.doubleValue() >= 1d){
+ throw new ConfigurationException(MIN_CHUNK_SCORE,
+ "The configured minumum chunk score '"+minScore+"' MUST BE a "
+ + "floating point number in the range > 0 < 1. Values >= 0 will "
+ + "deactivate this feature.");
+ } else if(minScore == null || minScore.doubleValue() <= 0){
+ this.minChunkScore = null;
+ } else {
+ log.info(" > set minimum chunk score to {} (Engine: {})",
+ minScore, getName());
+ this.minChunkScore = minScore;
+ }
+
+ //read the language configuration
+ configuredLanguages = new HashMap<String,String>();
+ excludedLanguages = new HashSet<String>();
+ allowAll = false;
+ if(properties.get(CONFIG_LANGUAGES) != null) {
+ String[] languages = (String[])properties.get(CONFIG_LANGUAGES);
+
+ for(String lang : languages) {
+ String modelName;
+ int seperatorIndex = lang.indexOf(';');
+ if(seperatorIndex >= 0){
+ if(seperatorIndex <lang.length()-2){
+ modelName = lang.substring(seperatorIndex+1).trim();
+ } else {
+ modelName = null;
+ }
+ lang = lang.substring(0, seperatorIndex).trim();
+ } else {
+ modelName = null;
+ }
+ if(lang.charAt(0) == '!'){ //exclude
+ lang = lang.substring(1);
+ if(configuredLanguages.containsKey(lang)){
+ throw new ConfigurationException(CONFIG_LANGUAGES,
+ "Langauge '"+lang+"' is both included and excluded (config: "
+ + Arrays.toString(languages)+"");
+ }
+ excludedLanguages.add(lang);
+ if(modelName != null){
+ log.warn("Parsed model names are ignored for excluded languages "
+ + "(langauge: {}, modelName: {})!", lang,modelName);
+ }
+ } else if("*".equals(lang)){
+ allowAll = true;
+ if(modelName != null){
+ log.warn("A parsed model name is ignored for the wildcard "
+ + "langauge (modelName: {})!", lang,modelName);
+ }
+ } else if(!lang.isEmpty()){
+ if(excludedLanguages.contains(lang)){
+ throw new ConfigurationException(CONFIG_LANGUAGES,
+ "Langauge '"+lang+"' is both included and excluded (config: "
+ + Arrays.toString(languages)+"");
+ }
+ configuredLanguages.put(lang,modelName);
+ }
+ }
+ } else {
+ allowAll = true;
+ }
+ }
+
+ @Deactivate
+ protected void deactivate(ComponentContext context){
+ this.allowAll = false;
+ this.configuredLanguages = null;
+ this.excludedLanguages = null;
+ this.minChunkScore = null;
+ super.deactivate(context);
+ }
+
+
+ /**
+ * Used in {@link #canEnhance(ContentItem)} to check if a {@link ContentItem}
+ * should be processed based on the language configuration of this engine.
+ * @param language the language
+ * @param exception <code>false</code> id used in {@link #canEnhance(ContentItem)}
+ * and <code>true</code> when called from {@link #computeEnhancements(ContentItem)}
+ * @return the state
+ * @throws IllegalStateException if exception is <code>true</code> and the
+ * language is not configured as beeing processed.
+ */
+ boolean isLangaugeConfigured(String language, boolean exception){
+ boolean state = allowAll ?
+ (!excludedLanguages.contains(language)) :
+ configuredLanguages.containsKey(language);
+ if(!state && exception){
+ throw new IllegalStateException("Language "+language+" is not included "
+ + "by the LanguageConfiguration of this engine (name "+ getName()
+ + "). As this is also checked in canEnhancer this may indicate an Bug in the "
+ + "used EnhancementJobManager!");
+ } else {
+ return state;
+ }
+ }
+
+ private ChunkerME initChunker(String language) {
+ isLangaugeConfigured(language, true); //check if the parsed language is ok
+ String modelName = configuredLanguages.get(language);
+ ChunkerModel model;
+ try {
+ if(modelName == null){ // the default model
+ model = openNLP.getChunkerModel(language);
+ } else {
+ model = openNLP.getModel(ChunkerModel.class, modelName, null);
+ }
+ }catch (IOException e) {
+ log.warn("Unable to load Chunker model for language '"+language
+ + "' (model: "+(modelName == null ? "default" : modelName)+")",
+ e);
+ return null;
+ } catch (RuntimeException e){
+ log.warn("Error while creating ChunkerModel for language '"+language
+ + "' (model: "+(modelName == null ? "default" : modelName)+")",
+ e);
+ return null;
+ }
+ if(model == null){
+ log.trace("no Chunker Model for language {}",language);
+ return null;
+ } else {
+ return new ChunkerME(model);
+ }
+ }
+
+}
Added: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-chunker/src/main/resources/OSGI-INF/metatype/metatype.properties
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-chunker/src/main/resources/OSGI-INF/metatype/metatype.properties?rev=1387488&view=auto
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-chunker/src/main/resources/OSGI-INF/metatype/metatype.properties (added)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-chunker/src/main/resources/OSGI-INF/metatype/metatype.properties Wed Sep 19 08:48:32 2012
@@ -0,0 +1,27 @@
+#
+# Copyright 2012, FORMCEPT [http://www.formcept.com]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.stanbol.enhancer.engines.opennlp.chunker.services.ChunkingEngine.name=Apache Stanbol Enhancer Engine: Chunking / Noun Phrase Detection
+
+stanbol.enhancer.engine.name.name=Noun Phrase Detector
+stanbol.enhancer.engine.name.description=The name of the enhancement engine as \
+used in the RESTful interface '/engine/<name>'
+service.ranking.name=Ranking
+service.ranking.description=If two enhancement engines with the same name are active the \
+one with the higher ranking will be used to process parsed content items.
+
+org.apache.stanbol.enhancer.chunker.languages.name=Language configuration
+org.apache.stanbol.enhancer.chunker.languages.description=Takes a list of ISO \
+ language codes of supported languages. Currently supported are the languages given as default value.
Added: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-chunker/src/main/resources/config/org.apache.stanbol.enhancer.engines.chunker.services.ChunkingEngine-snml.config
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-chunker/src/main/resources/config/org.apache.stanbol.enhancer.engines.chunker.services.ChunkingEngine-snml.config?rev=1387488&view=auto
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-chunker/src/main/resources/config/org.apache.stanbol.enhancer.engines.chunker.services.ChunkingEngine-snml.config (added)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-chunker/src/main/resources/config/org.apache.stanbol.enhancer.engines.chunker.services.ChunkingEngine-snml.config Wed Sep 19 08:48:32 2012
@@ -0,0 +1,2 @@
+stanbol.enhancer.chain.name="person-enhancer"
+stanbol.enhancer.chain.list.enginelist=["tika","langid","snml-person-enhancer"]
Propchange: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Wed Sep 19 08:48:32 2012
@@ -0,0 +1,7 @@
+.project
+
+target
+
+.classpath
+
+.settings
Added: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/pom.xml?rev=1387488&view=auto
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/pom.xml (added)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/pom.xml Wed Sep 19 08:48:32 2012
@@ -0,0 +1,101 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>org.apache.stanbol.enhancer.parent</artifactId>
+ <groupId>org.apache.stanbol</groupId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <relativePath>../../parent</relativePath>
+ </parent>
+
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.opennlp.pos</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <packaging>bundle</packaging>
+
+ <name>Apache Stanbol Enhancer Enhancement Engine: POS Tagging</name>
+ <description>A Stanbol engine adding a POS content part to a content item for further processing</description>
+
+ <inceptionYear>2012</inceptionYear>
+
+ <scm>
+ <connection>
+ scm:svn:http://svn.apache.org/repos/asf/incubator/stanbol/trunk/enhancer/engines/opennlp-pos/
+ </connection>
+ <developerConnection>
+ scm:svn:https://svn.apache.org/repos/asf/incubator/stanbol/trunk/enhancer/engines/opennlp-pos/
+ </developerConnection>
+ <url>http://incubator.apache.org/stanbol/</url>
+ </scm>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <!-- Enable this for including your enhancement chain configuration -->
+ <!-- <Install-Path>config</Install-Path> -->
+ <Export-Package>
+ org.apache.stanbol.enhancer.engines.opennlp.pos.model;version=${project.version}
+ </Export-Package>
+ <Private-Package>
+ org.apache.stanbol.enhancer.engines.opennlp.pos.services;version=${project.version}
+ </Private-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <!-- AL20 License -->
+ <exclude>src/license/THIRD-PARTY.properties</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-scr-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.servicesapi</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.commons.opennlp</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.nlp</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>org.apache.felix.scr.annotations</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
Added: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/model/PosTagSetRegistry.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/model/PosTagSetRegistry.java?rev=1387488&view=auto
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/model/PosTagSetRegistry.java (added)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/model/PosTagSetRegistry.java Wed Sep 19 08:48:32 2012
@@ -0,0 +1,240 @@
+package org.apache.stanbol.enhancer.engines.opennlp.pos.model;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.stanbol.commons.opennlp.PosTagsCollectionEnum;
+import org.apache.stanbol.enhancer.nlp.TagSet;
+import org.apache.stanbol.enhancer.nlp.pos.LexicalCategory;
+import org.apache.stanbol.enhancer.nlp.pos.PosTag;
+import org.apache.stanbol.enhancer.nlp.pos.olia.English;
+import org.apache.stanbol.enhancer.nlp.pos.olia.German;
+import org.apache.stanbol.enhancer.nlp.pos.olia.Spanish;
+
+/**
+ * {@link TagSet}s for known <a herf="http://opennlp.apache.org/">OpenNLP</a>
+ * POS models.<p>
+ * When available this refers to models defined by the
+ * <a herf="http://nlp2rdf.lod2.eu/olia/">OLIA</a> Ontologies. Other TagSets
+ * are - for now - directly defined in this class.
+ * <p>
+ * Specifications in this class are based on {@link PosTagsCollectionEnum}.
+ * Links/defines to the POS {@link TagSet}s used by
+ *
+ * @author Rupert Westenthaler
+ *
+ */
+public final class PosTagSetRegistry {
+
+ private static PosTagSetRegistry instance = new PosTagSetRegistry();
+
+ private PosTagSetRegistry(){}
+
+ private final Map<String, TagSet<PosTag>> models = new HashMap<String,TagSet<PosTag>>();
+
+ public static PosTagSetRegistry getInstance(){
+ return instance;
+ }
+
+ private void add(TagSet<PosTag> model) {
+ for(String lang : model.getLanguages()){
+ if(models.put(lang, model) != null){
+ throw new IllegalStateException("Multiple Models for Language '"
+ + lang+"'! This is an error in the static confituration of "
+ + "this class. Please report this to the stanbol-dev mailing"
+ + "list!");
+ }
+ }
+ }
+ /**
+ * Getter for the {@link TagSet} by language. If no {@link TagSet}
+ * is available for an Language this will return <code>null</code>
+ * @param language the language
+ * @return the AnnotationModel or <code>null</code> if non is defined
+ */
+ public TagSet<PosTag> getTagSet(String language){
+ return models.get(language);
+ }
+
+ /**
+ * Links to the Penn Treebank model as defined by the
+ * <a herf="http://nlp2rdf.lod2.eu/olia/">OLIA</a> Ontology.
+ * @see English#PENN_TREEBANK
+ */
+ public static final TagSet<PosTag> ENGLISH = English.PENN_TREEBANK;
+
+ static { //adds the English model to the getInstance()
+ getInstance().add(ENGLISH);
+ }
+ /**
+ * Links to the STTS model as defined by the
+ * <a herf="http://nlp2rdf.lod2.eu/olia/">OLIA</a> Ontology.
+ * @see German#STTS
+ */
+ public static final TagSet<PosTag> GERMAN = German.STTS;
+
+ static { //adds the English model to the getInstance()
+ getInstance().add(GERMAN);
+ }
+ /**
+ * Links to the PAROLE model as defined by the
+ * <a herf="http://nlp2rdf.lod2.eu/olia/">OLIA</a> Ontology.
+ * @see Spanish#PAROLE
+ */
+ public static final TagSet<PosTag> SPANISH = Spanish.PAROLE;
+
+ static { //adds the Spanish model to the getInstance()
+ getInstance().add(SPANISH);
+ }
+ /**
+ * POS types representing Nouns for Danish based on the PAROLE Tagset as
+ * described by <a href="http://korpus.dsl.dk/paroledoc_en.pdf">this paper</a>
+ */
+ public static final TagSet<PosTag> DANISH = new TagSet<PosTag>("PAROLE Danish","da");
+
+ static {
+ DANISH.addTag(new PosTag("N",LexicalCategory.Noun));
+ DANISH.addTag(new PosTag("NP",LexicalCategory.Noun));
+ DANISH.addTag(new PosTag("NC",LexicalCategory.Noun));
+ DANISH.addTag(new PosTag("AC",LexicalCategory.Quantifier)); //numbers
+ DANISH.addTag(new PosTag("AO",LexicalCategory.Quantifier)); //numbers
+ DANISH.addTag(new PosTag("XX",LexicalCategory.Noun)); //unsure
+ DANISH.addTag(new PosTag("XF",LexicalCategory.Noun)); //foreign word
+ DANISH.addTag(new PosTag("XR",LexicalCategory.Quantifier)); //number letters
+ DANISH.addTag(new PosTag("XR",LexicalCategory.Quantifier)); //symbol letters
+ DANISH.addTag(new PosTag("XA",LexicalCategory.Noun)); //abbreviations
+ DANISH.addTag(new PosTag("XX",LexicalCategory.Quantifier)); //tokenizer errors
+ DANISH.addTag(new PosTag("V",LexicalCategory.Verb));
+ DANISH.addTag(new PosTag("VA",LexicalCategory.Verb));
+ DANISH.addTag(new PosTag("VAD",LexicalCategory.Verb));
+ DANISH.addTag(new PosTag("VAF",LexicalCategory.Verb));
+ DANISH.addTag(new PosTag("VAG",LexicalCategory.Verb));
+ DANISH.addTag(new PosTag("VAPR",LexicalCategory.Verb));
+ DANISH.addTag(new PosTag("VAPA",LexicalCategory.Verb));
+ DANISH.addTag(new PosTag("VE",LexicalCategory.Verb));
+ DANISH.addTag(new PosTag("VED",LexicalCategory.Verb));
+ DANISH.addTag(new PosTag("VEF",LexicalCategory.Verb));
+ DANISH.addTag(new PosTag("XP",LexicalCategory.Punctuation));
+ DANISH.addTag(new PosTag("CC",LexicalCategory.Conjuction));
+ DANISH.addTag(new PosTag("CS",LexicalCategory.Conjuction));
+ DANISH.addTag(new PosTag("U",LexicalCategory.Noun)); //unknown tokens
+ DANISH.addTag(new PosTag("SP",LexicalCategory.Adposition));
+ DANISH.addTag(new PosTag("AN",LexicalCategory.Adjective)); //unsure
+ DANISH.addTag(new PosTag("R",LexicalCategory.Adverb)); //unsure
+ DANISH.addTag(new PosTag("RG",LexicalCategory.Adverb)); //unsure
+ DANISH.addTag(new PosTag("PD",LexicalCategory.PronounOrDeterminer)); //unsure
+ DANISH.addTag(new PosTag("PI",LexicalCategory.PronounOrDeterminer)); //unsure
+ DANISH.addTag(new PosTag("PT",LexicalCategory.PronounOrDeterminer)); //unsure
+ DANISH.addTag(new PosTag("PP",LexicalCategory.PronounOrDeterminer)); //unsure
+ DANISH.addTag(new PosTag("PO",LexicalCategory.PronounOrDeterminer)); //unsure
+ DANISH.addTag(new PosTag("PC",LexicalCategory.PronounOrDeterminer)); //unsure
+ DANISH.addTag(new PosTag("U=",LexicalCategory.Unique)); //unsure
+ DANISH.addTag(new PosTag("I=",LexicalCategory.Interjection)); //unsure
+ getInstance().add(DANISH);
+ }
+
+ /**
+ * POS tags for the Portuguese POS model of OpenNLP based the
+ * <a href="http://beta.visl.sdu.dk/visl/pt/symbolset-floresta.html">PALAVRAS tag set</a>
+ * <p>
+ */
+ public static final TagSet<PosTag> PORTUGUESE = new TagSet<PosTag>("PALAVRAS Portuguese","pt");
+
+ static {
+ PORTUGUESE.addTag(new PosTag("n",LexicalCategory.Noun));
+ PORTUGUESE.addTag(new PosTag("prop",LexicalCategory.Noun));
+ PORTUGUESE.addTag(new PosTag("v-fin",LexicalCategory.Verb));
+ PORTUGUESE.addTag(new PosTag("v-inf",LexicalCategory.Verb));
+ PORTUGUESE.addTag(new PosTag("v-pcp",LexicalCategory.Verb));
+ PORTUGUESE.addTag(new PosTag("v-ger",LexicalCategory.Verb));
+ PORTUGUESE.addTag(new PosTag("art",LexicalCategory.PronounOrDeterminer));
+ PORTUGUESE.addTag(new PosTag("pron-pers",LexicalCategory.PronounOrDeterminer));
+ PORTUGUESE.addTag(new PosTag("pron-det",LexicalCategory.PronounOrDeterminer));
+ PORTUGUESE.addTag(new PosTag("pron-indp",LexicalCategory.PronounOrDeterminer));
+ PORTUGUESE.addTag(new PosTag("adv",LexicalCategory.Adverb));
+ PORTUGUESE.addTag(new PosTag("num",LexicalCategory.Quantifier));
+ PORTUGUESE.addTag(new PosTag("prp",LexicalCategory.Adposition));
+ PORTUGUESE.addTag(new PosTag("in",LexicalCategory.Interjection));
+ PORTUGUESE.addTag(new PosTag("conj-s",LexicalCategory.Conjuction));
+ PORTUGUESE.addTag(new PosTag("conj-c",LexicalCategory.Conjuction));
+ PORTUGUESE.addTag(new PosTag("punc",LexicalCategory.Punctuation)); //missing on the webpage ^
+ getInstance().add(PORTUGUESE);
+ }
+ /**
+ * POS tags used by the Dutch POS model of OpenNLP for Dutch.<p>
+ * Source: J.T. Berghmans, "WOTAN: Een automatische grammatikale tagger
+ * voor het Nederlands", doctoral dissertation, Department of language &
+ * Speech, Nijmegen University (renamed to Radboud University),
+ * december 1994.<p>
+ *
+ */
+ public static final TagSet<PosTag> DUTCH = new TagSet<PosTag>("WOTAN Dutch","nl");
+
+ static {
+ DUTCH.addTag(new PosTag("Adj",LexicalCategory.Adjective));
+ DUTCH.addTag(new PosTag("Adv",LexicalCategory.Adverb));
+ DUTCH.addTag(new PosTag("Art",LexicalCategory.PronounOrDeterminer));
+ DUTCH.addTag(new PosTag("Conj",LexicalCategory.Conjuction));
+ DUTCH.addTag(new PosTag("Int",LexicalCategory.Interjection));
+ DUTCH.addTag(new PosTag("N",LexicalCategory.Noun));
+ DUTCH.addTag(new PosTag("Num",LexicalCategory.Quantifier));
+ DUTCH.addTag(new PosTag("Misc",null));
+ DUTCH.addTag(new PosTag("Prep",LexicalCategory.Adposition));
+ DUTCH.addTag(new PosTag("Pron",LexicalCategory.PronounOrDeterminer));
+ DUTCH.addTag(new PosTag("Punc",LexicalCategory.Punctuation));
+ DUTCH.addTag(new PosTag("V",LexicalCategory.Verb));
+ getInstance().add(DUTCH);
+ }
+ /**
+ * POS tags used by the Swedish POS model of OpenNLP for Swedish based on the
+ * <a href="http://w3.msi.vxu.se/users/nivre/research/MAMBAlex.html">
+ * Lexical categories in MAMBA</a>
+ */
+ public static final TagSet<PosTag> SWEDISH = new TagSet<PosTag>("MAMBA Swedish","sv");
+
+ static {
+ SWEDISH.addTag(new PosTag("PN",LexicalCategory.Noun));
+ SWEDISH.addTag(new PosTag("MN",LexicalCategory.Noun));
+ SWEDISH.addTag(new PosTag("AN",LexicalCategory.Noun));
+ SWEDISH.addTag(new PosTag("VN",LexicalCategory.Noun));
+ SWEDISH.addTag(new PosTag("NN",LexicalCategory.Noun));
+ SWEDISH.addTag(new PosTag("PO",LexicalCategory.PronounOrDeterminer));
+ SWEDISH.addTag(new PosTag("EN",LexicalCategory.Quantifier));
+ SWEDISH.addTag(new PosTag("RO",LexicalCategory.Quantifier));
+ SWEDISH.addTag(new PosTag("AJ",LexicalCategory.Adjective));
+ SWEDISH.addTag(new PosTag("AV",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("BV",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("HV",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("WV",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("QV",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("MV",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("KV",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("SV",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("GV",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("FV",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("VV",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("TP",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("SP",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("AB",LexicalCategory.Adverb));
+ SWEDISH.addTag(new PosTag("PR",LexicalCategory.Adposition));
+ SWEDISH.addTag(new PosTag("IM",LexicalCategory.Verb));
+ SWEDISH.addTag(new PosTag("++",LexicalCategory.Conjuction));
+ SWEDISH.addTag(new PosTag("UK",LexicalCategory.Conjuction));
+ SWEDISH.addTag(new PosTag("IK",LexicalCategory.Punctuation));
+ SWEDISH.addTag(new PosTag("IP",LexicalCategory.Punctuation));
+ SWEDISH.addTag(new PosTag("I?",LexicalCategory.Punctuation));
+ SWEDISH.addTag(new PosTag("IU",LexicalCategory.Punctuation));
+ SWEDISH.addTag(new PosTag("IQ",LexicalCategory.Punctuation));
+ SWEDISH.addTag(new PosTag("IS",LexicalCategory.Punctuation));
+ SWEDISH.addTag(new PosTag("IT",LexicalCategory.Punctuation));
+ SWEDISH.addTag(new PosTag("IR",LexicalCategory.Punctuation));
+ SWEDISH.addTag(new PosTag("IC",LexicalCategory.Punctuation));
+ SWEDISH.addTag(new PosTag("PU",LexicalCategory.Punctuation));
+ SWEDISH.addTag(new PosTag("IG",LexicalCategory.Punctuation));
+ SWEDISH.addTag(new PosTag("YY",LexicalCategory.Conjuction));
+ SWEDISH.addTag(new PosTag("ID",LexicalCategory.Noun));
+ SWEDISH.addTag(new PosTag("XX",null));
+ getInstance().add(SWEDISH);
+ }
+
+}
Added: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/POSTaggingEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/POSTaggingEngine.java?rev=1387488&view=auto
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/POSTaggingEngine.java (added)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/POSTaggingEngine.java Wed Sep 19 08:48:32 2012
@@ -0,0 +1,414 @@
+/*
+ * Copyright (c) 2012 Sebastian Schaffert
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stanbol.enhancer.engines.opennlp.pos.services;
+
+import static java.util.Collections.singleton;
+import static org.apache.stanbol.enhancer.nlp.NlpAnnotations.POSAnnotation;
+import static org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper.getLanguage;
+import static org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper.initAnalysedText;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Dictionary;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import opennlp.tools.postag.POSModel;
+import opennlp.tools.postag.POSTagger;
+import opennlp.tools.postag.POSTaggerME;
+import opennlp.tools.sentdetect.SentenceDetector;
+import opennlp.tools.sentdetect.SentenceDetectorME;
+import opennlp.tools.sentdetect.SentenceModel;
+import opennlp.tools.tokenize.Tokenizer;
+import opennlp.tools.util.Sequence;
+
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.ConfigurationPolicy;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Properties;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.Service;
+import org.apache.stanbol.commons.opennlp.OpenNLP;
+import org.apache.stanbol.enhancer.engines.opennlp.pos.model.PosTagSetRegistry;
+import org.apache.stanbol.enhancer.nlp.NlpAnnotations;
+import org.apache.stanbol.enhancer.nlp.TagSet;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedText;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedTextFactory;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedTextUtils;
+import org.apache.stanbol.enhancer.nlp.model.Section;
+import org.apache.stanbol.enhancer.nlp.model.Sentence;
+import org.apache.stanbol.enhancer.nlp.model.Span;
+import org.apache.stanbol.enhancer.nlp.model.Span.SpanTypeEnum;
+import org.apache.stanbol.enhancer.nlp.model.Token;
+import org.apache.stanbol.enhancer.nlp.model.annotation.Value;
+import org.apache.stanbol.enhancer.nlp.pos.PosTag;
+import org.apache.stanbol.enhancer.nlp.utils.LanguageConfiguration;
+import org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper;
+import org.apache.stanbol.enhancer.servicesapi.Blob;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.enhancer.servicesapi.EngineException;
+import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.helper.ContentItemHelper;
+import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
+import org.apache.stanbol.enhancer.servicesapi.impl.AbstractEnhancementEngine;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A german language POS tagger. Requires that the content item has a text/plain part and a
+ * language id of "de". Adds a POSContentPart to the content item that can be used for further
+ * processing by other modules.
+ * <p/>
+ * Author: Sebastian Schaffert
+ */
+
+@Component(immediate = true, metatype = true, configurationFactory = true, policy = ConfigurationPolicy.REQUIRE)
+@Service
+@Properties(value={
+ @Property(name= EnhancementEngine.PROPERTY_NAME,value="pos-tagger")
+})
+public class POSTaggingEngine extends AbstractEnhancementEngine<RuntimeException,RuntimeException> {
+
+
+ /**
+ * Language configuration. Takes a list of ISO language codes of supported languages. Currently supported
+ * are the languages given as default value.
+ */
+ @Property(value = {"en","de","da","es","sv","pt","nl"})
+ public static final String CONFIG_LANGUAGES = "org.apache.stanbol.enhancer.pos.languages";
+
+
+ public static final String[] AVAILABLE_LANGUAGES = new String[] {"en","de","da","es","sv","pt","nl"};
+
+
+ private static Logger log = LoggerFactory.getLogger(POSTaggingEngine.class);
+
+ //Langauge configuration
+ private LanguageConfiguration languageConfig = new LanguageConfiguration(CONFIG_LANGUAGES, AVAILABLE_LANGUAGES);
+// private Set<String> configuredLanguages;
+// private Set<String> excludedLanguages;
+// private boolean allowAll;
+
+ @Reference
+ private OpenNLP openNLP;
+ /**
+ * Provides known {@link TagSet}s used by OpenNLP
+ */
+ private PosTagSetRegistry tagSetRegistry = PosTagSetRegistry.getInstance();
+
+ @Reference
+ private AnalysedTextFactory analysedTextFactory;
+
+ /**
+ * Indicate if this engine can enhance supplied ContentItem, and if it
+ * suggests enhancing it synchronously or asynchronously. The
+ * {@link org.apache.stanbol.enhancer.servicesapi.EnhancementJobManager} can force sync/async mode if desired, it is
+ * just a suggestion from the engine.
+ * <p/>
+ * Returns ENHANCE_ASYNC in case there is a text/plain content part and a tagger for the language identified for
+ * the content item, CANNOT_ENHANCE otherwise.
+ *
+ * @throws org.apache.stanbol.enhancer.servicesapi.EngineException
+ * if the introspecting process of the content item
+ * fails
+ */
+ @Override
+ public int canEnhance(ContentItem ci) throws EngineException {
+ // check if content is present
+ Map.Entry<UriRef,Blob> entry = NlpEngineHelper.getPlainText(this, ci, false);
+ if(entry == null || entry.getValue() == null) {
+ return CANNOT_ENHANCE;
+ }
+
+ String language = getLanguage(this,ci,false);
+ if(language == null) {
+ return CANNOT_ENHANCE;
+ }
+ if(!languageConfig.isLanguage(language)){
+ log.trace(" > can NOT enhance ContentItem {} because language {} is "
+ + "not enabled by this engines configuration",ci,language);
+ return CANNOT_ENHANCE;
+ }
+
+ if(getPOSTagger(language) == null) {
+ log.trace(" > can NOT enhance ContentItem {} because no POSTagger is"
+ + "is present for language {}",ci,language);
+ return CANNOT_ENHANCE;
+ }
+
+ log.trace(" > can enhance ContentItem {} with language {}",ci,language);
+ return ENHANCE_ASYNC;
+ }
+
+ /**
+ * Compute enhancements for supplied ContentItem. The results of the process
+ * are expected to be stored in the metadata of the content item.
+ * <p/>
+ * The client (usually an {@link org.apache.stanbol.enhancer.servicesapi.EnhancementJobManager}) should take care of
+ * persistent storage of the enhanced {@link org.apache.stanbol.enhancer.servicesapi.ContentItem}.
+ * <p/>
+ * This method creates a new POSContentPart using {@link org.apache.stanbol.enhancer.engines.pos.api.POSTaggerHelper#createContentPart} from a text/plain part and
+ * stores it as a new part in the content item. The metadata is not changed.
+ *
+ * @throws org.apache.stanbol.enhancer.servicesapi.EngineException
+ * if the underlying process failed to work as
+ * expected
+ */
+ @Override
+ public void computeEnhancements(ContentItem ci) throws EngineException {
+ AnalysedText at = initAnalysedText(this,analysedTextFactory,ci);
+ String language = getLanguage(this, ci, true);
+
+ POSTagger posTagger = getPOSTagger(language);
+ if(posTagger == null){
+ //this means that the POS tagger became unavailable in-between
+ //the call to canEnhance and computeEnhancement
+ throw new EngineException("PosTagger for langauge '"+language
+ + "is not available.");
+ }
+ TagSet<PosTag> tagSet = tagSetRegistry.getTagSet(language);
+ if(tagSet == null){
+ log.warn("No POS TagSet registered for Language '{}'. Will build an "
+ + "adhoc set based on encountered Tags!",language);
+ //for now only created to avoid checks for tagSet == null
+ //TODO: in future we might want to automatically create posModels based
+ //on tagged texts. However this makes no sense as long we can not
+ //persist TagSets.
+ tagSet = new TagSet<PosTag>("dummy", language);
+ }
+ //holds PosTags created for POS tags that where not part of the posModel
+ //(will hold all PosTags in case tagSet is NULL
+ Map<String,PosTag> adhocTags = new HashMap<String,PosTag>();
+ //(1) Sentence detection
+
+ //Try to read existing Sentence Annotations
+ Iterator<Sentence> sentences = at.getSentences();
+ List<Section> sentenceList;
+ if(!sentences.hasNext()){
+ //if non try to detect sentences
+ log.trace(" > detect sentences for {}",at);
+ sentenceList = detectSentences(at,language);
+ }
+ if(sentences.hasNext()){ //check if we have detected sentences
+ log.trace(" > use existing Sentence annotations for {}",at);
+ sentenceList = new ArrayList<Section>();
+ AnalysedTextUtils.appandToList(sentences, sentenceList);
+ } else {
+ //no sentence detected ... treat the whole text as a single sentence
+ //TODO: maybe apply here a limit to the text size!
+ log.trace(" > unable to detect Sentences for {} (langauge: {})",at,language);
+ sentenceList = Collections.singletonList((Section)at);
+ }
+
+ //for all sentences (or the whole Text - if no sentences available)
+ for(Section sentence : sentenceList){
+
+ //(2) Tokenize Sentences
+
+ List<Token> tokenList;
+ //check if there are already tokens
+ Iterator<Token> tokens = sentence.getTokens();
+ if(!tokens.hasNext()){ //no tokens present -> tokenize
+ log.trace(" > tokenize {}",sentence);
+ tokenList = tokenize(sentence,language);
+ } else { //use existing
+ log.trace(" > use existing Tokens for {}",sentence);
+ tokenList = new ArrayList<Token>(); //ensure an ArrayList is used
+ AnalysedTextUtils.appandToList(tokens, tokenList);
+ }
+
+ //(3) POS Tagging
+
+ posTag(tokenList, posTagger,tagSet,adhocTags);
+
+ }
+
+ logAnnotations(at);
+ }
+
+ private void logAnnotations(AnalysedText at){
+ Iterator<Span> it = at.getEnclosed(EnumSet.of(SpanTypeEnum.Sentence, SpanTypeEnum.Token));
+ while(it.hasNext()){
+ Span span = it.next();
+ log.info(" > {}",span);
+ for(Value<PosTag> value : span.getAnnotations(POSAnnotation)){
+ log.info(" - {}",value);
+ }
+ }
+ }
+ /**
+ * POS tags the parsed tokens by using the pos tagger. Annotations are
+ * added based on the posModel and already created adhoc tags.
+ * @param tokenList
+ * @param posTagger
+ * @param posModel
+ * @param adhocTags
+ */
+ private void posTag(List<Token> tokenList,
+ POSTagger posTagger,
+ TagSet<PosTag> posModel,
+ Map<String,PosTag> adhocTags) {
+ String[] tokenTexts = new String[tokenList.size()];
+ for(int i=0;i<tokenList.size(); i++){
+ tokenTexts[i] = tokenList.get(i).getSpan();
+ }
+ //get the topK POS tags and props and copy it over to the 2dim Arrays
+ Sequence[] posSequences = posTagger.topKSequences(tokenTexts);
+ //extract the POS tags and props for the current token from the
+ //posSequences.
+ //NOTE: Sequence includes always POS tags for all Tokens. If
+ // less then posSequences.length are available it adds the
+ // best match for all followings.
+ // We do not want such copies.
+ PosTag[] actPos = new PosTag[posSequences.length];
+ double[] actProp = new double[posSequences.length];
+ for(int i=0;i<tokenTexts.length;i++){
+ Token token = tokenList.get(i);
+ boolean done = false;
+ int j = 0;
+ while( j < posSequences.length && !done){
+ String p = posSequences[j].getOutcomes().get(i);
+ done = j > 0 && p.equals(actPos[0].getTag());
+ if(!done){
+ actPos[j] = getPosTag(posModel,adhocTags,p);
+ actProp[j] = posSequences[j].getProbs()[i];
+ j++;
+ }
+ }
+ //create the POS values
+ token.addAnnotations(POSAnnotation, Value.values(actPos, actProp,j));
+ }
+
+ }
+
+ private PosTag getPosTag(TagSet<PosTag> model, Map<String,PosTag> adhocTags, String tag) {
+ PosTag posTag = model.getTag(tag);
+ if(posTag != null){
+ return posTag;
+ }
+ posTag = adhocTags.get(tag);
+ if(posTag != null){
+ return posTag;
+ }
+ posTag = new PosTag(tag);
+ adhocTags.put(tag, posTag);
+ log.warn("Encountered unknown POS tag '{}' for langauge '{}'",tag,model.getLanguages());
+ return posTag;
+ }
+
+ private List<Token> tokenize(Section section,String langauge) {
+ Tokenizer tokenizer = getTokenizer(langauge);
+ String text = section.getSpan();
+ List<Token> tokens = new ArrayList<Token>(text.length()/5); //assume avr. token length is 5
+ opennlp.tools.util.Span[] tokenSpans = tokenizer.tokenizePos(section.getSpan());
+ for(int i=0;i<tokenSpans.length;i++){
+ Token token = section.addToken(tokenSpans[i].getStart(), tokenSpans[i].getEnd());
+ log.trace(" > add {}",token);
+ tokens.add(token);
+ }
+ return tokens;
+ }
+
+ private List<Section> detectSentences(AnalysedText at, String language) {
+ SentenceDetector sentenceDetector = getSentenceDetector(language);
+ List<Section> sentences;
+ if(sentenceDetector != null){
+ sentences = new ArrayList<Section>();
+ for(opennlp.tools.util.Span sentSpan : sentenceDetector.sentPosDetect(at.getSpan())) {
+ Sentence sentence = at.addSentence(sentSpan.getStart(), sentSpan.getEnd());
+ log.trace(" > add {}",sentence);
+ sentences.add(sentence);
+ }
+ } else {
+ sentences = null;
+ }
+ return sentences;
+ }
+
+ /**
+ * Activate and read the properties. Configures and initialises a POSTagger for each language configured in
+ * CONFIG_LANGUAGES.
+ *
+ * @param ce the {@link org.osgi.service.component.ComponentContext}
+ */
+ @Activate
+ protected void activate(ComponentContext ce) throws ConfigurationException {
+ log.info("activating POS tagging engine");
+ super.activate(ce);
+ @SuppressWarnings("unchecked")
+ Dictionary<String, Object> properties = ce.getProperties();
+
+ languageConfig.setConfiguration(properties);
+ }
+
+ @Deactivate
+ protected void deactivate(ComponentContext context) {
+ languageConfig.setDefault();
+ super.deactivate(context);
+ }
+
+
+ private SentenceDetector getSentenceDetector(String language) {
+ try {
+ SentenceModel model = openNLP.getSentenceModel(language);
+ if(model != null) {
+ log.debug("Sentence Detection Model {} for lanugage '{}' version: {}",
+ new Object[]{model.getClass().getSimpleName(),
+ model.getLanguage(),
+ model.getVersion() != null ? model.getVersion() : "undefined"});
+ return new SentenceDetectorME(model);
+ }
+ } catch (Exception e) {
+ }
+ log.debug("Sentence Detection Model for Language '{}' not available.", language);
+ return null;
+ }
+ private POSTagger getPOSTagger(String language) {
+ try {
+ POSModel model = openNLP.getPartOfSpeachModel(language);
+ if(model != null) {
+ log.debug("POS Tagger Model {} for lanugage '{}' version: {}",
+ new Object[]{model.getClass().getSimpleName(),
+ model.getLanguage(),
+ model.getVersion() != null ? model.getVersion() : "undefined"});
+ return new POSTaggerME(model);
+ }
+ } catch (Exception e) {
+ log.warn("Unable to load POS model for language '"+language+"'!",e);
+ }
+ log.debug("POS tagging Model for Language '{}' not available.", language);
+ return null;
+ }
+
+ private Tokenizer getTokenizer(String language){
+ return openNLP.getTokenizer(language);
+ }
+
+}
Added: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/resources/OSGI-INF/metatype/metatype.properties
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/resources/OSGI-INF/metatype/metatype.properties?rev=1387488&view=auto
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/resources/OSGI-INF/metatype/metatype.properties (added)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/resources/OSGI-INF/metatype/metatype.properties Wed Sep 19 08:48:32 2012
@@ -0,0 +1,29 @@
+#
+# Copyright 2012, FORMCEPT [http://www.formcept.com]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.stanbol.enhancer.engines.opennlp.pos.services.POSTaggingEngine.name=Apache Stanbol Enhancer Engine: POS Tagging
+
+
+stanbol.enhancer.engine.name.name=POS Tagging Engine
+stanbol.enhancer.engine.name.description=The name of the enhancement engine as \
+used in the RESTful interface '/engine/<name>'
+service.ranking.name=Ranking
+service.ranking.description=If two enhancement engines with the same name are active the \
+one with the higher ranking will be used to process parsed content items.
+
+org.apache.stanbol.enhancer.pos.languages.name=Language configuration
+org.apache.stanbol.enhancer.pos.languages.description=Takes a list of ISO \
+ language codes of supported languages. Currently supported are the languages given as default value.
Added: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/resources/config/org.apache.stanbol.enhancer.engines.pos.services.POSTaggingEngine-snml.config
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/resources/config/org.apache.stanbol.enhancer.engines.pos.services.POSTaggingEngine-snml.config?rev=1387488&view=auto
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/resources/config/org.apache.stanbol.enhancer.engines.pos.services.POSTaggingEngine-snml.config (added)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/opennlp-pos/src/main/resources/config/org.apache.stanbol.enhancer.engines.pos.services.POSTaggingEngine-snml.config Wed Sep 19 08:48:32 2012
@@ -0,0 +1,2 @@
+stanbol.enhancer.chain.name="pos-tagging"
+stanbol.enhancer.chain.list.enginelist=["tika","langid","pos-tagger"]
Modified: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/pom.xml?rev=1387488&r1=1387487&r2=1387488&view=diff
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/pom.xml (original)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/pom.xml Wed Sep 19 08:48:32 2012
@@ -58,5 +58,12 @@
<module>zemanta</module> <!-- htt://zemanta.com -->
<!-- DBpedia.org Spotlight Enhancement Engines (STANBOL-706) -->
<module>dbpedia-spotlight</module>
+
+ <!-- NLP engines -->
+ <module>opennlp-pos</module>
+ <module>opennlp-chunker</module>
+ <module>sentiment-wordclassifier</module>
+ <module>nlp2rdf</module> <!-- converts AnalyzedText ContentPart to RDF -->
+
</modules>
</project>
Propchange: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Wed Sep 19 08:48:32 2012
@@ -0,0 +1,7 @@
+target
+
+.settings
+
+.classpath
+
+.project
Added: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/pom.xml?rev=1387488&view=auto
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/pom.xml (added)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/pom.xml Wed Sep 19 08:48:32 2012
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>org.apache.stanbol.enhancer.parent</artifactId>
+ <groupId>org.apache.stanbol</groupId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <relativePath>../../parent</relativePath>
+ </parent>
+
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.sentiment.wordclassifier</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <packaging>bundle</packaging>
+
+ <name>Apache Stanbol Enhancer Enhancement Engine: Sentiment Word Tagging Engine</name>
+ <description>
+ Sentiment Tagging Engine for the Stanbol NLP chain. Tags sentiment
+ values in word granularity based on SentimentClassifier implementations
+ registered as OSGI services. This engine includes implementations for
+ English (Sentiment Word Net) and German (SentiWs) data (please note the
+ license of the according data sets).
+ </description>
+
+ <inceptionYear>2012</inceptionYear>
+
+ <scm>
+ <connection>
+ scm:svn:http://svn.apache.org/repos/asf/incubator/stanbol/trunk/enhancer/engines/sentiment-wordtagging/
+ </connection>
+ <developerConnection>
+ scm:svn:https://svn.apache.org/repos/asf/incubator/stanbol/trunk/enhancer/engines/sentiment-wordtagging/
+ </developerConnection>
+ <url>http://incubator.apache.org/stanbol/</url>
+ </scm>
+
+ <properties>
+ <sentiment.model.path>org/apache/stanbol/data/sentiment</sentiment.model.path>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <!-- Extension used to provide files in that directory to the
+ DataFileProvider -->
+ <Data-Files>
+ ${sentiment.model.path}/de,
+ ${sentiment.model.path}/en
+ </Data-Files>
+ <!-- Use a priority lower than 0 to allow providers without a
+ defined ranking to override this default data. -->
+ <Data-Files-Priority>
+ -100
+ </Data-Files-Priority>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <!-- AL20 License -->
+ <exclude>src/license/THIRD-PARTY.properties</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-scr-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.servicesapi</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.nlp</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.commons.solr.core</artifactId>
+ <version>0.10.1-incubating-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>org.apache.felix.scr.annotations</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
Added: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/src/main/java/org/apache/stanbol/enhancer/engines/sentiment/api/LexicalCategoryClassifier.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/src/main/java/org/apache/stanbol/enhancer/engines/sentiment/api/LexicalCategoryClassifier.java?rev=1387488&view=auto
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/src/main/java/org/apache/stanbol/enhancer/engines/sentiment/api/LexicalCategoryClassifier.java (added)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/src/main/java/org/apache/stanbol/enhancer/engines/sentiment/api/LexicalCategoryClassifier.java Wed Sep 19 08:48:32 2012
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2012 Sebastian Schaffert
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stanbol.enhancer.engines.sentiment.api;
+
+import org.apache.stanbol.enhancer.nlp.pos.LexicalCategory;
+import org.apache.stanbol.enhancer.nlp.pos.PosTag;
+
+/**
+ * Implements the {@link #isAdjective(PosTag)} and {@link #isNoun(PosTag)}
+ * methods by using the {@link LexicalCategory} of the parsed {@link PosTag}.<p>
+ * This should be sufficient for all POS TagSets that are mapped to
+ * {@link LexicalCategory}. For other TagSets users will need to manually
+ * check {@link PosTag#getTag()}.
+ *
+ * @author Rupert Westenthaler
+ *
+ */
+public abstract class LexicalCategoryClassifier implements SentimentClassifier {
+
+ public abstract double classifyWord(String word);
+
+ @Override
+ public boolean isAdjective(PosTag posTag) {
+ return posTag.getCategory() != null &&
+ LexicalCategory.Adjective == posTag.getCategory();
+ }
+
+ @Override
+ public boolean isNoun(PosTag posTag) {
+ return posTag.getCategory() != null &&
+ LexicalCategory.Noun == posTag.getCategory();
+ }
+
+}
Added: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/src/main/java/org/apache/stanbol/enhancer/engines/sentiment/api/SentimentClassifier.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/src/main/java/org/apache/stanbol/enhancer/engines/sentiment/api/SentimentClassifier.java?rev=1387488&view=auto
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/src/main/java/org/apache/stanbol/enhancer/engines/sentiment/api/SentimentClassifier.java (added)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/src/main/java/org/apache/stanbol/enhancer/engines/sentiment/api/SentimentClassifier.java Wed Sep 19 08:48:32 2012
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2012 Sebastian Schaffert
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stanbol.enhancer.engines.sentiment.api;
+
+import org.apache.stanbol.enhancer.nlp.pos.PosTag;
+import org.osgi.framework.BundleContext;
+
+/**
+ * A simple classifier that determines for a given word a positive or negative
+ * value between -1 and 1. Unknown words will return the value 0.
+ * <p/>
+ * This Interface need to be implemented by Sentiment frameworks so that they
+ * can be used with this engine. Implementations need to be
+ * {@link BundleContext#registerService(String, Object, java.util.Dictionary)
+ * registered as OSGI service}.
+ * @author Sebastian Schaffert
+ */
+public interface SentimentClassifier {
+
+ /**
+ * Given the word passed as argument, return a value between -1 and 1 indicating its sentiment value from
+ * very negative to very positive. Unknown words should return the value 0.
+ *
+ * @param word
+ * @return
+ */
+ public double classifyWord(String word);
+
+
+ /**
+ * Helper method. Return true if the given POS tag indicates an adjective in the language implemented by
+ * this classifier.
+ *
+ * @param posTag
+ * @return
+ */
+ public boolean isAdjective(PosTag posTag);
+
+ /**
+ * Helper method. Return true if the given POS tag indicates a noun in the language implemented by this
+ * classifier.
+ *
+ * @param posTag
+ * @return
+ */
+ public boolean isNoun(PosTag posTag);
+
+ /**
+ * The language of this WordClassifier
+ * @return the language
+ */
+ public String getLanguage();
+}
Added: incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/src/main/java/org/apache/stanbol/enhancer/engines/sentiment/classifiers/SentiWSComponent.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/src/main/java/org/apache/stanbol/enhancer/engines/sentiment/classifiers/SentiWSComponent.java?rev=1387488&view=auto
==============================================================================
--- incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/src/main/java/org/apache/stanbol/enhancer/engines/sentiment/classifiers/SentiWSComponent.java (added)
+++ incubator/stanbol/branches/stanbol-nlp-processing/enhancer/engines/sentiment-wordclassifier/src/main/java/org/apache/stanbol/enhancer/engines/sentiment/classifiers/SentiWSComponent.java Wed Sep 19 08:48:32 2012
@@ -0,0 +1,259 @@
+/*
+ * Copyright (c) 2012 Sebastian Schaffert
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stanbol.enhancer.engines.sentiment.classifiers;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Dictionary;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Hashtable;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileListener;
+import org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileTracker;
+import org.apache.stanbol.enhancer.engines.sentiment.api.LexicalCategoryClassifier;
+import org.apache.stanbol.enhancer.engines.sentiment.api.SentimentClassifier;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.ServiceRegistration;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A German word classifier based on SentiWS. Reads the SentiWS positive and negative word lists and parses them
+ * into an appropriate hash table, so lookups should be extremely fast.
+ * <p/>
+ * @author Sebastian Schaffert
+ */
+@Component(immediate=true)
+public class SentiWSComponent {
+
+ private static Logger log = LoggerFactory.getLogger(SentiWSComponent.class);
+ private static final Map<String,String> modelProperties = new HashMap<String,String>();
+ static {
+ modelProperties.put("Description", "Sentiment Word List (German)");
+ modelProperties.put("Download Location", "http://wortschatz.informatik.uni-leipzig.de/download/");
+ }
+ @Reference
+ private DataFileTracker dataFileProvider;
+
+ protected BundleContext bundleContext;
+
+ /**
+ * Registration for the {@link SentiWsClassifierDE}. Will be created as
+ * soon as both required data files are available and successfully loaded
+ */
+ protected ServiceRegistration sentiWsClassifierService;
+ protected SentiWsClassifierDE sentiWsClassifier;
+
+
+ private ModelListener modelListener = new ModelListener();
+
+ protected Set<String> sentiWsFileNames = new HashSet<String>();
+ protected Set<String> loadedSentiWsFiles = new HashSet<String>();
+
+
+ public SentiWSComponent() {}
+
+ /**
+ * Tracks the SentiWS files and triggers the registration of the service
+ */
+ private class ModelListener implements DataFileListener {
+
+ @Override
+ public boolean available(String resourceName, InputStream is) {
+ if(sentiWsFileNames.contains(resourceName)){
+ log.info("sentiWs resource {} available",resourceName);
+ try {
+ long start = System.currentTimeMillis();
+ if(sentiWsClassifier != null){
+ sentiWsClassifier.parseSentiWS(is);
+ loadedSentiWsFiles.add(resourceName);
+ log.info(" ... loaded in {} ms",(System.currentTimeMillis()-start));
+ }
+ } catch (IOException e) {
+ log.warn("Unable to load sentiWs resource '"+resourceName+"!",e);
+ return false; //keep tracking
+ } catch (RuntimeException e) {
+ log.error("RuntimeException while loading sentiWs resource '"
+ +resourceName+"!",e);
+ return false; //keep tracking
+ }
+ } else {
+ log.warn("Tracker notified event for non-tracked resource '{}'"
+ + "(tracked: {})!",resourceName,sentiWsFileNames);
+ }
+ //all resources available ... start the service
+ if(loadedSentiWsFiles.equals(sentiWsFileNames)){
+ registerService();
+ }
+ //remove registration
+ return true;
+ }
+
+ @Override
+ public boolean unavailable(String resourceName) {
+ //not used;
+ return false;
+ }
+
+ }
+
+ @Activate
+ protected void activate(ComponentContext ctx){
+ bundleContext = ctx.getBundleContext();
+ //TODO: make Filenames configurable
+ sentiWsFileNames.add("SentiWS_v1.8b_Negative.txt");
+ sentiWsFileNames.add("SentiWS_v1.8b_Positive.txt");
+
+ //register files with the DataFileTracker
+ for(String sentiWsFile : sentiWsFileNames){
+ dataFileProvider.add(modelListener, sentiWsFile, modelProperties);
+ }
+ sentiWsClassifier = new SentiWsClassifierDE();
+ }
+
+ protected void registerService() {
+ Dictionary<String,Object> serviceProperties = new Hashtable<String,Object>();
+ serviceProperties.put("language", "de"); //set the language
+ BundleContext bc = bundleContext;
+ if(bc != null){
+ sentiWsClassifierService = bc.registerService(
+ SentimentClassifier.class.getName(), sentiWsClassifier,
+ serviceProperties);
+ }
+
+ }
+
+ protected void deactivate(ComponentContext ctx) {
+ //end datafile tracking
+ dataFileProvider.removeAll(modelListener);
+ sentiWsFileNames.clear();
+ loadedSentiWsFiles.clear();
+ //remove service registration
+ if(sentiWsClassifierService != null){
+ sentiWsClassifierService.unregister();
+ }
+ //free up resources
+ if(sentiWsClassifier != null){
+ sentiWsClassifier.close();
+ sentiWsClassifier = null;
+ }
+ bundleContext = null;
+ }
+
+ /**
+ * The OSGI service registered as soon as the required DataFiles are
+ * available
+ */
+ public static class SentiWsClassifierDE extends LexicalCategoryClassifier implements SentimentClassifier {
+
+ private ReadWriteLock lock = new ReentrantReadWriteLock();
+ private Map<String,Double> wordMap = new TreeMap<String,Double>();
+
+ protected SentiWsClassifierDE(){}
+
+ protected void parseSentiWS(InputStream is) throws IOException {
+ log.debug("parsing SentiWS word lists ...");
+ BufferedReader in = new BufferedReader(new InputStreamReader(is));
+ lock.writeLock().lock();
+ try {
+ for(String line = in.readLine(); line != null; line = in.readLine()) {
+ // input file will have a space- or tab-separated list per line:
+ // - first component is the main word with a specification what kind of word it is (can be result from POS tagging)
+ // - second component is the positive or negative sentiment associated with the word
+ // - third argument is a comma-separated list of deflections of the word as they might also occur in text
+ String[] components = line.split("\\s");
+
+ // parse the weight
+ Double weight = Double.parseDouble(components[1]);
+
+ // get the main word
+ String[] mainWord = components[0].split("\\|");
+ wordMap.put(mainWord[0],weight);
+
+ // get the remaining words (deflections)
+ if(components.length > 2) {
+ for(String word : components[2].split(",")) {
+ wordMap.put(word,weight);
+ }
+ }
+ }
+ } finally {
+ lock.writeLock().unlock();
+ IOUtils.closeQuietly(in);
+ }
+ }
+
+
+ public int getWordCount() {
+ lock.readLock().lock();
+ try {
+ return wordMap.size();
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public String getLanguage() {
+ return "de";
+ }
+
+ /**
+ * Given the word passed as argument, return a value between -1 and 1 indicating its sentiment value from
+ * very negative to very positive. Unknown words should return the value 0.
+ *
+ * @param word
+ * @return
+ */
+ @Override
+ public double classifyWord(String word) {
+ lock.readLock().lock();
+ try {
+ Double sentiment = wordMap.get(word);
+ return sentiment != null ? sentiment.doubleValue() : 0.0;
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+ /**
+ * Internally used to free up resources when the service is
+ * unregistered
+ */
+ protected void close(){
+ lock.writeLock().lock();
+ try {
+ wordMap.clear();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+ }
+
+}