You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2012/10/19 10:54:52 UTC
svn commit: r1400003 [1/2] - in
/stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src:
main/java/org/apache/stanbol/enhancer/engines/celi/
main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/
test/java/org/apache/stanbol/enh...
Author: rwesten
Date: Fri Oct 19 08:54:52 2012
New Revision: 1400003
URL: http://svn.apache.org/viewvc?rev=1400003&view=rev
Log:
STANBOL-739: Added a CELI based Lemmatizer Engine that writes the Results to the AnalyzedText ContentPart; Adapted the CeliTagSetRegistry to directly return PosTags instead of TagSets; Also added support for Unmapped Tags - as fallback for missing/wrong mappings AND to make it more easy to detect such wrong/missing mappings; Several Adaptions to changes in the enhancer.nlp.morpho pachage
Added:
stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliAnalyzedTextLemmatizerEngine.java
stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/test/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliAnalyzedTextLemmatizerEngineTest.java
Removed:
stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/PosTagSetRegistry.java
Modified:
stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/CeliMorphoFeatures.java
stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/CeliTagSetRegistry.java
stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliLemmatizerEnhancementEngine.java
stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/LemmatizerClientHTTP.java
stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/test/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliLemmatizerEnhancementEngineTest.java
Modified: stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/CeliMorphoFeatures.java
URL: http://svn.apache.org/viewvc/stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/CeliMorphoFeatures.java?rev=1400003&r1=1400002&r2=1400003&view=diff
==============================================================================
--- stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/CeliMorphoFeatures.java (original)
+++ stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/CeliMorphoFeatures.java Fri Oct 19 08:54:52 2012
@@ -9,6 +9,7 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.util.Vector;
@@ -23,15 +24,25 @@ import org.apache.clerezza.rdf.core.impl
import org.apache.clerezza.rdf.core.impl.TripleImpl;
import org.apache.commons.io.IOUtils;
import org.apache.stanbol.enhancer.engines.celi.lemmatizer.impl.CeliLemmatizerEnhancementEngine;
+import org.apache.stanbol.enhancer.engines.celi.lemmatizer.impl.Reading;
import org.apache.stanbol.enhancer.nlp.model.Token;
import org.apache.stanbol.enhancer.nlp.model.annotation.Annotation;
+import org.apache.stanbol.enhancer.nlp.model.tag.TagSet;
import org.apache.stanbol.enhancer.nlp.morpho.Case;
+import org.apache.stanbol.enhancer.nlp.morpho.CaseTag;
import org.apache.stanbol.enhancer.nlp.morpho.Definitness;
+import org.apache.stanbol.enhancer.nlp.morpho.DefinitnessTag;
import org.apache.stanbol.enhancer.nlp.morpho.Gender;
+import org.apache.stanbol.enhancer.nlp.morpho.GenderTag;
+import org.apache.stanbol.enhancer.nlp.morpho.MorphoFeatures;
import org.apache.stanbol.enhancer.nlp.morpho.NumberFeature;
+import org.apache.stanbol.enhancer.nlp.morpho.NumberTag;
import org.apache.stanbol.enhancer.nlp.morpho.Person;
+import org.apache.stanbol.enhancer.nlp.morpho.PersonTag;
import org.apache.stanbol.enhancer.nlp.morpho.Tense;
+import org.apache.stanbol.enhancer.nlp.morpho.TenseTag;
import org.apache.stanbol.enhancer.nlp.morpho.VerbMood;
+import org.apache.stanbol.enhancer.nlp.morpho.VerbMoodTag;
import org.apache.stanbol.enhancer.nlp.pos.LexicalCategory;
import org.apache.stanbol.enhancer.nlp.pos.PosTag;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
@@ -52,144 +63,97 @@ import org.slf4j.LoggerFactory;
* @author Alessio Bosca
*
*/
-public class CeliMorphoFeatures {
+public class CeliMorphoFeatures extends MorphoFeatures{
- public static final UriRef HAS_NUMBER = new UriRef("http://purl.org/olia/olia.owl#hasNumber");
- public static final UriRef HAS_GENDER = new UriRef("http://purl.org/olia/olia.owl#hasGender");
- public static final UriRef HAS_PERSON = new UriRef("http://purl.org/olia/olia.owl#hasPerson");
- public static final UriRef HAS_CASE = new UriRef("http://purl.org/olia/olia.owl#hasCase");
- public static final UriRef HAS_DEFINITENESS = new UriRef("http://purl.org/olia/olia.owl#hasDefiniteness");
- public static final UriRef HAS_MOOD = new UriRef("http://purl.org/olia/olia.owl#hasMood");
- public static final UriRef HAS_TENSE = new UriRef("http://purl.org/olia/olia.owl#hasTense");
-
- private static final Logger log = LoggerFactory.getLogger(CeliMorphoFeatures.class);
-
- private String lemma;
-
- private Set<LexicalCategory> posSet=new HashSet<LexicalCategory>();
- private Set<Gender> genderSet= new HashSet<Gender>();
- private Set<NumberFeature> numberSet=new HashSet<NumberFeature>();
- private Set<Case> caseFeatureSet=new HashSet<Case>();
- private Set<Person> personSet=new HashSet<Person>();
- private Set<Definitness> definitnessSet=new HashSet<Definitness>();
- private Set<VerbMood> verbFormSet=new HashSet<VerbMood>();
- private Set<Tense> tenseSet=new HashSet<Tense>();
-
- public CeliMorphoFeatures(String lemma) {
- if (lemma == null) {
- throw new IllegalArgumentException("The parsed lemma MUST NOT be NULL!");
- }
- this.lemma = lemma;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof CeliMorphoFeatures && lemma.equals(((CeliMorphoFeatures) o).lemma)) {
- CeliMorphoFeatures lt = (CeliMorphoFeatures) o;
- return ((genderSet != null && genderSet.equals(lt.genderSet)) || (genderSet == null && lt.genderSet == null)) && ((caseFeatureSet != null && caseFeatureSet.equals(lt.caseFeatureSet)) || (caseFeatureSet == null && lt.caseFeatureSet == null))
- && ((tenseSet != null && tenseSet.equals(lt.tenseSet)) || (tenseSet == null && lt.tenseSet == null)) && ((numberSet != null && numberSet.equals(lt.numberSet)) || (numberSet == null && lt.numberSet == null))
- && ((definitnessSet != null && definitnessSet.equals(lt.definitnessSet)) || (definitnessSet == null && lt.definitnessSet == null)) && ((personSet != null && personSet.equals(lt.personSet)) || (personSet == null && lt.personSet == null))
- && ((verbFormSet != null && verbFormSet.equals(lt.verbFormSet)) || (verbFormSet == null && lt.verbFormSet == null));
- } else {
- return false;
- }
- }
-
- @Override
- public int hashCode() {
- return lemma.hashCode() + posSet.hashCode() + genderSet.hashCode() + personSet.hashCode() + caseFeatureSet.hashCode() + definitnessSet.hashCode() + verbFormSet.hashCode() + tenseSet.hashCode();
- }
-
- public final void addCase(Case caseFeature) {
- this.caseFeatureSet.add(caseFeature);
- }
-
- public final void addDefinitness(Definitness definitness) {
- this.definitnessSet.add(definitness);
- }
-
- public final void addGender(Gender gender) {
- this.genderSet.add(gender);
- }
-
- public final void addNumber(NumberFeature number) {
- this.numberSet.add(number);
- }
-
- public void addPerson(Person person) {
- this.personSet.add(person);
- }
-
- public void addPos(LexicalCategory pos) {
- this.posSet.add(pos);
- }
+ private static CeliTagSetRegistry tagRegistry = CeliTagSetRegistry.getInstance();
- public void addTense(Tense tense) {
- this.tenseSet.add(tense);
- }
-
- public void addVerbForm(VerbMood verbForm) {
- this.verbFormSet.add(verbForm);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("MorphoTag(");
- sb.append(lemma);
- for(LexicalCategory pos: posSet){
- sb.append("|pos:").append(pos);
- }
- for(Gender gender: genderSet){
- sb.append("|gender:").append(gender);
- }
- for(NumberFeature num:numberSet) {
- sb.append("|number:").append(num);
- }
- for(Person pers: personSet) {
- sb.append("|person:").append(pers);
- }
- for(Definitness def: definitnessSet) {
- sb.append("|definitness:").append(def);
- }
- for(Case caseFeat:caseFeatureSet) {
- sb.append("|case:").append(caseFeat);
- }
- for (VerbMood vf:verbFormSet) {
- sb.append("|verbForm:").append(vf);
- }
- for(Tense t:tenseSet) {
- sb.append("|tense:").append(t);
- }
- sb.append(')');
- return sb.toString();
+ public static final UriRef HAS_NUMBER = new UriRef("http://purl.org/olia/olia.owl#hasNumber");
+ public static final UriRef HAS_GENDER = new UriRef("http://purl.org/olia/olia.owl#hasGender");
+ public static final UriRef HAS_PERSON = new UriRef("http://purl.org/olia/olia.owl#hasPerson");
+ public static final UriRef HAS_CASE = new UriRef("http://purl.org/olia/olia.owl#hasCase");
+ public static final UriRef HAS_DEFINITENESS = new UriRef("http://purl.org/olia/olia.owl#hasDefiniteness");
+ public static final UriRef HAS_MOOD = new UriRef("http://purl.org/olia/olia.owl#hasMood");
+ public static final UriRef HAS_TENSE = new UriRef("http://purl.org/olia/olia.owl#hasTense");
+
+ public static CeliMorphoFeatures parseFrom(Reading reading, String lang){
+ if(reading == null){
+ return null;
+ }
+ CeliMorphoFeatures morphoFeature = new CeliMorphoFeatures(reading.getLemma());
+ //parse the key,value pairs of the reading using the language as context
+ for (Entry<String, List<String>> entry : reading.getLexicalFeatures().entrySet()) {
+ String feature = entry.getKey();
+ for (String value : entry.getValue()) {
+ if (feature.equals("POS")) {
+ morphoFeature.addPos(tagRegistry.getPosTag(lang,value));
+ } else if (feature.equals("CASE")) {
+ morphoFeature.addCase(tagRegistry.getCaseTag(lang,value));
+ } else if (feature.equals("GENDER")) {
+ morphoFeature.addGender(tagRegistry.getGenderTag(lang,value));
+ } else if (feature.equals("NUMBER")) {
+ morphoFeature.addNumber(tagRegistry.getNumberTag(lang,value));
+ } else if (feature.equals("PERSON")) {
+ morphoFeature.addPerson(tagRegistry.getPersonTag(lang,value));
+ } else if (feature.equals("VERB_FORM") || feature.equals("VFORM")) {
+ morphoFeature.addVerbForm(tagRegistry.getVerbMoodTag(lang,value));
+ } else if (feature.equals("TENSE") || feature.equals("VERB_TENSE")) {
+ morphoFeature.addTense(tagRegistry.getTenseTag(lang,value));
+ }
+ }
+ }
+ return morphoFeature;
+ }
+ /**
+ * Use {@link #parseFrom(Reading, String)} to instantiate
+ * @param lemma
+ */
+ private CeliMorphoFeatures(String lemma) {
+ super(lemma);
}
public Collection<? extends Triple> featuresAsTriples(UriRef textAnnotation, Language lang) {
Collection<TripleImpl> result = new Vector<TripleImpl>();
- result.add(new TripleImpl(textAnnotation, CeliLemmatizerEnhancementEngine.hasLemmaForm, new PlainLiteralImpl(this.lemma, lang)));
- for(LexicalCategory pos: posSet){
- result.add(new TripleImpl(textAnnotation, RDF_TYPE, pos.getUri()));
- }
- for(NumberFeature num: numberSet){
- result.add(new TripleImpl(textAnnotation, HAS_NUMBER, num.getUri()));
- }
- for(Person pers: personSet){
- result.add(new TripleImpl(textAnnotation, HAS_PERSON, pers.getUri()));
- }
- for(Gender gender: genderSet){
- result.add(new TripleImpl(textAnnotation, HAS_GENDER, gender.getUri()));
- }
- for(Definitness def: definitnessSet){
- result.add(new TripleImpl(textAnnotation, HAS_DEFINITENESS, def.getUri()));
- }
- for(Case caseFeat:caseFeatureSet){
- result.add(new TripleImpl(textAnnotation, HAS_CASE, caseFeat.getUri()));
- }
- for (VerbMood vf:verbFormSet){
- result.add(new TripleImpl(textAnnotation, HAS_MOOD, vf.getUri()));
- }
- for(Tense tense:tenseSet){
- result.add(new TripleImpl(textAnnotation, HAS_TENSE, tense.getUri()));
+ result.add(new TripleImpl(textAnnotation, CeliLemmatizerEnhancementEngine.hasLemmaForm,
+ new PlainLiteralImpl(getLemma(), lang)));
+ for(PosTag pos: getPosList()){
+ if(pos.getCategory() != null){
+ result.add(new TripleImpl(textAnnotation, RDF_TYPE, pos.getCategory().getUri()));
+ }
+ }
+ for(NumberTag num : getNumberList()){
+ if(num.getNumber() != null){
+ result.add(new TripleImpl(textAnnotation, HAS_NUMBER, num.getNumber().getUri()));
+ }
+ }
+ for(PersonTag pers : getPersonList()){
+ if(pers.getPerson() != null){
+ result.add(new TripleImpl(textAnnotation, HAS_PERSON, pers.getPerson().getUri()));
+ }
+ }
+ for(GenderTag gender : getGenderList()){
+ if(gender.getGender() != null){
+ result.add(new TripleImpl(textAnnotation, HAS_GENDER, gender.getGender().getUri()));
+ }
+ }
+ for(DefinitnessTag def : getDefinitnessList()){
+ if(def.getDefinitness() != null){
+ result.add(new TripleImpl(textAnnotation, HAS_DEFINITENESS, def.getDefinitness().getUri()));
+ }
+ }
+ for(CaseTag caseFeat : getCaseList()){
+ if(caseFeat.getCase() != null){
+ result.add(new TripleImpl(textAnnotation, HAS_CASE, caseFeat.getCase().getUri()));
+ }
+ }
+ for (VerbMoodTag vf : getVerbMoodList()){
+ if(vf.getVerbForm() != null){
+ result.add(new TripleImpl(textAnnotation, HAS_MOOD, vf.getVerbForm().getUri()));
+ }
+ }
+ for(TenseTag tense : getTenseList()){
+ if(tense.getTense() != null){
+ result.add(new TripleImpl(textAnnotation, HAS_TENSE, tense.getTense().getUri()));
+ }
}
return result;
}
Modified: stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/CeliTagSetRegistry.java
URL: http://svn.apache.org/viewvc/stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/CeliTagSetRegistry.java?rev=1400003&r1=1400002&r2=1400003&view=diff
==============================================================================
--- stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/CeliTagSetRegistry.java (original)
+++ stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/CeliTagSetRegistry.java Fri Oct 19 08:54:52 2012
@@ -1,8 +1,12 @@
package org.apache.stanbol.enhancer.engines.celi;
+import java.lang.reflect.InvocationTargetException;
import java.util.HashMap;
import java.util.Map;
+import javax.swing.text.DefaultEditorKit.PasteAction;
+
+import org.apache.stanbol.enhancer.nlp.model.tag.Tag;
import org.apache.stanbol.enhancer.nlp.model.tag.TagSet;
import org.apache.stanbol.enhancer.nlp.morpho.Case;
import org.apache.stanbol.enhancer.nlp.morpho.CaseTag;
@@ -20,446 +24,576 @@ import org.apache.stanbol.enhancer.nlp.m
import org.apache.stanbol.enhancer.nlp.morpho.VerbMoodTag;
import org.apache.stanbol.enhancer.nlp.pos.LexicalCategory;
import org.apache.stanbol.enhancer.nlp.pos.PosTag;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * {@link TagSet}s for known CELI (linguagrid.org) POS and morphological features models.<p>
+ * {@link TagSet}s for known CELI (linguagrid.org) POS and morphological features models.
+ * <p>
*
* @author Rupert Westenthaler
- *
+ * @author Alessio Bosca
+ *
*/
public final class CeliTagSetRegistry {
-
+
+ private final static Logger log = LoggerFactory.getLogger(CeliTagSetRegistry.class);
+
private static CeliTagSetRegistry instance = new CeliTagSetRegistry();
-
- private CeliTagSetRegistry(){}
-
- private final Map<String, TagSet<PosTag>> posMappingsByLanguage = new HashMap<String,TagSet<PosTag>>();
- private final Map<String, TagSet<GenderTag>> genderMappingsByLanguage = new HashMap<String, TagSet<GenderTag>>();
- private final Map<String, TagSet<NumberTag>> numberMappingsByLanguage=new HashMap<String, TagSet<NumberTag>>();
- private final Map<String, TagSet<PersonTag>> personMappingsByLanguage=new HashMap<String, TagSet<PersonTag>>();
- private final Map<String, TagSet<CaseTag>> caseMappingsByLanguage=new HashMap<String, TagSet<CaseTag>>();
- private final Map<String, TagSet<DefinitnessTag>> definitenessMappingsByLanguage=new HashMap<String, TagSet<DefinitnessTag>>();
- private final Map<String, TagSet<VerbMoodTag>> verbFormMappingsByLanguage=new HashMap<String, TagSet<VerbMoodTag>>();
- private final Map<String, TagSet<TenseTag>> tenseMappingsByLanguage=new HashMap<String, TagSet<TenseTag>>();
-
-
- public static CeliTagSetRegistry getInstance(){
+
+ private CeliTagSetRegistry() {}
+
+ private final Map<String,TagSet<PosTag>> posMappingsByLanguage = new HashMap<String,TagSet<PosTag>>();
+ private final Map<String,Map<String,PosTag>> unmappedPosTagsByLanguage = new HashMap<String,Map<String,PosTag>>();
+
+ private final Map<String,TagSet<GenderTag>> genderMappingsByLanguage = new HashMap<String,TagSet<GenderTag>>();
+ private final Map<String,Map<String,GenderTag>> unmappedGenderTagsByLanguage = new HashMap<String,Map<String,GenderTag>>();
+
+ private final Map<String,TagSet<NumberTag>> numberMappingsByLanguage = new HashMap<String,TagSet<NumberTag>>();
+ private final Map<String,Map<String,NumberTag>> unmappedNumberTagsByLanguage = new HashMap<String,Map<String,NumberTag>>();
+
+ private final Map<String,TagSet<PersonTag>> personMappingsByLanguage = new HashMap<String,TagSet<PersonTag>>();
+ private final Map<String,Map<String,PersonTag>> unmappedPersonTagsByLanguage = new HashMap<String,Map<String,PersonTag>>();
+
+ private final Map<String,TagSet<CaseTag>> caseMappingsByLanguage = new HashMap<String,TagSet<CaseTag>>();
+ private final Map<String,Map<String,CaseTag>> unmappedCaseTagsByLanguage = new HashMap<String,Map<String,CaseTag>>();
+
+ private final Map<String,TagSet<DefinitnessTag>> definitenessMappingsByLanguage = new HashMap<String,TagSet<DefinitnessTag>>();
+ private final Map<String,Map<String,DefinitnessTag>> unmappedDefinitnessTagsByLanguage = new HashMap<String,Map<String,DefinitnessTag>>();
+
+ private final Map<String,TagSet<VerbMoodTag>> verbFormMappingsByLanguage = new HashMap<String,TagSet<VerbMoodTag>>();
+ private final Map<String,Map<String,VerbMoodTag>> unmappedVerbMoodTagsByLanguage = new HashMap<String,Map<String,VerbMoodTag>>();
+
+ private final Map<String,TagSet<TenseTag>> tenseMappingsByLanguage = new HashMap<String,TagSet<TenseTag>>();
+ private final Map<String,Map<String,TenseTag>> unmappedTenseTagsByLanguage = new HashMap<String,Map<String,TenseTag>>();
+
+ public static CeliTagSetRegistry getInstance() {
return instance;
}
-
+
/**
- * Setter for the mappings of {@link TagSet} by language.
- * @param mappings expressed with a {@link TagSet}
+ * Setter for the mappings of {@link TagSet} by language.
+ *
+ * @param mappings
+ * expressed with a {@link TagSet}
*/
private void addPosTagset(TagSet<PosTag> model) {
- for(String lang : model.getLanguages()){
- if(posMappingsByLanguage.put(lang, model) != null){
- throw new IllegalStateException("Multiple Models for Language '"
- + lang+"'! This is an error in the static confituration of "
- + "this class. Please report this to the stanbol-dev mailing"
- + "list!");
+ for (String lang : model.getLanguages()) {
+ if (posMappingsByLanguage.put(lang, model) != null) {
+ throw new IllegalStateException("Multiple Models for Language '" + lang
+ + "'! This is an error in the static confituration of "
+ + "this class. Please report this to the stanbol-dev mailing"
+ + "list!");
+ }
+ }
+ }
+
+ /**
+ * Getter for a {@link PosTag} based on the Registry configuration
+ *
+ * @param language
+ * the language
+ * @param tag
+ * the {@link String} tag as returned by CELI
+ * @return the {@link PosTag}
+ */
+ public PosTag getPosTag(String language, String tag) {
+ return getTag(posMappingsByLanguage, unmappedPosTagsByLanguage, PosTag.class, language, tag);
+ }
+
+ /**
+ * Setter for the mappings of {@link TagSet} by language.
+ *
+ * @param mappings
+ * expressed with a {@link TagSet}
+ */
+ private void addGenderTagset(TagSet<GenderTag> model) {
+ for (String lang : model.getLanguages()) {
+ if (genderMappingsByLanguage.put(lang, model) != null) {
+ throw new IllegalStateException("Multiple Models for Language '" + lang
+ + "'! This is an error in the static confituration of "
+ + "this class. Please report this to the stanbol-dev mailing"
+ + "list!");
+ }
+ }
+ }
+
+ /**
+ * Getter for a {@link GenderTag} based on the Registry configuration
+ *
+ * @param language
+ * the language
+ * @param tag
+ * the {@link String} tag as returned by CELI
+ * @return the {@link GenderTag}
+ */
+ public GenderTag getGenderTag(String language, String tag) {
+ return getTag(genderMappingsByLanguage, unmappedGenderTagsByLanguage, GenderTag.class, language, tag);
+ }
+
+ /**
+ * Setter for the mappings of {@link TagSet} by language.
+ *
+ * @param mappings
+ * expressed with a {@link TagSet}
+ */
+ private void addNumberTagset(TagSet<NumberTag> model) {
+ for (String lang : model.getLanguages()) {
+ if (numberMappingsByLanguage.put(lang, model) != null) {
+ throw new IllegalStateException("Multiple Models for Language '" + lang
+ + "'! This is an error in the static confituration of "
+ + "this class. Please report this to the stanbol-dev mailing"
+ + "list!");
+ }
+ }
+ }
+
+ /**
+ * Getter for a {@link NumberTag} based on the Registry configuration
+ *
+ * @param language
+ * the language
+ * @param tag
+ * the {@link String} tag as returned by CELI
+ * @return the {@link NumberTag}
+ */
+ public NumberTag getNumberTag(String language, String tag) {
+ return getTag(numberMappingsByLanguage, unmappedNumberTagsByLanguage, NumberTag.class, language, tag);
+ }
+
+ /**
+ * Setter for the mappings of {@link TagSet} by language.
+ *
+ * @param mappings
+ * expressed with a {@link TagSet}
+ */
+ private void addPersonTagset(TagSet<PersonTag> model) {
+ for (String lang : model.getLanguages()) {
+ if (personMappingsByLanguage.put(lang, model) != null) {
+ throw new IllegalStateException("Multiple Models for Language '" + lang
+ + "'! This is an error in the static confituration of "
+ + "this class. Please report this to the stanbol-dev mailing"
+ + "list!");
+ }
+ }
+ }
+
+ /**
+ * Getter for a {@link PersonTag} based on the Registry configuration
+ *
+ * @param language
+ * the language
+ * @param tag
+ * the {@link String} tag as returned by CELI
+ * @return the {@link PersonTag}
+ */
+ public PersonTag getPersonTag(String language, String tag) {
+ return getTag(personMappingsByLanguage, unmappedPersonTagsByLanguage, PersonTag.class, language, tag);
+ }
+
+ /**
+ * Setter for the mappings of {@link TagSet} by language.
+ *
+ * @param mappings
+ * expressed with a {@link TagSet}
+ */
+ private void addCaseTagset(TagSet<CaseTag> model) {
+ for (String lang : model.getLanguages()) {
+ if (caseMappingsByLanguage.put(lang, model) != null) {
+ throw new IllegalStateException("Multiple Models for Language '" + lang
+ + "'! This is an error in the static confituration of "
+ + "this class. Please report this to the stanbol-dev mailing"
+ + "list!");
+ }
+ }
+ }
+
+ /**
+ * Getter for a {@link CaseTag} based on the Registry configuration
+ *
+ * @param language
+ * the language
+ * @param tag
+ * the {@link String} tag as returned by CELI
+ * @return the {@link PersonTag}
+ */
+ public CaseTag getCaseTag(String language, String tag) {
+ return getTag(caseMappingsByLanguage, unmappedCaseTagsByLanguage, CaseTag.class, language, tag);
+ }
+
+ /**
+ * Setter for the mappings of {@link TagSet} by language.
+ *
+ * @param mappings
+ * expressed with a {@link TagSet}
+ */
+ private void addDefinitnessTagset(TagSet<DefinitnessTag> model) {
+ for (String lang : model.getLanguages()) {
+ if (definitenessMappingsByLanguage.put(lang, model) != null) {
+ throw new IllegalStateException("Multiple Models for Language '" + lang
+ + "'! This is an error in the static confituration of "
+ + "this class. Please report this to the stanbol-dev mailing"
+ + "list!");
}
}
}
+ /**
+ * Getter for a {@link DefinitnessTag} based on the Registry configuration
+ *
+ * @param language
+ * the language
+ * @param tag
+ * the {@link String} tag as returned by CELI
+ * @return the {@link DefinitnessTag}
+ */
+ public DefinitnessTag getDefinitnessTag(String language, String tag) {
+ return getTag(definitenessMappingsByLanguage, unmappedDefinitnessTagsByLanguage,
+ DefinitnessTag.class, language, tag);
+ }
+
+ /**
+ * Setter for the mappings of {@link TagSet} by language.
+ *
+ * @param mappings
+ * expressed with a {@link TagSet}
+ */
+ private void addVerbFormTagset(TagSet<VerbMoodTag> model) {
+ for (String lang : model.getLanguages()) {
+ if (verbFormMappingsByLanguage.put(lang, model) != null) {
+ throw new IllegalStateException("Multiple Models for Language '" + lang
+ + "'! This is an error in the static confituration of "
+ + "this class. Please report this to the stanbol-dev mailing"
+ + "list!");
+ }
+ }
+ }
/**
- * Getter for the {@link TagSet} by language. If no {@link TagSet}
- * is available for an Language this will return <code>null</code>
- * @param language the language
- * @return the AnnotationModel or <code>null</code> if non is defined
- */
- public TagSet<PosTag> getPosTagset(String language){
- return posMappingsByLanguage.get(language);
- }
-
- /**
- * Setter for the mappings of {@link TagSet} by language.
- * @param mappings expressed with a {@link TagSet}
- */
- private void addGenderTagset(TagSet<GenderTag> model) {
- for(String lang : model.getLanguages()){
- if(genderMappingsByLanguage.put(lang, model) != null){
- throw new IllegalStateException("Multiple Models for Language '"
- + lang+"'! This is an error in the static confituration of "
- + "this class. Please report this to the stanbol-dev mailing"
- + "list!");
- }
- }
- }
-
- /**
- * Getter for the {@link TagSet} by language. If no {@link TagSet}
- * is available for an Language this will return <code>null</code>
- * @param language the language
- * @return the AnnotationModel or <code>null</code> if non is defined
- */
- public TagSet<GenderTag> getGenderTagset(String language){
- return genderMappingsByLanguage.get(language);
- }
-
- /**
- * Setter for the mappings of {@link TagSet} by language.
- * @param mappings expressed with a {@link TagSet}
- */
- private void addNumberTagset(TagSet<NumberTag> model) {
- for(String lang : model.getLanguages()){
- if(numberMappingsByLanguage.put(lang, model) != null){
- throw new IllegalStateException("Multiple Models for Language '"
- + lang+"'! This is an error in the static confituration of "
- + "this class. Please report this to the stanbol-dev mailing"
- + "list!");
- }
- }
- }
-
- /**
- * Getter for the {@link TagSet} by language. If no {@link TagSet}
- * is available for an Language this will return <code>null</code>
- * @param language the language
- * @return the AnnotationModel or <code>null</code> if non is defined
- */
- public TagSet<NumberTag> getNumberTagset(String language){
- return numberMappingsByLanguage.get(language);
- }
-
-
- /**
- * Setter for the mappings of {@link TagSet} by language.
- * @param mappings expressed with a {@link TagSet}
- */
- private void addPersonTagset(TagSet<PersonTag> model) {
- for(String lang : model.getLanguages()){
- if(personMappingsByLanguage.put(lang, model) != null){
- throw new IllegalStateException("Multiple Models for Language '"
- + lang+"'! This is an error in the static confituration of "
- + "this class. Please report this to the stanbol-dev mailing"
- + "list!");
- }
- }
- }
-
- /**
- * Getter for the {@link TagSet} by language. If no {@link TagSet}
- * is available for an Language this will return <code>null</code>
- * @param language the language
- * @return the AnnotationModel or <code>null</code> if non is defined
- */
- public TagSet<PersonTag> getPersonTagset(String language){
- return personMappingsByLanguage.get(language);
- }
-
- /**
- * Setter for the mappings of {@link TagSet} by language.
- * @param mappings expressed with a {@link TagSet}
- */
- private void addCaseTagset(TagSet<CaseTag> model) {
- for(String lang : model.getLanguages()){
- if(caseMappingsByLanguage.put(lang, model) != null){
- throw new IllegalStateException("Multiple Models for Language '"
- + lang+"'! This is an error in the static confituration of "
- + "this class. Please report this to the stanbol-dev mailing"
- + "list!");
- }
- }
- }
-
- /**
- * Getter for the {@link TagSet} by language. If no {@link TagSet}
- * is available for an Language this will return <code>null</code>
- * @param language the language
- * @return the AnnotationModel or <code>null</code> if non is defined
- */
- public TagSet<CaseTag> getCaseTagset(String language){
- return caseMappingsByLanguage.get(language);
- }
-
- /**
- * Setter for the mappings of {@link TagSet} by language.
- * @param mappings expressed with a {@link TagSet}
- */
- private void addDefinitnessTagset(TagSet<DefinitnessTag> model) {
- for(String lang : model.getLanguages()){
- if(definitenessMappingsByLanguage.put(lang, model) != null){
- throw new IllegalStateException("Multiple Models for Language '"
- + lang+"'! This is an error in the static confituration of "
- + "this class. Please report this to the stanbol-dev mailing"
- + "list!");
- }
- }
- }
-
- /**
- * Getter for the {@link TagSet} by language. If no {@link TagSet}
- * is available for an Language this will return <code>null</code>
- * @param language the language
- * @return the AnnotationModel or <code>null</code> if non is defined
- */
- public TagSet<DefinitnessTag> getDefinitnessTagset(String language){
- return definitenessMappingsByLanguage.get(language);
- }
-
- /**
- * Setter for the mappings of {@link TagSet} by language.
- * @param mappings expressed with a {@link TagSet}
- */
- private void addVerbFormTagset(TagSet<VerbMoodTag> model) {
- for(String lang : model.getLanguages()){
- if(verbFormMappingsByLanguage.put(lang, model) != null){
- throw new IllegalStateException("Multiple Models for Language '"
- + lang+"'! This is an error in the static confituration of "
- + "this class. Please report this to the stanbol-dev mailing"
- + "list!");
- }
- }
- }
-
- /**
- * Getter for the {@link TagSet} by language. If no {@link TagSet}
- * is available for an Language this will return <code>null</code>
- * @param language the language
- * @return the AnnotationModel or <code>null</code> if non is defined
- */
- public TagSet<VerbMoodTag> getVerbFormTagset(String language){
- return verbFormMappingsByLanguage.get(language);
- }
-
- /**
- * Setter for the mappings of {@link TagSet} by language.
- * @param mappings expressed with a {@link TagSet}
- */
- private void addTenseTagset(TagSet<TenseTag> model) {
- for(String lang : model.getLanguages()){
- if(tenseMappingsByLanguage.put(lang, model) != null){
- throw new IllegalStateException("Multiple Models for Language '"
- + lang+"'! This is an error in the static confituration of "
- + "this class. Please report this to the stanbol-dev mailing"
- + "list!");
- }
- }
- }
-
- /**
- * Getter for the {@link TagSet} by language. If no {@link TagSet}
- * is available for an Language this will return <code>null</code>
- * @param language the language
- * @return the AnnotationModel or <code>null</code> if non is defined
- */
- public TagSet<TenseTag> getTenseTagset(String language){
- return tenseMappingsByLanguage.get(language);
- }
-
-
- /*****************************************************************
- * POS TAGSETS MAPPINGS TO OLIA ONTOLOGY *
- ****************************************************************/
-
-
- public static final TagSet<PosTag> ITALIAN = new TagSet<PosTag>("CELI Italian POS tags", "it");
- static {
- ITALIAN.addTag(new PosTag("N", LexicalCategory.Noun));
- ITALIAN.addTag(new PosTag("NF", LexicalCategory.Noun));
- ITALIAN.addTag(new PosTag("NM", LexicalCategory.Noun));
- ITALIAN.addTag(new PosTag("ADJ", LexicalCategory.Adjective));
- ITALIAN.addTag(new PosTag("ADV", LexicalCategory.Adverb));
- ITALIAN.addTag(new PosTag("ART", LexicalCategory.PronounOrDeterminer));
- ITALIAN.addTag(new PosTag("PRON", LexicalCategory.PronounOrDeterminer));
- ITALIAN.addTag(new PosTag("PRONWH", LexicalCategory.PronounOrDeterminer));
- ITALIAN.addTag(new PosTag("DETDEMO", LexicalCategory.PronounOrDeterminer));
- ITALIAN.addTag(new PosTag("DETINDEF", LexicalCategory.PronounOrDeterminer));
- ITALIAN.addTag(new PosTag("DETPOSS", LexicalCategory.PronounOrDeterminer));
- ITALIAN.addTag(new PosTag("DETWH", LexicalCategory.PronounOrDeterminer));
- ITALIAN.addTag(new PosTag("CHEWH", LexicalCategory.PronounOrDeterminer));
- ITALIAN.addTag(new PosTag("CLI", LexicalCategory.Clitic));
- ITALIAN.addTag(new PosTag("CONJ", LexicalCategory.Conjuction));
- ITALIAN.addTag(new PosTag("CONGWH", LexicalCategory.Conjuction));
- ITALIAN.addTag(new PosTag("PREP", LexicalCategory.Adposition));
- ITALIAN.addTag(new PosTag("V", LexicalCategory.Verb));
- ITALIAN.addTag(new PosTag("INT", LexicalCategory.Interjection));
- ITALIAN.addTag(new PosTag("NEG", LexicalCategory.Adverb));
- ITALIAN.addTag(new PosTag("NUM", LexicalCategory.Numeral));
- getInstance().addPosTagset(ITALIAN);
- }
-
-
- public static final TagSet<PosTag> GERMAN = new TagSet<PosTag>("CELI German POS tags", "de");
- static {
- GERMAN.addTag(new PosTag("N", LexicalCategory.Noun));
- GERMAN.addTag(new PosTag("ADJ", LexicalCategory.Adjective));
- GERMAN.addTag(new PosTag("ADV", LexicalCategory.Adverb));
- GERMAN.addTag(new PosTag("ART", LexicalCategory.PronounOrDeterminer));
- GERMAN.addTag(new PosTag("PRON", LexicalCategory.PronounOrDeterminer));
- GERMAN.addTag(new PosTag("CONJ", LexicalCategory.Conjuction));
- GERMAN.addTag(new PosTag("PREP", LexicalCategory.Adposition));
- GERMAN.addTag(new PosTag("PREPART", LexicalCategory.Adposition));
- GERMAN.addTag(new PosTag("V", LexicalCategory.Verb));
- GERMAN.addTag(new PosTag("INT", LexicalCategory.Interjection));
- GERMAN.addTag(new PosTag("NUM", LexicalCategory.Numeral));
- getInstance().addPosTagset(GERMAN);
- }
-
- public static final TagSet<PosTag> DANISH = new TagSet<PosTag>("CELI Danish POS tags", "da");
- static {
- DANISH.addTag(new PosTag("N", LexicalCategory.Noun));
- DANISH.addTag(new PosTag("NF", LexicalCategory.Noun));
- DANISH.addTag(new PosTag("NN", LexicalCategory.Noun));
- DANISH.addTag(new PosTag("NP", LexicalCategory.ProperNoun));
- DANISH.addTag(new PosTag("ADJ", LexicalCategory.Adjective));
- DANISH.addTag(new PosTag("ADV", LexicalCategory.Adverb));
- DANISH.addTag(new PosTag("ART", LexicalCategory.PronounOrDeterminer));
- DANISH.addTag(new PosTag("PRON", LexicalCategory.PronounOrDeterminer));
- DANISH.addTag(new PosTag("CONJ", LexicalCategory.Conjuction));
- DANISH.addTag(new PosTag("PREP", LexicalCategory.Adposition));
- DANISH.addTag(new PosTag("V", LexicalCategory.Verb));
- DANISH.addTag(new PosTag("INT", LexicalCategory.Interjection));
- DANISH.addTag(new PosTag("INVAR", LexicalCategory.Residual));
- DANISH.addTag(new PosTag("NUM", LexicalCategory.Numeral));
- getInstance().addPosTagset(DANISH);
- }
-
- public static final TagSet<PosTag> RUSSIAN = new TagSet<PosTag>("CELI Russian POS tags", "ru");
- static {
- RUSSIAN.addTag(new PosTag("N", LexicalCategory.Noun));
- RUSSIAN.addTag(new PosTag("NF", LexicalCategory.Noun));
- RUSSIAN.addTag(new PosTag("NM", LexicalCategory.Noun));
- RUSSIAN.addTag(new PosTag("NN", LexicalCategory.Noun));
- RUSSIAN.addTag(new PosTag("NP", LexicalCategory.ProperNoun));
- RUSSIAN.addTag(new PosTag("NPLUR", LexicalCategory.Noun));
- RUSSIAN.addTag(new PosTag("ADJ", LexicalCategory.Adjective));
- RUSSIAN.addTag(new PosTag("ADV", LexicalCategory.Adverb));
- RUSSIAN.addTag(new PosTag("PRON", LexicalCategory.PronounOrDeterminer));
- RUSSIAN.addTag(new PosTag("CONJ", LexicalCategory.Conjuction));
- RUSSIAN.addTag(new PosTag("PREP", LexicalCategory.Adposition));
- RUSSIAN.addTag(new PosTag("V", LexicalCategory.Verb));
- RUSSIAN.addTag(new PosTag("INT", LexicalCategory.Interjection));
- RUSSIAN.addTag(new PosTag("NUM", LexicalCategory.Numeral));
- RUSSIAN.addTag(new PosTag("NUMNON", LexicalCategory.PronounOrDeterminer));
- RUSSIAN.addTag(new PosTag("PART", LexicalCategory.Unique));
- getInstance().addPosTagset(RUSSIAN);
- }
-
- public static final TagSet<PosTag> ROMANIAN = new TagSet<PosTag>("CELI Romanian POS tags", "ro");
- static {
- ROMANIAN.addTag(new PosTag("N", LexicalCategory.Noun));
- ROMANIAN.addTag(new PosTag("AJ", LexicalCategory.Adjective));
- ROMANIAN.addTag(new PosTag("AV", LexicalCategory.Adverb));
- ROMANIAN.addTag(new PosTag("AT", LexicalCategory.PronounOrDeterminer));
- ROMANIAN.addTag(new PosTag("PD", LexicalCategory.PronounOrDeterminer));
- ROMANIAN.addTag(new PosTag("C", LexicalCategory.Conjuction));
- ROMANIAN.addTag(new PosTag("AP", LexicalCategory.Adposition));
- ROMANIAN.addTag(new PosTag("V", LexicalCategory.Verb));
- ROMANIAN.addTag(new PosTag("I", LexicalCategory.Interjection));
- ROMANIAN.addTag(new PosTag("INVAR", LexicalCategory.Residual));
- ROMANIAN.addTag(new PosTag("NU", LexicalCategory.Numeral));
- ROMANIAN.addTag(new PosTag("R", LexicalCategory.Residual));
- ROMANIAN.addTag(new PosTag("U", LexicalCategory.Unique));
- getInstance().addPosTagset(ROMANIAN);
- }
-
+ * Getter for a {@link VerbMoodTag} based on the Registry configuration
+ *
+ * @param language
+ * the language
+ * @param tag
+ * the {@link String} tag as returned by CELI
+ * @return the {@link VerbMoodTag}
+ */
+ public VerbMoodTag getVerbMoodTag(String language, String tag) {
+ return getTag(verbFormMappingsByLanguage, unmappedVerbMoodTagsByLanguage, VerbMoodTag.class,
+ language, tag);
+ }
+
+ /**
+ * Setter for the mappings of {@link TagSet} by language.
+ *
+ * @param mappings
+ * expressed with a {@link TagSet}
+ */
+ private void addTenseTagset(TagSet<TenseTag> model) {
+ for (String lang : model.getLanguages()) {
+ if (tenseMappingsByLanguage.put(lang, model) != null) {
+ throw new IllegalStateException("Multiple Models for Language '" + lang
+ + "'! This is an error in the static confituration of "
+ + "this class. Please report this to the stanbol-dev mailing"
+ + "list!");
+ }
+ }
+ }
+
+ /**
+ * Getter for a {@link TenseTag} based on the Registry configuration
+ *
+ * @param language
+ * the language
+ * @param tag
+ * the {@link String} tag as returned by CELI
+ * @return the {@link TenseTag}
+ */
+ public TenseTag getTenseTag(String language, String tag) {
+ return getTag(tenseMappingsByLanguage, unmappedTenseTagsByLanguage, TenseTag.class, language, tag);
+ }
+
+ /**
+ * Utility that uses generics and reflection to
+ * <ul>
+ * <li>lookup mapped {@link Tag}s
+ * <li>lookup already created unmapped {@link Tag}s
+ * <li>create unmapped {@link Tag}s for {@link String} tags that are encountered the first time
+ * </ul>
+ *
+ * @param tagSets
+ * the {@link TagSet} with the mapped Tags. The key represents the language
+ * @param unmapped
+ * unmapped {@link Tag}s. The key of the outer map is the language. The key of the inner map is
+ * the {@link String} tag.
+ * @param clazz
+ * the {@link Class} of the {@link Tag}. Used to create an instance via reflection.
+ * @param language
+ * the language
+ * @param tag
+ * the string tag
+ * @return the Tag
+ */
+ private static <T extends Tag<T>> T getTag(Map<String,TagSet<T>> tagSets,
+ Map<String,Map<String,T>> unmapped,
+ Class<T> clazz,
+ String language,
+ String tag) {
+ T t = null;
+ TagSet<T> tagSet = tagSets.get(language);
+ if (tagSet != null) {
+ t = tagSet.getTag(tag);
+ }
+ if (t == null) {
+ // warn about missing mappings for Tags to Olia!
+ log.warn("Unrecoquniced {} '{}' for Language {}", new Object[] {clazz.getSimpleName(), tag,
+ language});
+ Map<String,T> u = unmapped.get(language);
+ if (u == null) {
+ u = new HashMap<String,T>();
+ unmapped.put(language, u);
+ }
+ t = u.get(tag);
+ if (t == null) {
+ try {
+ t = clazz.getConstructor(String.class).newInstance(tag);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalStateException("Unable to instantiate " + clazz.getSimpleName()
+ + "with String tag '" + tag + "'!", e);
+ } catch (SecurityException e) {
+ throw new IllegalStateException("Unable to instantiate " + clazz.getSimpleName()
+ + "with String tag '" + tag + "'!", e);
+ } catch (InstantiationException e) {
+ throw new IllegalStateException("Unable to instantiate " + clazz.getSimpleName()
+ + "with String tag '" + tag + "'!", e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalStateException("Unable to instantiate " + clazz.getSimpleName()
+ + "with String tag '" + tag + "'!", e);
+ } catch (InvocationTargetException e) {
+ throw new IllegalStateException("Unable to instantiate " + clazz.getSimpleName()
+ + "with String tag '" + tag + "'!", e);
+ } catch (NoSuchMethodException e) {
+ throw new IllegalStateException("Unable to instantiate " + clazz.getSimpleName()
+ + "with String tag '" + tag + "'!", e);
+ }
+ u.put(tag, t);
+ }
+ }
+ return t;
+ }
+
+ /*****************************************************************
+ * POS TAGSETS MAPPINGS TO OLIA ONTOLOGY *
+ ****************************************************************/
+
+ public static final TagSet<PosTag> ITALIAN = new TagSet<PosTag>("CELI Italian POS tags", "it");
+ static {
+ ITALIAN.addTag(new PosTag("N", LexicalCategory.Noun));
+ ITALIAN.addTag(new PosTag("NF", LexicalCategory.Noun));
+ ITALIAN.addTag(new PosTag("NM", LexicalCategory.Noun));
+ ITALIAN.addTag(new PosTag("ADJ", LexicalCategory.Adjective));
+ ITALIAN.addTag(new PosTag("ADV", LexicalCategory.Adverb));
+ ITALIAN.addTag(new PosTag("ART", LexicalCategory.PronounOrDeterminer));
+ ITALIAN.addTag(new PosTag("PRON", LexicalCategory.PronounOrDeterminer));
+ ITALIAN.addTag(new PosTag("PRONWH", LexicalCategory.PronounOrDeterminer));
+ ITALIAN.addTag(new PosTag("DETDEMO", LexicalCategory.PronounOrDeterminer));
+ ITALIAN.addTag(new PosTag("DETINDEF", LexicalCategory.PronounOrDeterminer));
+ ITALIAN.addTag(new PosTag("DETPOSS", LexicalCategory.PronounOrDeterminer));
+ ITALIAN.addTag(new PosTag("DETWH", LexicalCategory.PronounOrDeterminer));
+ ITALIAN.addTag(new PosTag("CHEWH", LexicalCategory.PronounOrDeterminer));
+ ITALIAN.addTag(new PosTag("CLI", LexicalCategory.Clitic));
+ ITALIAN.addTag(new PosTag("CONJ", LexicalCategory.Conjuction));
+ ITALIAN.addTag(new PosTag("CONGWH", LexicalCategory.Conjuction));
+ ITALIAN.addTag(new PosTag("PREP", LexicalCategory.Adposition));
+ ITALIAN.addTag(new PosTag("V", LexicalCategory.Verb));
+ ITALIAN.addTag(new PosTag("INT", LexicalCategory.Interjection));
+ ITALIAN.addTag(new PosTag("NEG", LexicalCategory.Adverb));
+ ITALIAN.addTag(new PosTag("NUM", LexicalCategory.Numeral));
+ getInstance().addPosTagset(ITALIAN);
+ }
+
+ public static final TagSet<PosTag> GERMAN = new TagSet<PosTag>("CELI German POS tags", "de");
+ static {
+ GERMAN.addTag(new PosTag("N", LexicalCategory.Noun));
+ GERMAN.addTag(new PosTag("ADJ", LexicalCategory.Adjective));
+ GERMAN.addTag(new PosTag("ADV", LexicalCategory.Adverb));
+ GERMAN.addTag(new PosTag("ART", LexicalCategory.PronounOrDeterminer));
+ GERMAN.addTag(new PosTag("PRON", LexicalCategory.PronounOrDeterminer));
+ GERMAN.addTag(new PosTag("CONJ", LexicalCategory.Conjuction));
+ GERMAN.addTag(new PosTag("PREP", LexicalCategory.Adposition));
+ GERMAN.addTag(new PosTag("PREPART", LexicalCategory.Adposition));
+ GERMAN.addTag(new PosTag("V", LexicalCategory.Verb));
+ GERMAN.addTag(new PosTag("INT", LexicalCategory.Interjection));
+ GERMAN.addTag(new PosTag("NUM", LexicalCategory.Numeral));
+ getInstance().addPosTagset(GERMAN);
+ }
+
+ public static final TagSet<PosTag> DANISH = new TagSet<PosTag>("CELI Danish POS tags", "da");
+ static {
+ DANISH.addTag(new PosTag("N", LexicalCategory.Noun));
+ DANISH.addTag(new PosTag("NF", LexicalCategory.Noun));
+ DANISH.addTag(new PosTag("NN", LexicalCategory.Noun));
+ DANISH.addTag(new PosTag("NP", LexicalCategory.ProperNoun));
+ DANISH.addTag(new PosTag("ADJ", LexicalCategory.Adjective));
+ DANISH.addTag(new PosTag("ADV", LexicalCategory.Adverb));
+ DANISH.addTag(new PosTag("ART", LexicalCategory.PronounOrDeterminer));
+ DANISH.addTag(new PosTag("PRON", LexicalCategory.PronounOrDeterminer));
+ DANISH.addTag(new PosTag("CONJ", LexicalCategory.Conjuction));
+ DANISH.addTag(new PosTag("PREP", LexicalCategory.Adposition));
+ DANISH.addTag(new PosTag("V", LexicalCategory.Verb));
+ DANISH.addTag(new PosTag("INT", LexicalCategory.Interjection));
+ DANISH.addTag(new PosTag("INVAR", LexicalCategory.Residual));
+ DANISH.addTag(new PosTag("NUM", LexicalCategory.Numeral));
+ getInstance().addPosTagset(DANISH);
+ }
+
+ public static final TagSet<PosTag> RUSSIAN = new TagSet<PosTag>("CELI Russian POS tags", "ru");
+ static {
+ RUSSIAN.addTag(new PosTag("N", LexicalCategory.Noun));
+ RUSSIAN.addTag(new PosTag("NF", LexicalCategory.Noun));
+ RUSSIAN.addTag(new PosTag("NM", LexicalCategory.Noun));
+ RUSSIAN.addTag(new PosTag("NN", LexicalCategory.Noun));
+ RUSSIAN.addTag(new PosTag("NP", LexicalCategory.ProperNoun));
+ RUSSIAN.addTag(new PosTag("NPLUR", LexicalCategory.Noun));
+ RUSSIAN.addTag(new PosTag("ADJ", LexicalCategory.Adjective));
+ RUSSIAN.addTag(new PosTag("ADV", LexicalCategory.Adverb));
+ RUSSIAN.addTag(new PosTag("PRON", LexicalCategory.PronounOrDeterminer));
+ RUSSIAN.addTag(new PosTag("CONJ", LexicalCategory.Conjuction));
+ RUSSIAN.addTag(new PosTag("PREP", LexicalCategory.Adposition));
+ RUSSIAN.addTag(new PosTag("V", LexicalCategory.Verb));
+ RUSSIAN.addTag(new PosTag("INT", LexicalCategory.Interjection));
+ RUSSIAN.addTag(new PosTag("NUM", LexicalCategory.Numeral));
+ RUSSIAN.addTag(new PosTag("NUMNON", LexicalCategory.PronounOrDeterminer));
+ RUSSIAN.addTag(new PosTag("PART", LexicalCategory.Unique));
+ getInstance().addPosTagset(RUSSIAN);
+ }
+
+ public static final TagSet<PosTag> ROMANIAN = new TagSet<PosTag>("CELI Romanian POS tags", "ro");
+ static {
+ ROMANIAN.addTag(new PosTag("N", LexicalCategory.Noun));
+ ROMANIAN.addTag(new PosTag("AJ", LexicalCategory.Adjective));
+ ROMANIAN.addTag(new PosTag("AV", LexicalCategory.Adverb));
+ ROMANIAN.addTag(new PosTag("AT", LexicalCategory.PronounOrDeterminer));
+ ROMANIAN.addTag(new PosTag("PD", LexicalCategory.PronounOrDeterminer));
+ ROMANIAN.addTag(new PosTag("C", LexicalCategory.Conjuction));
+ ROMANIAN.addTag(new PosTag("AP", LexicalCategory.Adposition));
+ ROMANIAN.addTag(new PosTag("V", LexicalCategory.Verb));
+ ROMANIAN.addTag(new PosTag("I", LexicalCategory.Interjection));
+ ROMANIAN.addTag(new PosTag("INVAR", LexicalCategory.Residual));
+ ROMANIAN.addTag(new PosTag("NU", LexicalCategory.Numeral));
+ ROMANIAN.addTag(new PosTag("R", LexicalCategory.Residual));
+ ROMANIAN.addTag(new PosTag("U", LexicalCategory.Unique));
+ getInstance().addPosTagset(ROMANIAN);
+ }
+
/*****************************************************************
- * MORPHOLOGICAL FEATURES TAGSETS MAPPINGS TO OLIA ONTOLOGY:
- * GENDER, NUMBER, PERSON, CASE, DEFINITENESS, VERB_FORM, TENSE
- ****************************************************************/
-
- public static final TagSet<GenderTag> GENDER = new TagSet<GenderTag>("CELI GENDER tags", "da","de","it","ro","ru");
- static {
- GENDER.addTag(new GenderTag("F", Gender.Feminine));
- GENDER.addTag(new GenderTag("FEM", Gender.Feminine));
- GENDER.addTag(new GenderTag("M", Gender.Masculine));
- GENDER.addTag(new GenderTag("MAS", Gender.Masculine));
- GENDER.addTag(new GenderTag("MASC", Gender.Masculine));
- GENDER.addTag(new GenderTag("NE", Gender.Neuter));
- GENDER.addTag(new GenderTag("NEU", Gender.Neuter));
- GENDER.addTag(new GenderTag("UTR", Gender.Common));
- getInstance().addGenderTagset(GENDER);
- }
-
-
- public static final TagSet<NumberTag> NUMBER = new TagSet<NumberTag>("CELI NUMBER tags", "da","de","it","ro","ru");
- static {
- NUMBER.addTag(new NumberTag("SGL", NumberFeature.Singular));
- NUMBER.addTag(new NumberTag("SIN", NumberFeature.Singular));
- NUMBER.addTag(new NumberTag("SING", NumberFeature.Singular));
- NUMBER.addTag(new NumberTag("PLU", NumberFeature.Plural));
- getInstance().addNumberTagset(NUMBER);
- }
-
- public static final TagSet<PersonTag> PERSON = new TagSet<PersonTag>("CELI PERSON tags", "da","de","it","ro","ru");
- static {
- PERSON.addTag(new PersonTag("FIRST", Person.First));
- PERSON.addTag(new PersonTag("SECOND", Person.Second));
- PERSON.addTag(new PersonTag("THIRD", Person.Third));
- getInstance().addPersonTagset(PERSON);
- }
-
- public static final TagSet<CaseTag> CASE = new TagSet<CaseTag>("CELI CASE tags", "da","de","it","ro","ru");
- static {
- CASE.addTag(new CaseTag("NOMORPH", Case.Uninflected));
- CASE.addTag(new CaseTag("NOM", Case.Nominative));
- CASE.addTag(new CaseTag("GEN", Case.Genitive));
- CASE.addTag(new CaseTag("GEN2", Case.Genitive));
- CASE.addTag(new CaseTag("ACC", Case.Accusative));
- CASE.addTag(new CaseTag("ACCAN", Case.Accusative));
- CASE.addTag(new CaseTag("ACCNAN", Case.Accusative));
- CASE.addTag(new CaseTag("ACC2", Case.Accusative));
- CASE.addTag(new CaseTag("DAT", Case.Dative));
- CASE.addTag(new CaseTag("DAT22", Case.Dative));
- CASE.addTag(new CaseTag("INS", Case.Instrumental));
- CASE.addTag(new CaseTag("INS2", Case.Instrumental));
- CASE.addTag(new CaseTag("LOC", Case.Locative));
- getInstance().addCaseTagset(CASE);
- }
-
- public static final TagSet<DefinitnessTag> DEFINITNESS = new TagSet<DefinitnessTag>("CELI DEFINITNESS tags", "da","de","it","ro","ru");
- static {
- DEFINITNESS.addTag(new DefinitnessTag("DEF", Definitness.Definite));
- DEFINITNESS.addTag(new DefinitnessTag("INDEF", Definitness.Indefinite));
- getInstance().addDefinitnessTagset(DEFINITNESS);
- }
-
- public static final TagSet<VerbMoodTag> VERB_FORM = new TagSet<VerbMoodTag>("CELI VERB FORM tags", "da","de","it","ro","ru");
- static {
- VERB_FORM.addTag(new VerbMoodTag("GERUND", VerbMood.Gerund));
- VERB_FORM.addTag(new VerbMoodTag("GEROUNDPRS", VerbMood.Gerund));//
- VERB_FORM.addTag(new VerbMoodTag("IMPERATIVE", VerbMood.ImperativeVerb));
- VERB_FORM.addTag(new VerbMoodTag("IMP", VerbMood.ImperativeVerb));
- VERB_FORM.addTag(new VerbMoodTag("INDIC", VerbMood.IndicativeVerb));
- VERB_FORM.addTag(new VerbMoodTag("IND", VerbMood.IndicativeVerb));
- VERB_FORM.addTag(new VerbMoodTag("CONGIUNT", VerbMood.SubjunctiveVerb));
- VERB_FORM.addTag(new VerbMoodTag("SUBJ", VerbMood.SubjunctiveVerb));
- VERB_FORM.addTag(new VerbMoodTag("SUB", VerbMood.SubjunctiveVerb));
- VERB_FORM.addTag(new VerbMoodTag("INFIN", VerbMood.Infinitive));
- VERB_FORM.addTag(new VerbMoodTag("INF", VerbMood.Infinitive));
- VERB_FORM.addTag(new VerbMoodTag("PASTPART", VerbMood.Participle));
- VERB_FORM.addTag(new VerbMoodTag("PASPART", VerbMood.Participle));
- VERB_FORM.addTag(new VerbMoodTag("PCPRF", VerbMood.Participle));
- VERB_FORM.addTag(new VerbMoodTag("PRESPART", VerbMood.Participle));
- VERB_FORM.addTag(new VerbMoodTag("PRSPART", VerbMood.Participle));
- VERB_FORM.addTag(new VerbMoodTag("PCPRS", VerbMood.Participle));
- VERB_FORM.addTag(new VerbMoodTag("PART", VerbMood.Participle));
- VERB_FORM.addTag(new VerbMoodTag("PTC", VerbMood.Participle));
- VERB_FORM.addTag(new VerbMoodTag("CONDIZ", VerbMood.ConditionalVerb));
- VERB_FORM.addTag(new VerbMoodTag("SUP", VerbMood.Supine));
- getInstance().addVerbFormTagset(VERB_FORM);
- }
-
-
-
- public static final TagSet<TenseTag> TENSE = new TagSet<TenseTag>("CELI TENSE tags", "da","de","it","ro","ru");
- static {
- TENSE.addTag(new TenseTag("PRS", Tense.Present));
- TENSE.addTag(new TenseTag("PRES", Tense.Present));
- TENSE.addTag(new TenseTag("IMPER", Tense.Imperfect));
- TENSE.addTag(new TenseTag("PER", Tense.Perfect));
- TENSE.addTag(new TenseTag("PASSREM", Tense.RemotePast));
- TENSE.addTag(new TenseTag("PSTPER", Tense.PastPerfect));
- TENSE.addTag(new TenseTag("PST", Tense.Past));
- TENSE.addTag(new TenseTag("FUT", Tense.Future));
- TENSE.addTag(new TenseTag("PCPRF", Tense.Past));
- TENSE.addTag(new TenseTag("PCPRS", Tense.Present));
- TENSE.addTag(new TenseTag("PRT", Tense.Past));
- TENSE.addTag(new TenseTag("PRSFUT", Tense.Present));
- getInstance().addTenseTagset(TENSE);
- }
-
+ * MORPHOLOGICAL FEATURES TAGSETS MAPPINGS TO OLIA ONTOLOGY: GENDER, NUMBER, PERSON, CASE, DEFINITENESS,
+ * VERB_FORM, TENSE
+ ****************************************************************/
+
+ public static final TagSet<GenderTag> GENDER = new TagSet<GenderTag>("CELI GENDER tags", "da", "de",
+ "it", "ro", "ru");
+ static {
+ GENDER.addTag(new GenderTag("F", Gender.Feminine));
+ GENDER.addTag(new GenderTag("FEM", Gender.Feminine));
+ GENDER.addTag(new GenderTag("M", Gender.Masculine));
+ GENDER.addTag(new GenderTag("MAS", Gender.Masculine));
+ GENDER.addTag(new GenderTag("MASC", Gender.Masculine));
+ GENDER.addTag(new GenderTag("NE", Gender.Neuter));
+ GENDER.addTag(new GenderTag("NEU", Gender.Neuter));
+ GENDER.addTag(new GenderTag("UTR", Gender.Common));
+ getInstance().addGenderTagset(GENDER);
+ }
+
+ public static final TagSet<NumberTag> NUMBER = new TagSet<NumberTag>("CELI NUMBER tags", "da", "de",
+ "it", "ro", "ru");
+ static {
+ NUMBER.addTag(new NumberTag("SGL", NumberFeature.Singular));
+ NUMBER.addTag(new NumberTag("SIN", NumberFeature.Singular));
+ NUMBER.addTag(new NumberTag("SING", NumberFeature.Singular));
+ NUMBER.addTag(new NumberTag("PLU", NumberFeature.Plural));
+ getInstance().addNumberTagset(NUMBER);
+ }
+
+ public static final TagSet<PersonTag> PERSON = new TagSet<PersonTag>("CELI PERSON tags", "da", "de",
+ "it", "ro", "ru");
+ static {
+ PERSON.addTag(new PersonTag("FIRST", Person.First));
+ PERSON.addTag(new PersonTag("SECOND", Person.Second));
+ PERSON.addTag(new PersonTag("THIRD", Person.Third));
+ getInstance().addPersonTagset(PERSON);
+ }
+
+ public static final TagSet<CaseTag> CASE = new TagSet<CaseTag>("CELI CASE tags", "da", "de", "it", "ro",
+ "ru");
+ static {
+ CASE.addTag(new CaseTag("NOMORPH", Case.Uninflected));
+ CASE.addTag(new CaseTag("NOM", Case.Nominative));
+ CASE.addTag(new CaseTag("GEN", Case.Genitive));
+ CASE.addTag(new CaseTag("GEN2", Case.Genitive));
+ CASE.addTag(new CaseTag("ACC", Case.Accusative));
+ CASE.addTag(new CaseTag("ACCAN", Case.Accusative));
+ CASE.addTag(new CaseTag("ACCNAN", Case.Accusative));
+ CASE.addTag(new CaseTag("ACC2", Case.Accusative));
+ CASE.addTag(new CaseTag("DAT", Case.Dative));
+ CASE.addTag(new CaseTag("DAT22", Case.Dative));
+ CASE.addTag(new CaseTag("INS", Case.Instrumental));
+ CASE.addTag(new CaseTag("INS2", Case.Instrumental));
+ CASE.addTag(new CaseTag("LOC", Case.Locative));
+ getInstance().addCaseTagset(CASE);
+ }
+
+ public static final TagSet<DefinitnessTag> DEFINITNESS = new TagSet<DefinitnessTag>(
+ "CELI DEFINITNESS tags", "da", "de", "it", "ro", "ru");
+ static {
+ DEFINITNESS.addTag(new DefinitnessTag("DEF", Definitness.Definite));
+ DEFINITNESS.addTag(new DefinitnessTag("INDEF", Definitness.Indefinite));
+ getInstance().addDefinitnessTagset(DEFINITNESS);
+ }
+
+ public static final TagSet<VerbMoodTag> VERB_FORM = new TagSet<VerbMoodTag>("CELI VERB FORM tags", "da",
+ "de", "it", "ro", "ru");
+ static {
+ VERB_FORM.addTag(new VerbMoodTag("GERUND", VerbMood.Gerund));
+ VERB_FORM.addTag(new VerbMoodTag("GEROUNDPRS", VerbMood.Gerund));//
+ VERB_FORM.addTag(new VerbMoodTag("IMPERATIVE", VerbMood.ImperativeVerb));
+ VERB_FORM.addTag(new VerbMoodTag("IMP", VerbMood.ImperativeVerb));
+ VERB_FORM.addTag(new VerbMoodTag("INDIC", VerbMood.IndicativeVerb));
+ VERB_FORM.addTag(new VerbMoodTag("IND", VerbMood.IndicativeVerb));
+ VERB_FORM.addTag(new VerbMoodTag("CONGIUNT", VerbMood.SubjunctiveVerb));
+ VERB_FORM.addTag(new VerbMoodTag("SUBJ", VerbMood.SubjunctiveVerb));
+ VERB_FORM.addTag(new VerbMoodTag("SUB", VerbMood.SubjunctiveVerb));
+ VERB_FORM.addTag(new VerbMoodTag("INFIN", VerbMood.Infinitive));
+ VERB_FORM.addTag(new VerbMoodTag("INF", VerbMood.Infinitive));
+ VERB_FORM.addTag(new VerbMoodTag("PASTPART", VerbMood.Participle));
+ VERB_FORM.addTag(new VerbMoodTag("PASPART", VerbMood.Participle));
+ VERB_FORM.addTag(new VerbMoodTag("PCPRF", VerbMood.Participle));
+ VERB_FORM.addTag(new VerbMoodTag("PRESPART", VerbMood.Participle));
+ VERB_FORM.addTag(new VerbMoodTag("PRSPART", VerbMood.Participle));
+ VERB_FORM.addTag(new VerbMoodTag("PCPRS", VerbMood.Participle));
+ VERB_FORM.addTag(new VerbMoodTag("PART", VerbMood.Participle));
+ VERB_FORM.addTag(new VerbMoodTag("PTC", VerbMood.Participle));
+ VERB_FORM.addTag(new VerbMoodTag("CONDIZ", VerbMood.ConditionalVerb));
+ VERB_FORM.addTag(new VerbMoodTag("SUP", VerbMood.Supine));
+ getInstance().addVerbFormTagset(VERB_FORM);
+ }
+
+ public static final TagSet<TenseTag> TENSE = new TagSet<TenseTag>("CELI TENSE tags", "da", "de", "it",
+ "ro", "ru");
+ static {
+ TENSE.addTag(new TenseTag("PRS", Tense.Present));
+ TENSE.addTag(new TenseTag("PRES", Tense.Present));
+ TENSE.addTag(new TenseTag("IMPER", Tense.Imperfect));
+ TENSE.addTag(new TenseTag("PER", Tense.Perfect));
+ TENSE.addTag(new TenseTag("PASSREM", Tense.RemotePast));
+ TENSE.addTag(new TenseTag("PSTPER", Tense.PastPerfect));
+ TENSE.addTag(new TenseTag("PST", Tense.Past));
+ TENSE.addTag(new TenseTag("FUT", Tense.Future));
+ TENSE.addTag(new TenseTag("PCPRF", Tense.Past));
+ TENSE.addTag(new TenseTag("PCPRS", Tense.Present));
+ TENSE.addTag(new TenseTag("PRT", Tense.Past));
+ TENSE.addTag(new TenseTag("PRSFUT", Tense.Present));
+ getInstance().addTenseTagset(TENSE);
+ }
+
}
\ No newline at end of file
Added: stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliAnalyzedTextLemmatizerEngine.java
URL: http://svn.apache.org/viewvc/stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliAnalyzedTextLemmatizerEngine.java?rev=1400003&view=auto
==============================================================================
--- stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliAnalyzedTextLemmatizerEngine.java (added)
+++ stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliAnalyzedTextLemmatizerEngine.java Fri Oct 19 08:54:52 2012
@@ -0,0 +1,178 @@
+package org.apache.stanbol.enhancer.engines.celi.lemmatizer.impl;
+
+import static org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper.getAnalysedText;
+import static org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper.getLanguage;
+import static org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper.isLangaugeConfigured;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.Collections;
+import java.util.Dictionary;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.soap.SOAPException;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Properties;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.Service;
+import org.apache.stanbol.commons.stanboltools.offline.OnlineMode;
+import org.apache.stanbol.enhancer.engines.celi.CeliConstants;
+import org.apache.stanbol.enhancer.engines.celi.CeliMorphoFeatures;
+import org.apache.stanbol.enhancer.engines.celi.utils.Utils;
+import org.apache.stanbol.enhancer.nlp.NlpAnnotations;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedText;
+import org.apache.stanbol.enhancer.nlp.model.Token;
+import org.apache.stanbol.enhancer.nlp.model.annotation.Value;
+import org.apache.stanbol.enhancer.nlp.morpho.MorphoFeatures;
+import org.apache.stanbol.enhancer.nlp.pos.LexicalCategory;
+import org.apache.stanbol.enhancer.nlp.pos.PosTag;
+import org.apache.stanbol.enhancer.nlp.utils.LanguageConfiguration;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.enhancer.servicesapi.EngineException;
+import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.ServiceProperties;
+import org.apache.stanbol.enhancer.servicesapi.impl.AbstractEnhancementEngine;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+
+@Component(immediate = true, metatype = true)
+@Service
+@Properties(value = {
+ @Property(name = EnhancementEngine.PROPERTY_NAME, value = "celiLemmatizer"),
+ @Property(name = CeliConstants.CELI_LICENSE),
+ @Property(name = CeliConstants.CELI_TEST_ACCOUNT, boolValue = false) })
+public class CeliAnalyzedTextLemmatizerEngine extends AbstractEnhancementEngine<IOException, RuntimeException> implements EnhancementEngine, ServiceProperties {
+
+
+ /**
+ * This ensures that no connections to external services are made if Stanbol is started in offline mode as the OnlineMode service will only be available if OfflineMode is deactivated.
+ */
+ @SuppressWarnings("unused")
+ @Reference
+ private OnlineMode onlineMode;
+
+ @Property(value = "http://linguagrid.org/LSGrid/ws/morpho-analyser")
+ public static final String SERVICE_URL = "org.apache.stanbol.enhancer.engines.celi.lemmatizer.url";
+
+ @Property(value={"it", "da", "de", "ru","ro"})
+ public static final String PROPERTY_SUPPORTED_LANGUAGES = "org.apache.stanbol.enhancer.engines.celi.lemmatizer.languages";
+
+ /**
+ * The default value for the Execution of this Engine. Currently set to
+ * {@link ServiceProperties#ORDERING_CONTENT_EXTRACTION}
+ */
+ public static final Integer defaultOrder = ServiceProperties.ORDERING_CONTENT_EXTRACTION;
+
+ private LanguageConfiguration languageConfig = new LanguageConfiguration(
+ PROPERTY_SUPPORTED_LANGUAGES, new String[]{"it", "da", "de", "ru","ro"});
+
+ private String licenseKey;
+ private URL serviceURL;
+
+ private LemmatizerClientHTTP client;
+
+ @Override
+ @Activate
+ protected void activate(ComponentContext ctx) throws IOException, ConfigurationException {
+ super.activate(ctx);
+ Dictionary<String, Object> properties = ctx.getProperties();
+ this.licenseKey = Utils.getLicenseKey(properties, ctx.getBundleContext());
+ String url = (String) properties.get(SERVICE_URL);
+ if (url == null || url.isEmpty()) {
+ throw new ConfigurationException(SERVICE_URL, String.format("%s : please configure the URL of the CELI Web Service (e.g. by" + "using the 'Configuration' tab of the Apache Felix Web Console).", getClass().getSimpleName()));
+ }
+ this.serviceURL = new URL(url);
+ //parse the parsed language configuration
+ languageConfig.setConfiguration(properties);
+
+ this.client = new LemmatizerClientHTTP(this.serviceURL, this.licenseKey);
+ }
+
+ @Override
+ @Deactivate
+ protected void deactivate(ComponentContext ce) {
+ languageConfig.setDefault();
+ super.deactivate(ce);
+ }
+
+ @Override
+ public int canEnhance(ContentItem ci) throws EngineException {
+ String language = getLanguage(this, ci,false);
+ if(language == null){
+ return CANNOT_ENHANCE;
+ }
+ if(!isLangaugeConfigured(this,languageConfig,language,false)){
+ return CANNOT_ENHANCE;
+ }
+ if(getAnalysedText(this,ci,false) == null) {
+ return CANNOT_ENHANCE;
+ }
+ // default enhancement is synchronous enhancement
+ return ENHANCE_ASYNC;
+ }
+
+ @Override
+ public void computeEnhancements(ContentItem ci) throws EngineException {
+ AnalysedText at = getAnalysedText(this, ci, true);
+ String language = getLanguage(this, ci, true);
+ isLangaugeConfigured(this, languageConfig, language, true);
+ List<LexicalEntry> terms;
+ try {
+ terms = this.client.performMorfologicalAnalysis(at.getSpan(), language);
+ } catch (IOException e) {
+ throw new EngineException("Error while calling the CELI Lemmatizer"
+ + " service (configured URL: " + serviceURL + ")!", e);
+ } catch (SOAPException e) {
+ throw new EngineException("Error wile encoding/decoding the request/" +
+ "response to the CELI lemmatizer service!", e);
+ }
+ Map<LexicalCategory,Double> tokenLexCats = new EnumMap<LexicalCategory,Double>(LexicalCategory.class);
+ for(LexicalEntry term : terms){
+ if(term.getTermReadings().isEmpty()){
+ //TODO: maybe still add them and use the Lemmatizer as Tokenizer
+ continue; //ignore terms without readings
+ }
+ //Add the LexicalEntry as Token to the Text. NOTE that if a
+ //Token with the same start/end positions already exist this
+ //Method returns the existing instance
+ Token token = at.addToken(term.getFrom(), term.getTo());
+ //Now try to get POS annotations for the Token
+ for(Value<PosTag> posAnno : token.getAnnotations(NlpAnnotations.POS_ANNOTATION)){
+ if(posAnno.value().getCategory() != null){
+ tokenLexCats.put(posAnno.value().getCategory(), posAnno.probability());
+ }
+ }
+ for(Reading reading : term.getTermReadings()){
+ MorphoFeatures mf = CeliMorphoFeatures.parseFrom(reading, language);
+ //add the readings (MorphoFeatures)
+ if(mf != null){
+ //use the POS tags of the morpho analysis and compair it
+ //with existing POS tags.
+ double posProbability = -1;
+ for(PosTag mfPos : mf.getPosList()){
+ Double prob = tokenLexCats.get(mfPos.getCategory());
+ if(prob != null && posProbability < prob){
+ posProbability = prob;
+ }
+ }
+ //add the morpho features with the posProbabiliy
+ Value<MorphoFeatures> value = Value.value(mf,
+ posProbability < 0 ? Value.UNKNOWN_PROBABILITY : posProbability);
+ token.addAnnotation(NlpAnnotations.MORPHO_ANNOTATION, value);
+ }
+ }
+ }
+ }
+
+ @Override
+ public Map<String, Object> getServiceProperties() {
+ return Collections.unmodifiableMap(Collections.singletonMap(ENHANCEMENT_ENGINE_ORDERING, (Object) defaultOrder));
+ }
+
+}
Modified: stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliLemmatizerEnhancementEngine.java
URL: http://svn.apache.org/viewvc/stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliLemmatizerEnhancementEngine.java?rev=1400003&r1=1400002&r2=1400003&view=diff
==============================================================================
--- stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliLemmatizerEnhancementEngine.java (original)
+++ stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliLemmatizerEnhancementEngine.java Fri Oct 19 08:54:52 2012
@@ -258,57 +258,15 @@ public class CeliLemmatizerEnhancementEn
}
private List<CeliMorphoFeatures> convertLexicalEntryToMorphFeatures(LexicalEntry le, String lang) {
- CeliTagSetRegistry mappings = CeliTagSetRegistry.getInstance();
List<CeliMorphoFeatures> result = new Vector<CeliMorphoFeatures>();
if (!le.termReadings.isEmpty()) {
for (Reading r : le.termReadings) {
- CeliMorphoFeatures morphoFeature = new CeliMorphoFeatures(r.getLemma());
- for (Entry<String, List<String>> entry : r.lexicalFeatures.entrySet()) {
-
- String feature = entry.getKey();
- for (String value : entry.getValue()) {
- try {
- if (feature.equals("POS")) {
- LexicalCategory pos = mappings.getPosTagset(lang).getTag(value).getCategory();
- if (pos != null)
- morphoFeature.addPos(pos);
- } else if (feature.equals("CASE")) {
- Case c = mappings.getCaseTagset(lang).getTag(value).getCase();
- if (c != null)
- morphoFeature.addCase(c);
- } else if (feature.equals("GENDER")) {
- Gender gen = mappings.getGenderTagset(lang).getTag(value).getGender();
- if (gen != null)
- morphoFeature.addGender(gen);
- } else if (feature.equals("NUMBER")) {
- NumberFeature num = mappings.getNumberTagset(lang).getTag(value).getNumber();
- if (num != null)
- morphoFeature.addNumber(num);
- } else if (feature.equals("PERSON")) {
- Person pers = mappings.getPersonTagset(lang).getTag(value).getPerson();
- if (pers != null)
- morphoFeature.addPerson(pers);
- } else if (feature.equals("VERB_FORM") || feature.equals("VFORM")) {
- VerbMood vForm = mappings.getVerbFormTagset(lang).getTag(value).getVerbForm();
- if (vForm != null)
- morphoFeature.addVerbForm(vForm);
- } else if (feature.equals("TENSE") || feature.equals("VERB_TENSE")) {
- TagSet<TenseTag> map = mappings.getTenseTagset(lang);
- TenseTag tagg = map.getTag(value);
- Tense tense = tagg.getTense();
- if (tense != null)
- morphoFeature.addTense(tense);
- }
- } catch (Exception e) {
- System.err.println(entry.getKey() + " " + value);
- }
- }
-
+ CeliMorphoFeatures morphoFeature = CeliMorphoFeatures.parseFrom(r, lang);
+ if(morphoFeature != null){
+ result.add(morphoFeature);
}
- result.add(morphoFeature);
}
}
-
return result;
}
Modified: stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/LemmatizerClientHTTP.java
URL: http://svn.apache.org/viewvc/stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/LemmatizerClientHTTP.java?rev=1400003&r1=1400002&r2=1400003&view=diff
==============================================================================
--- stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/LemmatizerClientHTTP.java (original)
+++ stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/main/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/LemmatizerClientHTTP.java Fri Oct 19 08:54:52 2012
@@ -17,6 +17,7 @@
package org.apache.stanbol.enhancer.engines.celi.lemmatizer.impl;
import java.io.BufferedWriter;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
@@ -39,6 +40,7 @@ import javax.xml.soap.SOAPPart;
import javax.xml.transform.stream.StreamSource;
import org.apache.clerezza.rdf.core.impl.util.Base64;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.stanbol.enhancer.engines.celi.utils.Utils;
import org.slf4j.Logger;
@@ -101,7 +103,12 @@ public class LemmatizerClientHTTP {
long start = System.currentTimeMillis();
InputStream stream = con.getInputStream();
log.debug("Request to {} took {}ms",serviceEP,System.currentTimeMillis()-start);
-
+ if(log.isTraceEnabled()){
+ //log the response if trace is enabled
+ String soapResponse = IOUtils.toString(stream,"UTF-8");
+ log.trace("SoapResponse: \n{}\n",soapResponse);
+ stream = new ByteArrayInputStream(soapResponse.getBytes(Charset.forName("UTF-8")));
+ }
// Create SoapMessage
MessageFactory msgFactory = MessageFactory.newInstance();
SOAPMessage message = msgFactory.createMessage();
Added: stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/test/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliAnalyzedTextLemmatizerEngineTest.java
URL: http://svn.apache.org/viewvc/stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/test/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliAnalyzedTextLemmatizerEngineTest.java?rev=1400003&view=auto
==============================================================================
--- stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/test/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliAnalyzedTextLemmatizerEngineTest.java (added)
+++ stanbol/branches/stanbol-nlp-processing/enhancer/engines/celi/src/test/java/org/apache/stanbol/enhancer/engines/celi/lemmatizer/impl/CeliAnalyzedTextLemmatizerEngineTest.java Fri Oct 19 08:54:52 2012
@@ -0,0 +1,159 @@
+package org.apache.stanbol.enhancer.engines.celi.lemmatizer.impl;
+
+import static org.apache.stanbol.enhancer.engines.celi.lemmatizer.impl.CeliLemmatizerEnhancementEngine.MORPHOLOGICAL_ANALYSIS;
+import static org.apache.stanbol.enhancer.engines.celi.lemmatizer.impl.CeliLemmatizerEnhancementEngine.SERVICE_URL;
+import static org.apache.stanbol.enhancer.nlp.NlpAnnotations.POS_ANNOTATION;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_LANGUAGE;
+
+import java.io.IOException;
+import java.util.Dictionary;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl;
+import org.apache.clerezza.rdf.core.impl.TripleImpl;
+import org.apache.commons.logging.Log;
+import org.apache.stanbol.enhancer.contentitem.inmemory.InMemoryContentItemFactory;
+import org.apache.stanbol.enhancer.engines.celi.CeliConstants;
+import org.apache.stanbol.enhancer.engines.celi.testutils.MockComponentContext;
+import org.apache.stanbol.enhancer.nlp.NlpAnnotations;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedText;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedTextFactory;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedTextUtils;
+import org.apache.stanbol.enhancer.nlp.model.Token;
+import org.apache.stanbol.enhancer.nlp.model.annotation.Value;
+import org.apache.stanbol.enhancer.nlp.morpho.MorphoFeatures;
+import org.apache.stanbol.enhancer.nlp.pos.LexicalCategory;
+import org.apache.stanbol.enhancer.nlp.pos.PosTag;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.enhancer.servicesapi.ContentItemFactory;
+import org.apache.stanbol.enhancer.servicesapi.EngineException;
+import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
+import org.apache.stanbol.enhancer.servicesapi.impl.StringSource;
+import org.apache.stanbol.enhancer.test.helper.RemoteServiceHelper;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.osgi.service.cm.ConfigurationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CeliAnalyzedTextLemmatizerEngineTest {
+
+ private static final Logger log = LoggerFactory.getLogger(CeliAnalyzedTextLemmatizerEngineTest.class);
+
+ private static final ContentItemFactory ciFactory = InMemoryContentItemFactory.getInstance();
+ private static final AnalysedTextFactory atFactory = AnalysedTextFactory.getDefaultInstance();
+
+
+ /*
+ * Data for the GERMAN test
+ */
+ public static final String de_verb = "verbrachten";
+ public static final String de_adjective = "sensationellen"; //"schönen";
+ public static final String de_noun = "Urlaub";
+
+ public static final String de_text = String.format(
+ "Wir %s einen %s %s in der Schweiz",
+ de_verb,de_adjective,de_noun);
+
+ public static final int de_verbStart = de_text.indexOf(de_verb);
+ public static final double de_verbProb = 0.98765d;
+
+ public static final int de_adjectiveStart = de_text.indexOf(de_adjective);
+ public static final double de_adjectiveProb = 0.87654d;
+
+ public static final int de_nounStart = de_text.indexOf(de_noun);
+ public static final double de_nounProb = 0.998877d;
+
+ public static CeliAnalyzedTextLemmatizerEngine engine;
+
+ @BeforeClass
+ public static void initEngine() throws IOException, ConfigurationException {
+ Dictionary<String, Object> properties = new Hashtable<String, Object>();
+ properties.put(EnhancementEngine.PROPERTY_NAME, "celiLemmatizer");
+ properties.put(CeliConstants.CELI_TEST_ACCOUNT, "true");
+ properties.put(SERVICE_URL, "http://linguagrid.org/LSGrid/ws/morpho-analyser");
+ MockComponentContext context = new MockComponentContext(properties);
+ engine = new CeliAnalyzedTextLemmatizerEngine();
+ engine.activate(context);
+ }
+ @AfterClass
+ public static void deactivate(){
+ engine.deactivate(null);
+ engine = null;
+ }
+
+ @Test
+ public void testEngineDe() throws IOException, EngineException {
+ ContentItem ci = ciFactory.createContentItem(new StringSource(de_text));
+ Assert.assertNotNull(ci);
+ AnalysedText at = atFactory.createAnalysedText(ci, ci.getBlob());
+ Assert.assertNotNull(at);
+ ci.getMetadata().add(new TripleImpl(ci.getUri(), DC_LANGUAGE, new PlainLiteralImpl("de")));
+ Assert.assertEquals("de", EnhancementEngineHelper.getLanguage(ci));
+
+ //Add some Tokens with POS annotations to test the usage of
+ //existing POS annotations by the lemmatizer
+ Token verbrachten = at.addToken(de_verbStart,de_verbStart+de_verb.length());
+ verbrachten.addAnnotation(POS_ANNOTATION, Value.value(
+ new PosTag("V",LexicalCategory.Verb), de_verbProb));
+
+ Token schonen = at.addToken(de_adjectiveStart,de_adjectiveStart+de_adjective.length());
+ schonen.addAnnotation(POS_ANNOTATION, Value.value(
+ new PosTag("ADJ",LexicalCategory.Adjective), de_adjectiveProb));
+
+ Token urlaub = at.addToken(de_nounStart,de_nounStart+de_noun.length());
+ urlaub.addAnnotation(POS_ANNOTATION, Value.value(
+ new PosTag("NC",LexicalCategory.Noun), de_nounProb));
+
+ Assert.assertEquals("Can not enhance Test ContentItem",
+ EnhancementEngine.ENHANCE_ASYNC,engine.canEnhance(ci));
+ //compute the enhancements
+ try {
+ engine.computeEnhancements(ci);
+ } catch (EngineException e) {
+ RemoteServiceHelper.checkServiceUnavailable(e);
+ }
+ //now validate the enhancements
+ boolean foundVerbrachten = false;
+ boolean foundSchonen = false;
+ boolean foundUrlaub = false;
+ for(Iterator<Token> tokens = at.getTokens(); tokens.hasNext();){
+ Token token = tokens.next();
+ log.info("Token: {}",token);
+ List<Value<MorphoFeatures>> mfs = token.getAnnotations(NlpAnnotations.MORPHO_ANNOTATION);
+ if(de_verb.equals(token.getSpan())){
+ foundVerbrachten = !mfs.isEmpty();
+ validateMorphFeatureProbability(mfs,LexicalCategory.Verb,de_verbProb);
+ } else if(de_adjective.equals(token.getSpan())){
+ foundSchonen = !mfs.isEmpty();
+ validateMorphFeatureProbability(mfs,LexicalCategory.Adjective,de_adjectiveProb);
+ } else if(de_noun.equals(token.getSpan())){
+ foundUrlaub = !mfs.isEmpty();
+ validateMorphFeatureProbability(mfs,LexicalCategory.Noun,de_nounProb);
+ }
+ for(Value<MorphoFeatures> mf : mfs){
+ log.info(" - {}",mf);
+ Assert.assertNotNull(mf.value().getLemma());
+ }
+ }
+ Assert.assertTrue("No MorphoFeatures found for 'verbrachten'!",foundVerbrachten);
+ Assert.assertTrue("No MorphoFeatures found for 'schönen'!",foundSchonen);
+ Assert.assertTrue("No MorphoFeatures found for 'Urlaub'!",foundUrlaub);
+ }
+ private void validateMorphFeatureProbability(List<Value<MorphoFeatures>> mfs, LexicalCategory lc, double prob) {
+ for(Value<MorphoFeatures> mf : mfs){
+ for(PosTag pos : mf.value().getPosList()){
+ if(pos.getCategory() == lc){
+ Assert.assertEquals(prob, mf.probability());
+ }
+ }
+ }
+ }
+
+}