You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2012/12/09 07:30:44 UTC
svn commit: r1418822 - in /stanbol/trunk/enhancement-engines:
entitylinking/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/
langdetect/src/main/java/org/apache/stanbol/enhancer/engines/langdetect/
opennlp-chunker/src/main/java/o...
Author: rwesten
Date: Sun Dec 9 06:30:42 2012
New Revision: 1418822
URL: http://svn.apache.org/viewvc?rev=1418822&view=rev
Log:
minor: Just changed some logging levels to reduce the messages on INFO level. THe OpenNLP Chunker and Pos engine now keep ad-hoc Tags (unmapped String Tags). This is also to make loggings more meaningful as only the first time that such a Tag appear an INFO message is written to the log. The goal ist to make is more easy to find those Tags by using grebs over the log files
Modified:
stanbol/trunk/enhancement-engines/entitylinking/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngine.java
stanbol/trunk/enhancement-engines/langdetect/src/main/java/org/apache/stanbol/enhancer/engines/langdetect/LanguageDetectionEnhancementEngine.java
stanbol/trunk/enhancement-engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/OpenNlpChunkingEngine.java
stanbol/trunk/enhancement-engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java
stanbol/trunk/enhancement-engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/OpenNlpPosTaggingEngine.java
Modified: stanbol/trunk/enhancement-engines/entitylinking/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngine.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngine.java?rev=1418822&r1=1418821&r2=1418822&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngine.java (original)
+++ stanbol/trunk/enhancement-engines/entitylinking/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngine.java Sun Dec 9 06:30:42 2012
@@ -191,7 +191,7 @@ public class EntityLinkingEngine impleme
@Override
public int canEnhance(ContentItem ci) throws EngineException {
- log.info("canEnhancer {}",ci.getUri());
+ log.trace("canEnhancer {}",ci.getUri());
if(isOfflineMode() && !entitySearcher.supportsOfflineMode()){
log.warn("{} '{}' is inactive because EntitySearcher does not support Offline mode!",
getClass().getSimpleName(),getName());
@@ -212,12 +212,12 @@ public class EntityLinkingEngine impleme
@Override
public void computeEnhancements(ContentItem ci) throws EngineException {
- log.info(" enhance ci {}",ci.getUri());
+ log.trace(" enhance ci {}",ci.getUri());
if(isOfflineMode() && !entitySearcher.supportsOfflineMode()){
throw new EngineException(this,ci,"Offline mode is not supported by the used EntitySearcher!",null);
}
AnalysedText at = getAnalysedText(this, ci, true);
- log.info(" > AnalysedText {}",at);
+ log.debug(" > AnalysedText {}",at);
String language = getLanguage(this, ci, true);
if(log.isDebugEnabled()){
log.debug("computeEnhancements for ContentItem {} language {} text={}",
Modified: stanbol/trunk/enhancement-engines/langdetect/src/main/java/org/apache/stanbol/enhancer/engines/langdetect/LanguageDetectionEnhancementEngine.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/langdetect/src/main/java/org/apache/stanbol/enhancer/engines/langdetect/LanguageDetectionEnhancementEngine.java?rev=1418822&r1=1418821&r2=1418822&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/langdetect/src/main/java/org/apache/stanbol/enhancer/engines/langdetect/LanguageDetectionEnhancementEngine.java (original)
+++ stanbol/trunk/enhancement-engines/langdetect/src/main/java/org/apache/stanbol/enhancer/engines/langdetect/LanguageDetectionEnhancementEngine.java Sun Dec 9 06:30:42 2012
@@ -229,11 +229,11 @@ public class LanguageDetectionEnhancemen
List<Language> languages = null;
try {
languages = languageIdentifier.getLanguages(text);
- log.info("language identified: {}",languages);
+ log.debug("language identified: {}",languages);
}
catch (LangDetectException e) {
- log.warn("Could not identify language");
- return;
+ log.warn("Could not identify language", e);
+ throw new EngineException(this, ci, "Could not identify language", e);
}
// add language to metadata
Modified: stanbol/trunk/enhancement-engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/OpenNlpChunkingEngine.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/OpenNlpChunkingEngine.java?rev=1418822&r1=1418821&r2=1418822&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/OpenNlpChunkingEngine.java (original)
+++ stanbol/trunk/enhancement-engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/OpenNlpChunkingEngine.java Sun Dec 9 06:30:42 2012
@@ -29,15 +29,12 @@ import java.util.Collections;
import java.util.Dictionary;
import java.util.EnumSet;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import opennlp.tools.chunker.ChunkerME;
import opennlp.tools.chunker.ChunkerModel;
-import opennlp.tools.util.Sequence;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
@@ -49,11 +46,9 @@ import org.apache.felix.scr.annotations.
import org.apache.felix.scr.annotations.Service;
import org.apache.stanbol.commons.opennlp.OpenNLP;
import org.apache.stanbol.enhancer.engines.opennlp.chunker.model.PhraseTagSetRegistry;
-import org.apache.stanbol.enhancer.nlp.NlpAnnotations;
import org.apache.stanbol.enhancer.nlp.NlpProcessingRole;
import org.apache.stanbol.enhancer.nlp.NlpServiceProperties;
import org.apache.stanbol.enhancer.nlp.model.AnalysedText;
-import org.apache.stanbol.enhancer.nlp.model.AnalysedTextUtils;
import org.apache.stanbol.enhancer.nlp.model.Chunk;
import org.apache.stanbol.enhancer.nlp.model.Section;
import org.apache.stanbol.enhancer.nlp.model.Span;
@@ -64,12 +59,10 @@ import org.apache.stanbol.enhancer.nlp.m
import org.apache.stanbol.enhancer.nlp.phrase.PhraseTag;
import org.apache.stanbol.enhancer.nlp.pos.PosTag;
import org.apache.stanbol.enhancer.nlp.utils.LanguageConfiguration;
-import org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
import org.apache.stanbol.enhancer.servicesapi.EngineException;
import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
import org.apache.stanbol.enhancer.servicesapi.ServiceProperties;
-import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
import org.apache.stanbol.enhancer.servicesapi.impl.AbstractEnhancementEngine;
import org.osgi.framework.Constants;
import org.osgi.service.cm.ConfigurationException;
@@ -120,8 +113,6 @@ public class OpenNlpChunkingEngine exten
public static final String MIN_CHUNK_SCORE = "org.apache.stanbol.enhancer.chunker.minScore";
- public static final String[] AVAILABLE_LANGUAGES = new String[] {"en","de"};
-
private static final String MODEL_PARAM_NAME = "model";
private static Logger log = LoggerFactory.getLogger(OpenNlpChunkingEngine.class);
@@ -139,7 +130,15 @@ public class OpenNlpChunkingEngine exten
private Double minChunkScore;
-
+ /**
+ * Holds as key the languages and as values the ad-hoc (unmapped) phrase tags
+ * for that languages.<p>
+ * NOTE: Not synchronised as concurrent execution caused multiple adds will
+ * only create some additional {@link PhraseTag} instances and not actual
+ * problems.
+ */
+ private Map<String,Map<String,PhraseTag>> languageAdhocTags = new HashMap<String,Map<String,PhraseTag>>();
+
/**
* Indicate if this engine can enhance supplied ContentItem, and if it
* suggests enhancing it synchronously or asynchronously. The
@@ -206,8 +205,11 @@ public class OpenNlpChunkingEngine exten
}
//holds PosTags created for POS tags that where not part of the posModel
//(will hold all PosTags in case tagSet is NULL
- Map<String,PhraseTag> adhocTags = new HashMap<String,PhraseTag>();
-
+ Map<String,PhraseTag> adhocTags = languageAdhocTags.get(language);
+ if(adhocTags == null){
+ adhocTags = new HashMap<String,PhraseTag>();
+ languageAdhocTags.put(language, adhocTags);
+ }
ci.getLock().writeLock().lock();
try {
Iterator<? extends Section> sentences = at.getSentences();
@@ -286,7 +288,7 @@ public class OpenNlpChunkingEngine exten
}
if(start){ //create the new tag
tag = getPhraseTag(tagSet,adhocTags,
- chunkTags[i].substring(2)); //skip 'B-'
+ chunkTags[i].substring(2), language); //skip 'B-'
}
if(tag != null){ //count this token for the current chunk
@@ -335,7 +337,7 @@ public class OpenNlpChunkingEngine exten
}
}
- private PhraseTag getPhraseTag(TagSet<PhraseTag> model, Map<String,PhraseTag> adhocTags, String tag) {
+ private PhraseTag getPhraseTag(TagSet<PhraseTag> model, Map<String,PhraseTag> adhocTags, String tag, String language) {
PhraseTag phraseTag = model.getTag(tag);
if(phraseTag != null){
return phraseTag;
@@ -346,7 +348,7 @@ public class OpenNlpChunkingEngine exten
}
phraseTag = new PhraseTag(tag);
adhocTags.put(tag, phraseTag);
- log.warn("Encountered unknown POS tag '{}' for langauge '{}'",tag,model.getLanguages());
+ log.info("Encountered unknown POS tag '{}' for langauge '{}'",tag,language);
return phraseTag;
}
@@ -401,6 +403,7 @@ public class OpenNlpChunkingEngine exten
protected void deactivate(ComponentContext context){
this.languageConfiguration.setDefault();
this.minChunkScore = null;
+ this.languageAdhocTags.clear();
super.deactivate(context);
}
Modified: stanbol/trunk/enhancement-engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java?rev=1418822&r1=1418821&r2=1418822&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java (original)
+++ stanbol/trunk/enhancement-engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java Sun Dec 9 06:30:42 2012
@@ -27,7 +27,6 @@ public class NEREngineConfig {
static { //the default mappings for the default NER types
Map<String,UriRef> mappings = new TreeMap<String,UriRef>();
- mappings.put(OntologicalClasses.DBPEDIA_ORGANISATION.getUnicodeString(), OntologicalClasses.DBPEDIA_ORGANISATION);
mappings.put("person", OntologicalClasses.DBPEDIA_PERSON);
mappings.put("location", OntologicalClasses.DBPEDIA_PLACE);
mappings.put("organization", OntologicalClasses.DBPEDIA_ORGANISATION);
Modified: stanbol/trunk/enhancement-engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/OpenNlpPosTaggingEngine.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/OpenNlpPosTaggingEngine.java?rev=1418822&r1=1418821&r2=1418822&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/OpenNlpPosTaggingEngine.java (original)
+++ stanbol/trunk/enhancement-engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/OpenNlpPosTaggingEngine.java Sun Dec 9 06:30:42 2012
@@ -61,6 +61,7 @@ import org.apache.stanbol.enhancer.nlp.m
import org.apache.stanbol.enhancer.nlp.model.Token;
import org.apache.stanbol.enhancer.nlp.model.annotation.Value;
import org.apache.stanbol.enhancer.nlp.model.tag.TagSet;
+import org.apache.stanbol.enhancer.nlp.phrase.PhraseTag;
import org.apache.stanbol.enhancer.nlp.pos.PosTag;
import org.apache.stanbol.enhancer.nlp.utils.LanguageConfiguration;
import org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper;
@@ -134,7 +135,16 @@ public class OpenNlpPosTaggingEngine ext
@Reference
private AnalysedTextFactory analysedTextFactory;
-
+
+ /**
+ * Holds as key the languages and as values the ad-hoc (unmapped) phrase tags
+ * for that languages.<p>
+ * NOTE: Not synchronised as concurrent execution caused multiple adds will
+ * only create some additional {@link PhraseTag} instances and not actual
+ * problems.
+ */
+ private Map<String,Map<String,PosTag>> languageAdhocTags = new HashMap<String,Map<String,PosTag>>();
+
/**
* Indicate if this engine can enhance supplied ContentItem, and if it
* suggests enhancing it synchronously or asynchronously. The
@@ -214,7 +224,11 @@ public class OpenNlpPosTaggingEngine ext
}
//holds PosTags created for POS tags that where not part of the posModel
//(will hold all PosTags in case tagSet is NULL
- Map<String,PosTag> adhocTags = new HashMap<String,PosTag>();
+ Map<String,PosTag> adhocTags = languageAdhocTags.get(language);
+ if(adhocTags == null){
+ adhocTags = new HashMap<String,PosTag>();
+ languageAdhocTags.put(language, adhocTags);
+ }
//(1) Sentence detection
//Try to read existing Sentence Annotations
@@ -254,8 +268,7 @@ public class OpenNlpPosTaggingEngine ext
}
//(3) POS Tagging
-
- posTag(tokenList, posTagger,tagSet,adhocTags);
+ posTag(tokenList, posTagger,tagSet,adhocTags,language);
}
if(log.isTraceEnabled()){
@@ -286,11 +299,13 @@ public class OpenNlpPosTaggingEngine ext
* @param posTagger
* @param posModel
* @param adhocTags
+ * @param language
*/
private void posTag(List<Token> tokenList,
POSTagger posTagger,
TagSet<PosTag> posModel,
- Map<String,PosTag> adhocTags) {
+ Map<String,PosTag> adhocTags,
+ String language) {
String[] tokenTexts = new String[tokenList.size()];
for(int i=0;i<tokenList.size(); i++){
tokenTexts[i] = tokenList.get(i).getSpan();
@@ -313,7 +328,7 @@ public class OpenNlpPosTaggingEngine ext
String p = posSequences[j].getOutcomes().get(i);
done = j > 0 && p.equals(actPos[0].getTag());
if(!done){
- actPos[j] = getPosTag(posModel,adhocTags,p);
+ actPos[j] = getPosTag(posModel,adhocTags,p,language);
actProp[j] = posSequences[j].getProbs()[i];
j++;
}
@@ -324,7 +339,7 @@ public class OpenNlpPosTaggingEngine ext
}
- private PosTag getPosTag(TagSet<PosTag> model, Map<String,PosTag> adhocTags, String tag) {
+ private PosTag getPosTag(TagSet<PosTag> model, Map<String,PosTag> adhocTags, String tag, String language) {
PosTag posTag = model.getTag(tag);
if(posTag != null){
return posTag;
@@ -335,7 +350,7 @@ public class OpenNlpPosTaggingEngine ext
}
posTag = new PosTag(tag);
adhocTags.put(tag, posTag);
- log.warn("Encountered unknown POS tag '{}' for langauge '{}'",tag,model.getLanguages());
+ log.info("Encountered umapped POS tag '{}' for langauge '{}'",tag,language);
return posTag;
}