You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2012/12/09 07:30:44 UTC

svn commit: r1418822 - in /stanbol/trunk/enhancement-engines: entitylinking/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/ langdetect/src/main/java/org/apache/stanbol/enhancer/engines/langdetect/ opennlp-chunker/src/main/java/o...

Author: rwesten
Date: Sun Dec  9 06:30:42 2012
New Revision: 1418822

URL: http://svn.apache.org/viewvc?rev=1418822&view=rev
Log:
minor: Just changed some logging levels to reduce the messages on INFO level. THe OpenNLP Chunker and Pos engine now keep ad-hoc Tags (unmapped String Tags). This is also to make loggings more meaningful as only the first time that such a Tag appear an INFO message is written to the log. The goal ist to make is more easy to find those Tags by using grebs over the log files

Modified:
    stanbol/trunk/enhancement-engines/entitylinking/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngine.java
    stanbol/trunk/enhancement-engines/langdetect/src/main/java/org/apache/stanbol/enhancer/engines/langdetect/LanguageDetectionEnhancementEngine.java
    stanbol/trunk/enhancement-engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/OpenNlpChunkingEngine.java
    stanbol/trunk/enhancement-engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java
    stanbol/trunk/enhancement-engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/OpenNlpPosTaggingEngine.java

Modified: stanbol/trunk/enhancement-engines/entitylinking/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngine.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngine.java?rev=1418822&r1=1418821&r2=1418822&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngine.java (original)
+++ stanbol/trunk/enhancement-engines/entitylinking/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngine.java Sun Dec  9 06:30:42 2012
@@ -191,7 +191,7 @@ public class EntityLinkingEngine impleme
     
     @Override
     public int canEnhance(ContentItem ci) throws EngineException {
-        log.info("canEnhancer {}",ci.getUri());
+        log.trace("canEnhancer {}",ci.getUri());
         if(isOfflineMode() && !entitySearcher.supportsOfflineMode()){
             log.warn("{} '{}' is inactive because EntitySearcher does not support Offline mode!",
                 getClass().getSimpleName(),getName());
@@ -212,12 +212,12 @@ public class EntityLinkingEngine impleme
 
     @Override
     public void computeEnhancements(ContentItem ci) throws EngineException {
-        log.info(" enhance ci {}",ci.getUri());
+        log.trace(" enhance ci {}",ci.getUri());
         if(isOfflineMode() && !entitySearcher.supportsOfflineMode()){
             throw new EngineException(this,ci,"Offline mode is not supported by the used EntitySearcher!",null);
         }
         AnalysedText at = getAnalysedText(this, ci, true);
-        log.info("  > AnalysedText {}",at);
+        log.debug("  > AnalysedText {}",at);
         String language = getLanguage(this, ci, true);
         if(log.isDebugEnabled()){
             log.debug("computeEnhancements for ContentItem {} language {} text={}", 

Modified: stanbol/trunk/enhancement-engines/langdetect/src/main/java/org/apache/stanbol/enhancer/engines/langdetect/LanguageDetectionEnhancementEngine.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/langdetect/src/main/java/org/apache/stanbol/enhancer/engines/langdetect/LanguageDetectionEnhancementEngine.java?rev=1418822&r1=1418821&r2=1418822&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/langdetect/src/main/java/org/apache/stanbol/enhancer/engines/langdetect/LanguageDetectionEnhancementEngine.java (original)
+++ stanbol/trunk/enhancement-engines/langdetect/src/main/java/org/apache/stanbol/enhancer/engines/langdetect/LanguageDetectionEnhancementEngine.java Sun Dec  9 06:30:42 2012
@@ -229,11 +229,11 @@ public class LanguageDetectionEnhancemen
         List<Language> languages = null;
         try {
             languages = languageIdentifier.getLanguages(text);
-            log.info("language identified: {}",languages);
+            log.debug("language identified: {}",languages);
         }
         catch (LangDetectException e) {
-            log.warn("Could not identify language");
-            return;
+            log.warn("Could not identify language", e);
+            throw new EngineException(this, ci, "Could not identify language", e);
         }
         
         // add language to metadata

Modified: stanbol/trunk/enhancement-engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/OpenNlpChunkingEngine.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/OpenNlpChunkingEngine.java?rev=1418822&r1=1418821&r2=1418822&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/OpenNlpChunkingEngine.java (original)
+++ stanbol/trunk/enhancement-engines/opennlp-chunker/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/chunker/services/OpenNlpChunkingEngine.java Sun Dec  9 06:30:42 2012
@@ -29,15 +29,12 @@ import java.util.Collections;
 import java.util.Dictionary;
 import java.util.EnumSet;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import opennlp.tools.chunker.ChunkerME;
 import opennlp.tools.chunker.ChunkerModel;
-import opennlp.tools.util.Sequence;
 
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
@@ -49,11 +46,9 @@ import org.apache.felix.scr.annotations.
 import org.apache.felix.scr.annotations.Service;
 import org.apache.stanbol.commons.opennlp.OpenNLP;
 import org.apache.stanbol.enhancer.engines.opennlp.chunker.model.PhraseTagSetRegistry;
-import org.apache.stanbol.enhancer.nlp.NlpAnnotations;
 import org.apache.stanbol.enhancer.nlp.NlpProcessingRole;
 import org.apache.stanbol.enhancer.nlp.NlpServiceProperties;
 import org.apache.stanbol.enhancer.nlp.model.AnalysedText;
-import org.apache.stanbol.enhancer.nlp.model.AnalysedTextUtils;
 import org.apache.stanbol.enhancer.nlp.model.Chunk;
 import org.apache.stanbol.enhancer.nlp.model.Section;
 import org.apache.stanbol.enhancer.nlp.model.Span;
@@ -64,12 +59,10 @@ import org.apache.stanbol.enhancer.nlp.m
 import org.apache.stanbol.enhancer.nlp.phrase.PhraseTag;
 import org.apache.stanbol.enhancer.nlp.pos.PosTag;
 import org.apache.stanbol.enhancer.nlp.utils.LanguageConfiguration;
-import org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper;
 import org.apache.stanbol.enhancer.servicesapi.ContentItem;
 import org.apache.stanbol.enhancer.servicesapi.EngineException;
 import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
 import org.apache.stanbol.enhancer.servicesapi.ServiceProperties;
-import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
 import org.apache.stanbol.enhancer.servicesapi.impl.AbstractEnhancementEngine;
 import org.osgi.framework.Constants;
 import org.osgi.service.cm.ConfigurationException;
@@ -120,8 +113,6 @@ public class OpenNlpChunkingEngine exten
 
     public static final String MIN_CHUNK_SCORE = "org.apache.stanbol.enhancer.chunker.minScore";
     
-    public static final String[] AVAILABLE_LANGUAGES = new String[] {"en","de"};
-
     private static final String MODEL_PARAM_NAME = "model";
 
     private static Logger log = LoggerFactory.getLogger(OpenNlpChunkingEngine.class);
@@ -139,7 +130,15 @@ public class OpenNlpChunkingEngine exten
 
     private Double minChunkScore;
 
-
+    /**
+     * Holds as key the languages and as values the ad-hoc (unmapped) phrase tags
+     * for that languages.<p>
+     * NOTE: Not synchronised as concurrent execution caused multiple adds will
+     * only create some additional {@link PhraseTag} instances and not actual
+     * problems.
+     */
+    private Map<String,Map<String,PhraseTag>> languageAdhocTags = new HashMap<String,Map<String,PhraseTag>>();
+    
     /**
      * Indicate if this engine can enhance supplied ContentItem, and if it
      * suggests enhancing it synchronously or asynchronously. The
@@ -206,8 +205,11 @@ public class OpenNlpChunkingEngine exten
         }
         //holds PosTags created for POS tags that where not part of the posModel
         //(will hold all PosTags in case tagSet is NULL
-        Map<String,PhraseTag> adhocTags = new HashMap<String,PhraseTag>();
-        
+        Map<String,PhraseTag> adhocTags = languageAdhocTags.get(language);
+        if(adhocTags == null){
+            adhocTags = new HashMap<String,PhraseTag>();
+            languageAdhocTags.put(language, adhocTags);
+        }        
         ci.getLock().writeLock().lock();
         try {
             Iterator<? extends Section> sentences = at.getSentences();
@@ -286,7 +288,7 @@ public class OpenNlpChunkingEngine exten
                     }
                     if(start){ //create the new tag
                         tag = getPhraseTag(tagSet,adhocTags,
-                            chunkTags[i].substring(2)); //skip 'B-'
+                            chunkTags[i].substring(2), language); //skip 'B-'
                         
                     }
                     if(tag != null){ //count this token for the current chunk
@@ -335,7 +337,7 @@ public class OpenNlpChunkingEngine exten
         }
     }
 
-    private PhraseTag getPhraseTag(TagSet<PhraseTag> model, Map<String,PhraseTag> adhocTags, String tag) {
+    private PhraseTag getPhraseTag(TagSet<PhraseTag> model, Map<String,PhraseTag> adhocTags, String tag, String language) {
         PhraseTag phraseTag = model.getTag(tag);
         if(phraseTag != null){
             return phraseTag;
@@ -346,7 +348,7 @@ public class OpenNlpChunkingEngine exten
         }
         phraseTag = new PhraseTag(tag);
         adhocTags.put(tag, phraseTag);
-        log.warn("Encountered unknown POS tag '{}' for langauge '{}'",tag,model.getLanguages());
+        log.info("Encountered unknown POS tag '{}' for langauge '{}'",tag,language);
         return phraseTag;
     }
     
@@ -401,6 +403,7 @@ public class OpenNlpChunkingEngine exten
     protected void deactivate(ComponentContext context){
         this.languageConfiguration.setDefault();
         this.minChunkScore = null;
+        this.languageAdhocTags.clear();
         super.deactivate(context);
     }
     

Modified: stanbol/trunk/enhancement-engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java?rev=1418822&r1=1418821&r2=1418822&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java (original)
+++ stanbol/trunk/enhancement-engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineConfig.java Sun Dec  9 06:30:42 2012
@@ -27,7 +27,6 @@ public class NEREngineConfig {
     
     static { //the default mappings for the default NER types
         Map<String,UriRef> mappings = new TreeMap<String,UriRef>();
-        mappings.put(OntologicalClasses.DBPEDIA_ORGANISATION.getUnicodeString(), OntologicalClasses.DBPEDIA_ORGANISATION);
         mappings.put("person", OntologicalClasses.DBPEDIA_PERSON);
         mappings.put("location", OntologicalClasses.DBPEDIA_PLACE);
         mappings.put("organization", OntologicalClasses.DBPEDIA_ORGANISATION);

Modified: stanbol/trunk/enhancement-engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/OpenNlpPosTaggingEngine.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/OpenNlpPosTaggingEngine.java?rev=1418822&r1=1418821&r2=1418822&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/OpenNlpPosTaggingEngine.java (original)
+++ stanbol/trunk/enhancement-engines/opennlp-pos/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/pos/services/OpenNlpPosTaggingEngine.java Sun Dec  9 06:30:42 2012
@@ -61,6 +61,7 @@ import org.apache.stanbol.enhancer.nlp.m
 import org.apache.stanbol.enhancer.nlp.model.Token;
 import org.apache.stanbol.enhancer.nlp.model.annotation.Value;
 import org.apache.stanbol.enhancer.nlp.model.tag.TagSet;
+import org.apache.stanbol.enhancer.nlp.phrase.PhraseTag;
 import org.apache.stanbol.enhancer.nlp.pos.PosTag;
 import org.apache.stanbol.enhancer.nlp.utils.LanguageConfiguration;
 import org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper;
@@ -134,7 +135,16 @@ public class OpenNlpPosTaggingEngine ext
     
     @Reference
     private AnalysedTextFactory analysedTextFactory;
-    
+
+    /**
+     * Holds as key the languages and as values the ad-hoc (unmapped) phrase tags
+     * for that languages.<p>
+     * NOTE: Not synchronised as concurrent execution caused multiple adds will
+     * only create some additional {@link PhraseTag} instances and not actual
+     * problems.
+     */
+    private Map<String,Map<String,PosTag>> languageAdhocTags = new HashMap<String,Map<String,PosTag>>();
+     
     /**
      * Indicate if this engine can enhance supplied ContentItem, and if it
      * suggests enhancing it synchronously or asynchronously. The
@@ -214,7 +224,11 @@ public class OpenNlpPosTaggingEngine ext
         }
         //holds PosTags created for POS tags that where not part of the posModel
         //(will hold all PosTags in case tagSet is NULL
-        Map<String,PosTag> adhocTags = new HashMap<String,PosTag>();
+        Map<String,PosTag> adhocTags = languageAdhocTags.get(language);
+        if(adhocTags == null){
+                adhocTags =  new HashMap<String,PosTag>();
+                languageAdhocTags.put(language, adhocTags);
+        }
         //(1) Sentence detection
         
         //Try to read existing Sentence Annotations
@@ -254,8 +268,7 @@ public class OpenNlpPosTaggingEngine ext
             }
             
             //(3) POS Tagging
-            
-            posTag(tokenList, posTagger,tagSet,adhocTags);
+            posTag(tokenList, posTagger,tagSet,adhocTags,language);
             
         }
         if(log.isTraceEnabled()){
@@ -286,11 +299,13 @@ public class OpenNlpPosTaggingEngine ext
      * @param posTagger
      * @param posModel
      * @param adhocTags
+     * @param language
      */
     private void posTag(List<Token> tokenList,
                         POSTagger posTagger,
                         TagSet<PosTag> posModel,
-                        Map<String,PosTag> adhocTags) {
+                        Map<String,PosTag> adhocTags, 
+                        String language) {
         String[] tokenTexts = new String[tokenList.size()];
         for(int i=0;i<tokenList.size(); i++){
             tokenTexts[i] = tokenList.get(i).getSpan();
@@ -313,7 +328,7 @@ public class OpenNlpPosTaggingEngine ext
                 String p = posSequences[j].getOutcomes().get(i);
                 done = j > 0 && p.equals(actPos[0].getTag());
                 if(!done){
-                    actPos[j] = getPosTag(posModel,adhocTags,p);
+                    actPos[j] = getPosTag(posModel,adhocTags,p,language);
                     actProp[j] = posSequences[j].getProbs()[i];
                     j++;
                 }
@@ -324,7 +339,7 @@ public class OpenNlpPosTaggingEngine ext
 
     }
 
-    private PosTag getPosTag(TagSet<PosTag> model, Map<String,PosTag> adhocTags, String tag) {
+    private PosTag getPosTag(TagSet<PosTag> model, Map<String,PosTag> adhocTags, String tag, String language) {
         PosTag posTag = model.getTag(tag);
         if(posTag != null){
             return posTag;
@@ -335,7 +350,7 @@ public class OpenNlpPosTaggingEngine ext
         }
         posTag = new PosTag(tag);
         adhocTags.put(tag, posTag);
-        log.warn("Encountered unknown POS tag '{}' for langauge '{}'",tag,model.getLanguages());
+        log.info("Encountered umapped POS tag '{}' for langauge '{}'",tag,language);
         return posTag;
     }