You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2012/02/02 14:52:28 UTC
svn commit: r1239618 [2/2] - in /incubator/stanbol/trunk/enhancer:
engines/autotagging/src/main/java/org/apache/stanbol/enhancer/engines/autotagging/impl/
engines/entitytagging/src/main/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/
engin...
Modified: incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java?rev=1239618&r1=1239617&r2=1239618&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java Thu Feb 2 13:52:27 2012
@@ -23,7 +23,6 @@ import static org.apache.stanbol.enhance
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTED_TEXT;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTION_CONTEXT;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_START;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.NIE_PLAINTEXTCONTENT;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -35,6 +34,7 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import opennlp.tools.namefind.NameFinderME;
@@ -56,10 +56,12 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.stanbol.commons.opennlp.OpenNLP;
import org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider;
+import org.apache.stanbol.enhancer.servicesapi.Blob;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
import org.apache.stanbol.enhancer.servicesapi.EngineException;
import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
import org.apache.stanbol.enhancer.servicesapi.InvalidContentException;
+import org.apache.stanbol.enhancer.servicesapi.helper.ContentItemHelper;
import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
import org.apache.stanbol.enhancer.servicesapi.rdf.OntologicalClasses;
import org.apache.stanbol.enhancer.servicesapi.rdf.Properties;
@@ -71,6 +73,11 @@ import org.slf4j.LoggerFactory;
*/
public class NEREngineCore implements EnhancementEngine {
protected static final String TEXT_PLAIN_MIMETYPE = "text/plain";
+ /**
+ * Contains the only supported mimetype {@link #TEXT_PLAIN_MIMETYPE}
+ */
+ protected static final Set<String> SUPPORTED_MIMETYPES =
+ Collections.singleton(TEXT_PLAIN_MIMETYPE);
private final Logger log = LoggerFactory.getLogger(getClass());
private static Map<String,UriRef> entityTypes = new HashMap<String,UriRef>();
@@ -119,43 +126,44 @@ public class NEREngineCore implements En
//first check the langauge before processing the content (text)
String language = extractLanguage(ci);
if(language == null){
- log.warn("Unable to extract Language for ContentItem {}: The text" +
- "of this ContentItem will not be processed by the NER engine!",
- ci.getUri());
- return;
+ throw new IllegalStateException("Unable to extract Language for "
+ + "ContentItem "+ci.getUri()+": This is also checked in the canEnhance "
+ + "method! -> This indicated an Bug in the implementation of the "
+ + "EnhancementJobManager!");
}
if(!isProcessedLangage(language)){
- log.warn("The language {} of ContentItem {} is not configured to be" +
- "processed by this NER engine instance (processed {})!",
- new Object[]{language,ci.getUri(),processedLangs});
- return;
+ throw new IllegalStateException("The language '"+language+"' of ContentItem "+ci.getUri()
+ + " is not configured to be processed by this NER engine instance "
+ + "(processed "+processedLangs+"): This is also checked in the canEnhance "
+ + "method! -> This indicated an Bug in the implementation of the "
+ + "EnhancementJobManager!");
+ }
+ Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, SUPPORTED_MIMETYPES);
+ if(contentPart == null){
+ throw new IllegalStateException("No ContentPart with Mimetype '"
+ + TEXT_PLAIN_MIMETYPE+"' found for ContentItem "+ci.getUri()
+ + ": This is also checked in the canEnhance method! -> This "
+ + "indicated an Bug in the implementation of the "
+ + "EnhancementJobManager!");
}
- String mimeType = ci.getMimeType().split(";", 2)[0];
String text;
- if (TEXT_PLAIN_MIMETYPE.equals(mimeType)) {
- try {
- text = IOUtils.toString(ci.getStream(),"UTF-8");
- } catch (IOException e) {
- throw new InvalidContentException(this, ci, e);
- }
- } else {
- //TODO: change that as soon the Adapter Pattern is used for multiple
- // mimetype support.
- StringBuilder textBuilder = new StringBuilder();
- Iterator<Triple> it = ci.getMetadata().filter(ci.getUri(), NIE_PLAINTEXTCONTENT, null);
- while (it.hasNext()) {
- textBuilder.append(it.next().getObject());
- }
- text = textBuilder.toString();
+ try {
+ text = ContentItemHelper.getText(contentPart.getValue());
+ } catch (IOException e) {
+ throw new InvalidContentException(this, ci, e);
}
if (text.trim().length() == 0) {
// TODO: make the length of the data a field of the ContentItem
// interface to be able to filter out empty items in the canEnhance
// method
- log.warn("nothing to extract knowledge from in ContentItem {}", ci);
+ log.warn("ContentPart {} of ContentItem {} does not contain any text" +
+ "to extract knowledge from in ContentItem {}",
+ contentPart.getKey(),ci);
return;
}
- log.debug("computeEnhancements {} text={}", ci.getUri().getUnicodeString(), StringUtils.abbreviate(text, 100));
+ log.debug("computeEnhancements from ContentPart {} of ContentItem {}: text={}",
+ new Object[]{contentPart.getKey(),ci.getUri().getUnicodeString(),
+ StringUtils.abbreviate(text, 100)});
try {
for (Map.Entry<String,UriRef> type : entityTypes.entrySet()) {
String typeLabel = type.getKey();
@@ -190,57 +198,62 @@ public class NEREngineCore implements En
LiteralFactory literalFactory = LiteralFactory.getInstance();
MGraph g = ci.getMetadata();
Map<String,List<NameOccurrence>> entityNames = extractNameOccurrences(nameFinderModel, text);
-
- Map<String,UriRef> previousAnnotations = new LinkedHashMap<String,UriRef>();
- for (Map.Entry<String,List<NameOccurrence>> nameInContext : entityNames.entrySet()) {
-
- String name = nameInContext.getKey();
- List<NameOccurrence> occurrences = nameInContext.getValue();
-
- UriRef firstOccurrenceAnnotation = null;
-
- for (NameOccurrence occurrence : occurrences) {
- UriRef textAnnotation = EnhancementEngineHelper.createTextEnhancement(ci, this);
- g.add(new TripleImpl(textAnnotation, ENHANCER_SELECTED_TEXT, literalFactory
- .createTypedLiteral(name)));
- g.add(new TripleImpl(textAnnotation, ENHANCER_SELECTION_CONTEXT, literalFactory
- .createTypedLiteral(occurrence.context)));
- g.add(new TripleImpl(textAnnotation, DC_TYPE, typeUri));
- g.add(new TripleImpl(textAnnotation, ENHANCER_CONFIDENCE, literalFactory
- .createTypedLiteral(occurrence.confidence)));
- if (occurrence.start != null && occurrence.end != null) {
- g.add(new TripleImpl(textAnnotation, ENHANCER_START, literalFactory
- .createTypedLiteral(occurrence.start)));
- g.add(new TripleImpl(textAnnotation, ENHANCER_END, literalFactory
- .createTypedLiteral(occurrence.end)));
- }
-
- // add the subsumption relationship among occurrences of the same
- // name
- if (firstOccurrenceAnnotation == null) {
- // check already extracted annotations to find a first most
- // specific occurrence
- for (Map.Entry<String,UriRef> entry : previousAnnotations.entrySet()) {
- if (entry.getKey().contains(name)) {
- // we have found a most specific previous
- // occurrence, use it as subsumption target
- firstOccurrenceAnnotation = entry.getValue();
- g.add(new TripleImpl(textAnnotation, DC_RELATION, firstOccurrenceAnnotation));
- break;
- }
+ //lock the ContentItem while writing the RDF data for found Named Entities
+ ci.getLock().writeLock().lock();
+ try {
+ Map<String,UriRef> previousAnnotations = new LinkedHashMap<String,UriRef>();
+ for (Map.Entry<String,List<NameOccurrence>> nameInContext : entityNames.entrySet()) {
+
+ String name = nameInContext.getKey();
+ List<NameOccurrence> occurrences = nameInContext.getValue();
+
+ UriRef firstOccurrenceAnnotation = null;
+
+ for (NameOccurrence occurrence : occurrences) {
+ UriRef textAnnotation = EnhancementEngineHelper.createTextEnhancement(ci, this);
+ g.add(new TripleImpl(textAnnotation, ENHANCER_SELECTED_TEXT, literalFactory
+ .createTypedLiteral(name)));
+ g.add(new TripleImpl(textAnnotation, ENHANCER_SELECTION_CONTEXT, literalFactory
+ .createTypedLiteral(occurrence.context)));
+ g.add(new TripleImpl(textAnnotation, DC_TYPE, typeUri));
+ g.add(new TripleImpl(textAnnotation, ENHANCER_CONFIDENCE, literalFactory
+ .createTypedLiteral(occurrence.confidence)));
+ if (occurrence.start != null && occurrence.end != null) {
+ g.add(new TripleImpl(textAnnotation, ENHANCER_START, literalFactory
+ .createTypedLiteral(occurrence.start)));
+ g.add(new TripleImpl(textAnnotation, ENHANCER_END, literalFactory
+ .createTypedLiteral(occurrence.end)));
}
+
+ // add the subsumption relationship among occurrences of the same
+ // name
if (firstOccurrenceAnnotation == null) {
- // no most specific previous occurrence, I am the first,
- // most specific occurrence to be later used as a target
- firstOccurrenceAnnotation = textAnnotation;
- previousAnnotations.put(name, textAnnotation);
+ // check already extracted annotations to find a first most
+ // specific occurrence
+ for (Map.Entry<String,UriRef> entry : previousAnnotations.entrySet()) {
+ if (entry.getKey().contains(name)) {
+ // we have found a most specific previous
+ // occurrence, use it as subsumption target
+ firstOccurrenceAnnotation = entry.getValue();
+ g.add(new TripleImpl(textAnnotation, DC_RELATION, firstOccurrenceAnnotation));
+ break;
+ }
+ }
+ if (firstOccurrenceAnnotation == null) {
+ // no most specific previous occurrence, I am the first,
+ // most specific occurrence to be later used as a target
+ firstOccurrenceAnnotation = textAnnotation;
+ previousAnnotations.put(name, textAnnotation);
+ }
+ } else {
+ // I am referring to a most specific first occurrence of the
+ // same name
+ g.add(new TripleImpl(textAnnotation, DC_RELATION, firstOccurrenceAnnotation));
}
- } else {
- // I am referring to a most specific first occurrence of the
- // same name
- g.add(new TripleImpl(textAnnotation, DC_RELATION, firstOccurrenceAnnotation));
}
}
+ } finally {
+ ci.getLock().writeLock().unlock();
}
}
@@ -390,21 +403,11 @@ public class NEREngineCore implements En
}
public int canEnhance(ContentItem ci) {
- // in case text/pain;charSet=UTF8 is parsed
- String mimeType = ci.getMimeType().split(";", 2)[0];
- if(TEXT_PLAIN_MIMETYPE.equalsIgnoreCase(mimeType) || //plain test
- //or extracted text
- ci.getMetadata().filter(ci.getUri(), NIE_PLAINTEXTCONTENT, null).hasNext()){
- //TODO: check if the language metadata are already present when
- //canEnhance is called. If not than return ENHANCE_SYNCHRONOUS
- if(isProcessedLangage(extractLanguage(ci))){
- return ENHANCE_SYNCHRONOUS;
- } else {
- return CANNOT_ENHANCE;
- }
- } else { //no textual content available
- return CANNOT_ENHANCE;
+ if(ContentItemHelper.getBlob(ci, SUPPORTED_MIMETYPES) != null
+ && isProcessedLangage(extractLanguage(ci))){
+ return ENHANCE_ASYNC; //The NER engine now supports Async processing!
}
+ return CANNOT_ENHANCE;
}
/**
Modified: incubator/stanbol/trunk/enhancer/engines/taxonomylinking/src/main/java/org/apache/stanbol/enhancer/engines/taxonomy/impl/TaxonomyLinkingEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/taxonomylinking/src/main/java/org/apache/stanbol/enhancer/engines/taxonomy/impl/TaxonomyLinkingEngine.java?rev=1239618&r1=1239617&r2=1239618&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/taxonomylinking/src/main/java/org/apache/stanbol/enhancer/engines/taxonomy/impl/TaxonomyLinkingEngine.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/taxonomylinking/src/main/java/org/apache/stanbol/enhancer/engines/taxonomy/impl/TaxonomyLinkingEngine.java Thu Feb 2 13:52:27 2012
@@ -16,8 +16,6 @@
*/
package org.apache.stanbol.enhancer.engines.taxonomy.impl;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.NIE_PLAINTEXTCONTENT;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -29,6 +27,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
@@ -48,11 +47,9 @@ import opennlp.tools.util.Span;
import org.apache.clerezza.rdf.core.LiteralFactory;
import org.apache.clerezza.rdf.core.MGraph;
-import org.apache.clerezza.rdf.core.Triple;
import org.apache.clerezza.rdf.core.UriRef;
import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl;
import org.apache.clerezza.rdf.core.impl.TripleImpl;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
@@ -68,12 +65,14 @@ import org.apache.felix.scr.annotations.
import org.apache.felix.scr.annotations.Service;
import org.apache.stanbol.commons.opennlp.OpenNLP;
import org.apache.stanbol.commons.stanboltools.offline.OfflineMode;
+import org.apache.stanbol.enhancer.servicesapi.Blob;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
import org.apache.stanbol.enhancer.servicesapi.EngineException;
import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
import org.apache.stanbol.enhancer.servicesapi.InvalidContentException;
import org.apache.stanbol.enhancer.servicesapi.ServiceProperties;
import org.apache.stanbol.enhancer.servicesapi.helper.AbstractEnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.helper.ContentItemHelper;
import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
import org.apache.stanbol.enhancer.servicesapi.rdf.OntologicalClasses;
import org.apache.stanbol.enhancer.servicesapi.rdf.Properties;
@@ -150,6 +149,10 @@ public class TaxonomyLinkingEngine
protected static final String TEXT_PLAIN_MIMETYPE = "text/plain";
/**
+ * Contains the only supported mime type {@link #TEXT_PLAIN_MIMETYPE}
+ */
+ protected static final Set<String> SUPPORTED_MIMETYPES = Collections.singleton(TEXT_PLAIN_MIMETYPE);
+ /**
* The default value for the Execution of this Engine. Currently set to
* {@link ServiceProperties#ORDERING_EXTRACTION_ENHANCEMENT} + 10. It should run after Metaxa and LangId.
*/
@@ -360,17 +363,11 @@ public class TaxonomyLinkingEngine
@Override
public int canEnhance(ContentItem ci) throws EngineException {
- String mimeType = ci.getMimeType().split(";", 2)[0];
- if (TEXT_PLAIN_MIMETYPE.equalsIgnoreCase(mimeType)) {
- return ENHANCE_SYNCHRONOUS;
- }
- // check for existence of textual content in metadata
- UriRef subj = ci.getUri();
- Iterator<Triple> it = ci.getMetadata().filter(subj, NIE_PLAINTEXTCONTENT, null);
- if (it.hasNext()) {
+ if(ContentItemHelper.getBlob(ci, SUPPORTED_MIMETYPES) != null){
return ENHANCE_SYNCHRONOUS;
+ } else {
+ return ENHANCE_ASYNC;
}
- return CANNOT_ENHANCE;
}
@Override
@@ -398,35 +395,32 @@ public class TaxonomyLinkingEngine
} else { // null indicates to use the Entityhub to lookup Entities
site = null;
}
- String mimeType = ci.getMimeType().split(";", 2)[0];
+ Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, SUPPORTED_MIMETYPES);
+ if(contentPart == null){
+ throw new IllegalStateException("No ContentPart with a supported Mime Type"
+ + "found for ContentItem "+ci.getUri()+"(supported: '"
+ + SUPPORTED_MIMETYPES+"') -> this indicates that canEnhance was"
+ + "NOT called and indicates a bug in the used EnhancementJobManager!");
+ }
String text;
- if (TEXT_PLAIN_MIMETYPE.equals(mimeType)) {
- try {
- text = IOUtils.toString(ci.getStream(),"UTF-8");
- } catch (IOException e) {
- throw new InvalidContentException(this, ci, e);
- }
- } else {
- //TODO: change that as soon the Adapter Pattern is used for multiple
- // mimetype support.
- StringBuilder textBuilder = new StringBuilder();
- Iterator<Triple> it = ci.getMetadata().filter(ci.getUri(), NIE_PLAINTEXTCONTENT, null);
- while (it.hasNext()) {
- textBuilder.append(it.next().getObject());
- }
- text = textBuilder.toString();
+ try {
+ text = ContentItemHelper.getText(contentPart.getValue());
+ } catch (IOException e) {
+ throw new InvalidContentException(this, ci, e);
}
if (text.trim().length() == 0) {
// TODO: make the length of the data a field of the ContentItem
// interface to be able to filter out empty items in the canEnhance
// method
- log.warn("nothing to extract knowledge from in ContentItem {}", ci);
+ log.warn("ContentPart {} of ContentItem {} does not contain any text to extract knowledge from.",
+ contentPart.getKey(),ci.getUri());
return;
}
- //TODO: determin the language
+ //TODO: determine the language
String language = "en";
- log.debug("computeEnhancements for ContentItem {} language {} text={}",
- new Object []{ci.getUri().getUnicodeString(), language, StringUtils.abbreviate(text, 100)});
+ log.debug("computeEnhancements for ContentPart {} of ContentItem {} language {} text={}",
+ new Object [] { contentPart.getKey(),ci.getUri().getUnicodeString(),
+ language, StringUtils.abbreviate(text, 100) });
//first get the models
Tokenizer tokenizer = initTokenizer(language);
Modified: incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/engine/topic/TopicClassificationEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/engine/topic/TopicClassificationEngine.java?rev=1239618&r1=1239617&r2=1239618&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/engine/topic/TopicClassificationEngine.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/engine/topic/TopicClassificationEngine.java Thu Feb 2 13:52:27 2012
@@ -16,7 +16,6 @@
*/
package org.apache.stanbol.enhancer.engine.topic;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.NIE_PLAINTEXTCONTENT;
import java.io.File;
import java.io.IOException;
@@ -32,6 +31,7 @@ import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
@@ -62,11 +62,13 @@ import org.apache.solr.common.SolrInputD
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MoreLikeThisParams;
import org.apache.stanbol.commons.solr.utils.StreamQueryRequest;
+import org.apache.stanbol.enhancer.servicesapi.Blob;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
import org.apache.stanbol.enhancer.servicesapi.EngineException;
import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
import org.apache.stanbol.enhancer.servicesapi.InvalidContentException;
import org.apache.stanbol.enhancer.servicesapi.ServiceProperties;
+import org.apache.stanbol.enhancer.servicesapi.helper.ContentItemHelper;
import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
import org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses;
import org.apache.stanbol.enhancer.topic.Batch;
@@ -161,6 +163,15 @@ public class TopicClassificationEngine e
private static final Logger log = LoggerFactory.getLogger(TopicClassificationEngine.class);
+ /**
+ * The "text/plain" mime type
+ */
+ protected static final String PLAIN_TEXT_MIMETYPE = "text/plain";
+ /**
+ * Contains the only supported mime type {@link #PLAIN_TEXT_MIMETYPE}
+ */
+ protected static final Set<String> SUPPORTED_MIMETYPES = Collections.singleton(PLAIN_TEXT_MIMETYPE);
+
public static final String SOLR_NON_EMPTY_FIELD = "[\"\" TO *]";
// TODO: make the following fields configurable
@@ -267,20 +278,41 @@ public class TopicClassificationEngine e
@Override
public int canEnhance(ContentItem ci) throws EngineException {
- String text = getTextFromContentItem(ci);
- if (getActiveSolrServer() == null) {
- log.warn(String.format("Solr Core '%s' is not available.", solrCoreId));
- return CANNOT_ENHANCE;
- }
- if (text.trim().length() == 0) {
+ if(ContentItemHelper.getBlob(ci, SUPPORTED_MIMETYPES) != null &&
+ getActiveSolrServer() != null){
+ return ENHANCE_SYNCHRONOUS;
+ } else {
return CANNOT_ENHANCE;
}
- return ENHANCE_SYNCHRONOUS;
+ //TODO ogrisel: validate that it is no problem that this does no longer
+ //check that the text is not empty
+// if (text.trim().length() == 0) {
+// return CANNOT_ENHANCE;
+// }
}
@Override
public void computeEnhancements(ContentItem ci) throws EngineException {
- String text = getTextFromContentItem(ci);
+ Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, SUPPORTED_MIMETYPES);
+ if(contentPart == null){
+ throw new IllegalStateException("No ContentPart with a supported Mime Type"
+ + "found for ContentItem "+ci.getUri()+"(supported: '"
+ + SUPPORTED_MIMETYPES+"') -> this indicates that canEnhance was"
+ + "NOT called and indicates a bug in the used EnhancementJobManager!");
+ }
+ String text;
+ try {
+ text = ContentItemHelper.getText(contentPart.getValue());
+ } catch (IOException e) {
+ throw new InvalidContentException(String.format("Unable to extract "
+ +" textual content from ContentPart %s of ContentItem %s!",
+ contentPart.getKey(),ci.getUri()), e);
+ }
+ if(text.trim().isEmpty()){
+ log.warn("ContentPart {} of ContentItem {} does not contain any " +
+ "text to extract topics from",contentPart.getKey(),ci.getUri());
+ return;
+ }
MGraph metadata = ci.getMetadata();
List<TopicSuggestion> topics;
try {
@@ -288,18 +320,23 @@ public class TopicClassificationEngine e
} catch (ClassifierException e) {
throw new EngineException(e);
}
- for (TopicSuggestion topic : topics) {
- UriRef enhancement = EnhancementEngineHelper.createEntityEnhancement(ci, this);
- metadata.add(new TripleImpl(enhancement,
- org.apache.stanbol.enhancer.servicesapi.rdf.Properties.RDF_TYPE,
- TechnicalClasses.ENHANCER_TOPICANNOTATION));
-
- // add link to entity
- metadata.add(new TripleImpl(enhancement,
- org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_REFERENCE,
- new UriRef(topic.uri)));
- // TODO: make it possible to dereference and the path to the root the entities according to a
- // configuration parameter
+ ci.getLock().writeLock().lock();
+ try {
+ for (TopicSuggestion topic : topics) {
+ UriRef enhancement = EnhancementEngineHelper.createEntityEnhancement(ci, this);
+ metadata.add(new TripleImpl(enhancement,
+ org.apache.stanbol.enhancer.servicesapi.rdf.Properties.RDF_TYPE,
+ TechnicalClasses.ENHANCER_TOPICANNOTATION));
+
+ // add link to entity
+ metadata.add(new TripleImpl(enhancement,
+ org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_REFERENCE,
+ new UriRef(topic.uri)));
+ // TODO: make it possible to dereference and the path to the root the entities according to a
+ // configuration parameter
+ }
+ } finally {
+ ci.getLock().writeLock().unlock();
}
}
@@ -309,25 +346,6 @@ public class TopicClassificationEngine e
(Object) order));
}
- protected String getTextFromContentItem(ContentItem ci) throws InvalidContentException {
- // Refactor the following using an adapter.
- String text = "";
- if (ci.getMimeType().startsWith("text/plain")) {
- try {
- // TODO: handle explicit charsets if any and fallback to UTF-8 if missing
- text = IOUtils.toString(ci.getStream(), "UTF-8");
- } catch (IOException e) {
- throw new InvalidContentException(this, ci, e);
- }
- } else {
- Iterator<Triple> it = ci.getMetadata().filter(ci.getUri(), NIE_PLAINTEXTCONTENT, null);
- while (it.hasNext()) {
- text += it.next().getObject();
- }
- }
- return text;
- }
-
public static TopicClassificationEngine fromParameters(Dictionary<String,Object> config) throws ConfigurationException {
TopicClassificationEngine engine = new TopicClassificationEngine();
engine.configure(config);
Modified: incubator/stanbol/trunk/enhancer/engines/zemanta/src/main/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/zemanta/src/main/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngine.java?rev=1239618&r1=1239617&r2=1239618&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/zemanta/src/main/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngine.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/zemanta/src/main/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngine.java Thu Feb 2 13:52:27 2012
@@ -25,19 +25,20 @@ import static org.apache.stanbol.enhance
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_TYPE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTED_TEXT;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_START;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.NIE_PLAINTEXTCONTENT;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.RDF_TYPE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_CATEGORY;
import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_TEXTANNOTATION;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import org.apache.clerezza.rdf.core.Literal;
@@ -59,12 +60,14 @@ import org.apache.felix.scr.annotations.
import org.apache.felix.scr.annotations.Service;
import org.apache.stanbol.commons.stanboltools.offline.OnlineMode;
import org.apache.stanbol.enhancer.engines.zemanta.ZemantaOntologyEnum;
+import org.apache.stanbol.enhancer.servicesapi.Blob;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
import org.apache.stanbol.enhancer.servicesapi.EngineException;
import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
import org.apache.stanbol.enhancer.servicesapi.InvalidContentException;
import org.apache.stanbol.enhancer.servicesapi.ServiceProperties;
import org.apache.stanbol.enhancer.servicesapi.helper.AbstractEnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.helper.ContentItemHelper;
import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
import org.osgi.framework.BundleContext;
import org.osgi.service.cm.ConfigurationException;
@@ -73,6 +76,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
/**
* Apache Stanbol Enhancer Zemanta enhancement engine.
* This enhancement engine uses the the Zemanta API for enhancing content.
@@ -104,8 +108,9 @@ public class ZemantaEnhancementEngine
public static final String DMOZ_BASE_URL = "http://www.dmoz.org/";
public static final String ZEMANTA_DMOZ_PREFIX = "Top/";
- protected static final String TEXT_PLAIN_MIMETYPE = "text/plain";
- protected static final String TEXT_HTML_MIMETYPE = "text/html";
+ protected static final Set<String> SUPPORTED_MIMETYPES =
+ Collections.unmodifiableSet(new HashSet<String>(
+ Arrays.asList("text/plain","text/html")));
private static final Logger log = LoggerFactory.getLogger(ZemantaEnhancementEngine.class);
@@ -158,53 +163,31 @@ public class ZemantaEnhancementEngine
}
public int canEnhance(ContentItem ci) {
- if(isTextOrHtml(ci)){
- return ENHANCE_SYNCHRONOUS;
+ if(ContentItemHelper.getBlob(ci, SUPPORTED_MIMETYPES) != null){
+ return ENHANCE_ASYNC; //the ZEMANTA engine now supports async processing!
} else {
- // check for existence of textual content in metadata
- UriRef subj = ci.getUri();
- Iterator<Triple> it = ci.getMetadata().filter(subj, NIE_PLAINTEXTCONTENT, null);
- if (it.hasNext()) {
- return ENHANCE_SYNCHRONOUS;
- }
+ return CANNOT_ENHANCE;
}
- return CANNOT_ENHANCE;
}
- /**
- * @param ci
- */
- private boolean isTextOrHtml(ContentItem ci) {
- String mimeType = ci.getMimeType().split(";", 2)[0];
- if (TEXT_PLAIN_MIMETYPE.equalsIgnoreCase(mimeType)) {
- return true;
- } else if (TEXT_HTML_MIMETYPE.equalsIgnoreCase(mimeType)) {
- return true;
- } else {
- return false;
- }
- }
public void computeEnhancements(ContentItem ci) throws EngineException {
+ Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, SUPPORTED_MIMETYPES);
+ if(contentPart == null){
+ throw new IllegalStateException("No ContentPart with a supported Mime Type"
+ + "found for ContentItem "+ci.getUri()+"(supported: '"
+ + SUPPORTED_MIMETYPES+"') -> this indicates that canEnhance was"
+ + "NOT called and indicates a bug in the used EnhancementJobManager!");
+ }
String text;
- if(isTextOrHtml(ci)){
- try {
- text = IOUtils.toString(ci.getStream(),"UTF-8");
- } catch (IOException e) {
- throw new InvalidContentException(this, ci, e);
- }
- } else {
- //TODO: change that as soon the Adapter Pattern is used for multiple
- // mimetype support.
- StringBuilder textBuilder = new StringBuilder();
- Iterator<Triple> it = ci.getMetadata().filter(ci.getUri(), NIE_PLAINTEXTCONTENT, null);
- while (it.hasNext()) {
- textBuilder.append(it.next().getObject());
- }
- text = textBuilder.toString();
+ try {
+ text = ContentItemHelper.getText(contentPart.getValue());
+ } catch (IOException e) {
+ throw new InvalidContentException(this, ci, e);
}
if (text.trim().length() == 0) {
- log.warn("nothing to enhance");
+ log.warn("ContentPart {} of ContentItem {} does not contain any text to enhance",
+ contentPart.getKey(),ci.getUri());
return;
}
MGraph graph = ci.getMetadata();
@@ -219,8 +202,13 @@ public class ZemantaEnhancementEngine
}
//now we need to process the results and convert them into the Enhancer
//annotation structure
- processRecognition(results, graph, text, ciId);
- processCategories(results, graph, ciId);
+ ci.getLock().writeLock().lock();
+ try {
+ processRecognition(results, graph, text, ciId);
+ processCategories(results, graph, ciId);
+ } finally {
+ ci.getLock().writeLock().unlock();
+ }
}
public Map<String, Object> getServiceProperties() {
// TODO Auto-generated method stub
Modified: incubator/stanbol/trunk/enhancer/generic/servicesapi/src/main/java/org/apache/stanbol/enhancer/servicesapi/helper/ContentItemHelper.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/generic/servicesapi/src/main/java/org/apache/stanbol/enhancer/servicesapi/helper/ContentItemHelper.java?rev=1239618&r1=1239617&r2=1239618&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/generic/servicesapi/src/main/java/org/apache/stanbol/enhancer/servicesapi/helper/ContentItemHelper.java (original)
+++ incubator/stanbol/trunk/enhancer/generic/servicesapi/src/main/java/org/apache/stanbol/enhancer/servicesapi/helper/ContentItemHelper.java Thu Feb 2 13:52:27 2012
@@ -22,16 +22,21 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
+import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
import java.util.StringTokenizer;
import org.apache.clerezza.rdf.core.UriRef;
import org.apache.commons.io.IOUtils;
import org.apache.stanbol.enhancer.servicesapi.Blob;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.enhancer.servicesapi.NoSuchPartException;
/**
@@ -48,6 +53,8 @@ public class ContentItemHelper {
public static final int MAX_BUF_SIZE = 64 * 1024; // 64 kB
private static final char[] HEX_DIGITS = "0123456789abcdef".toCharArray();
+
+ public static final String UTF8 = "UTF-8";
// TODO: instead of using a static helper, build an OSGi component with a
// configurable site-wide URI namespace for ids that are local to the
@@ -214,4 +221,68 @@ public class ContentItemHelper {
}
return parsed;
}
+ /**
+ * Searches an {@link ContentItem#getPart(UriRef, Class) content part}
+ * of the type {@link Blob} with one of the the parsed mimeTypes. <p>
+ * NOTE:<ul>
+ * <li> MimeTypes are converted to lower case before compared with
+ * the entries of the parsed set. Therefore it is important that the parsed
+ * set only contains lower case values!
+ * <li> A read lock on the parsed {@link ContentItem} is applied while
+ * searching for a fitting {@link Blob}
+ * </ul>
+ * @param ci the contentITem
+ * @param mimeTypes List of possible mimeTypes
+ * @return the {@link UriRef URI} and the {@link Blob content} of the content
+ * part or <code>null</code> if not found
+ * @throws IllegalArgumentException If the parsed {@link ContentItem} is
+ * <code>null</code> or the parsed Set with the mimeTypes is <code>null</code>
+ * or {@link Set#isEmpty() empty}.
+ */
+ public static Entry<UriRef, Blob> getBlob(ContentItem ci, Set<String> mimeTypes){
+ if(ci == null){
+ throw new IllegalArgumentException("The parsed ContentItem MUST NOT be NULL!");
+ }
+ if(mimeTypes == null || mimeTypes.isEmpty()){
+ throw new IllegalArgumentException("The parsed Set with mime type MUST NOT be NULL nor empty!");
+ }
+ UriRef cpUri = null;
+ int index = 0;
+ ci.getLock().readLock().lock();
+ try {
+ do {
+ try {
+ cpUri = ci.getPartUri(index);
+ if(cpUri != null){
+ Blob blob = ci.getPart(cpUri, Blob.class);
+ if(blob != null && mimeTypes.contains(
+ blob.getMimeType().toLowerCase())){
+ return Collections.singletonMap(cpUri, blob)
+ .entrySet().iterator().next();
+ } // else no match
+ } // else no more parts
+ } catch (NoSuchPartException e) {/* ignore*/}
+ } while(cpUri != null);
+ } finally {
+ ci.getLock().readLock().unlock();
+ }
+ return null; // not found
+ }
+ /**
+ * Getter for the Text of an {@link Blob}. This method respects the
+ * "charset" if present in the {@link Blob#getParameter() parameter} of the
+ * Blob.
+ * @param blob the {@link Blob}. MUST NOT be <code>null</code>.
+ * @return the text
+ * @throws IOException on any exception while reading from the
+ * {@link InputStream} provided by the Blob.
+ * @throws IllegalArgumentException if the parsed Blob is <code>null</code>
+ */
+ public static String getText(Blob blob) throws IOException {
+ if(blob == null){
+ throw new IllegalArgumentException("The parsed Blob MUST NOT be NULL!");
+ }
+ String charset = blob.getParameter().get("charset");
+ return IOUtils.toString(blob.getStream(), charset != null ? charset : UTF8);
+ }
}
Modified: incubator/stanbol/trunk/enhancer/generic/servicesapi/src/main/java/org/apache/stanbol/enhancer/servicesapi/helper/ContentItemImpl.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/generic/servicesapi/src/main/java/org/apache/stanbol/enhancer/servicesapi/helper/ContentItemImpl.java?rev=1239618&r1=1239617&r2=1239618&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/generic/servicesapi/src/main/java/org/apache/stanbol/enhancer/servicesapi/helper/ContentItemImpl.java (original)
+++ incubator/stanbol/trunk/enhancer/generic/servicesapi/src/main/java/org/apache/stanbol/enhancer/servicesapi/helper/ContentItemImpl.java Thu Feb 2 13:52:27 2012
@@ -136,11 +136,16 @@ public abstract class ContentItemImpl im
@SuppressWarnings("unchecked")
@Override
public <T> T getPart(UriRef uri, Class<T> clazz) throws NoSuchPartException {
- if(parts.containsKey(uri)){
- return (T) parts.get(uri);
- } else {
- throw new NoSuchPartException(uri);
- }
+ readLock.lock();
+ try {
+ if(parts.containsKey(uri)){
+ return (T) parts.get(uri);
+ } else {
+ throw new NoSuchPartException(uri);
+ }
+ }finally {
+ readLock.unlock();
+ }
}
@Override
Modified: incubator/stanbol/trunk/enhancer/generic/servicesapi/src/main/java/org/apache/stanbol/enhancer/servicesapi/rdf/Properties.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/generic/servicesapi/src/main/java/org/apache/stanbol/enhancer/servicesapi/rdf/Properties.java?rev=1239618&r1=1239617&r2=1239618&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/generic/servicesapi/src/main/java/org/apache/stanbol/enhancer/servicesapi/rdf/Properties.java (original)
+++ incubator/stanbol/trunk/enhancer/generic/servicesapi/src/main/java/org/apache/stanbol/enhancer/servicesapi/rdf/Properties.java Thu Feb 2 13:52:27 2012
@@ -168,10 +168,6 @@ public class Properties {
public static final UriRef DC_LANGUAGE = new UriRef(NamespaceEnum.dc
+ "language");
- /**
- * Plain text content of a content item.
- */
- public static final UriRef NIE_PLAINTEXTCONTENT = new UriRef(NamespaceEnum.nie + "plainTextContent");
/**
* The topic of the resource. Used to relate a content item to a
Modified: incubator/stanbol/trunk/enhancer/jersey/src/main/java/org/apache/stanbol/enhancer/jersey/resource/ContentItemResource.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/jersey/src/main/java/org/apache/stanbol/enhancer/jersey/resource/ContentItemResource.java?rev=1239618&r1=1239617&r2=1239618&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/jersey/src/main/java/org/apache/stanbol/enhancer/jersey/resource/ContentItemResource.java (original)
+++ incubator/stanbol/trunk/enhancer/jersey/src/main/java/org/apache/stanbol/enhancer/jersey/resource/ContentItemResource.java Thu Feb 2 13:52:27 2012
@@ -32,7 +32,6 @@ import static org.apache.stanbol.enhance
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_REFERENCE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.GEO_LAT;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.GEO_LONG;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.NIE_PLAINTEXTCONTENT;
import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_ENTITYANNOTATION;
import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_TEXTANNOTATION;
@@ -51,6 +50,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.TreeMap;
import javax.servlet.ServletContext;
@@ -81,8 +81,10 @@ import org.apache.clerezza.rdf.core.spar
import org.apache.clerezza.rdf.ontologies.RDF;
import org.apache.commons.io.IOUtils;
import org.apache.stanbol.commons.web.base.resource.BaseStanbolResource;
+import org.apache.stanbol.enhancer.servicesapi.Blob;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
import org.apache.stanbol.enhancer.servicesapi.NoSuchPartException;
+import org.apache.stanbol.enhancer.servicesapi.helper.ContentItemHelper;
import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
import org.apache.stanbol.enhancer.servicesapi.helper.ExecutionMetadataHelper;
import org.apache.stanbol.enhancer.servicesapi.helper.ExecutionPlanHelper;
@@ -156,17 +158,13 @@ public class ContentItemResource extends
if (localId != null) {
URI rawURI = uriInfo.getBaseUriBuilder().path(storePath).path("raw").path(localId).build();
- if (ci.getMimeType().equals("text/plain")) {
- this.textContent = IOUtils.toString(ci.getStream(), "UTF-8");
- } else if (ci.getMimeType().startsWith("image/")) {
+ Entry<UriRef,Blob> plainTextContentPart = ContentItemHelper.getBlob(contentItem, Collections.singleton("text/plain"));
+ if (plainTextContentPart != null) {
+ this.textContent = ContentItemHelper.getText(plainTextContentPart.getValue());
+ }
+ if (ci.getMimeType().startsWith("image/")) {
this.imageSrc = rawURI;
}
- else {
- Iterator<Triple> it = ci.getMetadata().filter(ci.getUri(), NIE_PLAINTEXTCONTENT, null);
- if (it.hasNext()) {
- this.textContent = ((Literal)it.next().getObject()).getLexicalForm();
- }
- }
this.downloadHref = rawURI;
this.metadataHref = uriInfo.getBaseUriBuilder().path(storePath).path("metadata").path(localId).build();
}
Modified: incubator/stanbol/trunk/enhancer/jobmanager/event/src/main/java/org/apache/stanbol/enhancer/jobmanager/event/impl/EnhancementJobHandler.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/jobmanager/event/src/main/java/org/apache/stanbol/enhancer/jobmanager/event/impl/EnhancementJobHandler.java?rev=1239618&r1=1239617&r2=1239618&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/jobmanager/event/src/main/java/org/apache/stanbol/enhancer/jobmanager/event/impl/EnhancementJobHandler.java (original)
+++ incubator/stanbol/trunk/enhancer/jobmanager/event/src/main/java/org/apache/stanbol/enhancer/jobmanager/event/impl/EnhancementJobHandler.java Thu Feb 2 13:52:27 2012
@@ -143,11 +143,12 @@ public class EnhancementJobHandler imple
try {
processEvent(job, execution);
} catch (Throwable t) {
+ String message = String.format("Unexpected Exception while processing " +
+ "ContentItem %s with EnhancementJobManager: %s",
+ job.getContentItem().getUri(),EventJobManagerImpl.class);
//this ensures that an runtime exception does not
- job.setFailed(execution, null, new IllegalStateException(
- "Unexpected Exception while processing ContentItem '"
- + job.getContentItem().getUri()+"' with EnhancementJobManager: "
- + EventJobManagerImpl.class,t));
+ job.setFailed(execution, null, new IllegalStateException(message,t));
+ log.error(message,t);
}
//(2) trigger the next actions
log.debug("++ w: {}","check for next Executions");
@@ -218,8 +219,13 @@ public class EnhancementJobHandler imple
} catch (EngineException e) {
job.setFailed(execution, engine, e);
}
- } else { //required engine is unable to enhance the content
- job.setFailed(execution,engine,exception);
+ } else { //CANNOT_ENHANCE
+ if(exception != null){
+ job.setFailed(execution,engine,exception);
+ } else { //can not enhance is not an error
+ //it just says this engine can not enhance this content item
+ job.setCompleted(execution);
+ }
}
} else { //engine with that name is not available
job.setFailed(execution, null, null);