You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2012/08/23 11:57:40 UTC
svn commit: r1376420 - in
/incubator/stanbol/branches/dbpedia-spotlight-engines:
bundlelist/src/main/bundles/ engines/dbpedia-spotlight-annotate/
engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate...
Author: rwesten
Date: Thu Aug 23 09:57:39 2012
New Revision: 1376420
URL: http://svn.apache.org/viewvc?rev=1376420&view=rev
Log:
STANBOL-706: Applied chages similar to the Spot Engine to the candidates and annotate. Added functionality to the annotate engine to extract the best fitting dc:type value for the created TextAnnotation fromt the type information of the Annotation.
Added:
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/DBPSpotlightAnnotateEnhancementTest.java
- copied, changed from r1375110, incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/core/DBPSpotlightAnnotateEnhancementTest.java
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/DBPSpotlightCandidatesEnhancementTest.java
- copied, changed from r1375110, incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/core/DBPSpotlightCandidatesEnhancementTest.java
Removed:
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/core/
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlightannotate/
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/core/
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlightcandidates/
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-spot/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/spot/core/
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-spot/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlightspot/
Modified:
incubator/stanbol/branches/dbpedia-spotlight-engines/bundlelist/src/main/bundles/list.xml
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/pom.xml
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/Annotation.java
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/DBPSpotlightAnnotateEnhancementEngine.java
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/XMLParser.java
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/pom.xml
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/DBPSpotlightCandidatesEnhancementEngine.java
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/XMLParser.java
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-spot/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/spot/DBPSpotlightSpotEnhancementEngine.java
incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-spot/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/spot/DBPSpotlightSpotEnhancementTest.java
Modified: incubator/stanbol/branches/dbpedia-spotlight-engines/bundlelist/src/main/bundles/list.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/dbpedia-spotlight-engines/bundlelist/src/main/bundles/list.xml?rev=1376420&r1=1376419&r2=1376420&view=diff
==============================================================================
--- incubator/stanbol/branches/dbpedia-spotlight-engines/bundlelist/src/main/bundles/list.xml (original)
+++ incubator/stanbol/branches/dbpedia-spotlight-engines/bundlelist/src/main/bundles/list.xml Thu Aug 23 09:57:39 2012
@@ -198,12 +198,22 @@
<version>0.10.0-incubating-SNAPSHOT</version>
</bundle>
- <bundle> <!-- DBpedia Spotlight Engines (STANBOL-706) -->
+ <!-- DBpedia Spotlight Engines (STANBOL-706) -->
+ <bundle>
<groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.enhancer.engines.dbpspotlight.spot</artifactId>
<version>0.10.0-incubating-SNAPSHOT</version>
</bundle>
-
+ <bundle>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.dbpspotlight.candidates</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ </bundle>
+ <bundle>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.dbpspotlight.annotate</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ </bundle>
</startLevel>
Modified: incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/pom.xml?rev=1376420&r1=1376419&r2=1376420&view=diff
==============================================================================
--- incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/pom.xml (original)
+++ incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/pom.xml Thu Aug 23 09:57:39 2012
@@ -79,6 +79,12 @@
</properties>
<dependencies>
+ <dependency><!-- to ensure deactivation in offline mode -->
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.commons.stanboltools.offline</artifactId>
+ <version>0.9.0-incubating</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.enhancer.servicesapi</artifactId>
@@ -107,11 +113,29 @@
<artifactId>slf4j-api</artifactId>
</dependency>
+ <!-- test -->
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.test</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.core</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
Modified: incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/Annotation.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/Annotation.java?rev=1376420&r1=1376419&r2=1376420&view=diff
==============================================================================
--- incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/Annotation.java (original)
+++ incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/Annotation.java Thu Aug 23 09:57:39 2012
@@ -16,17 +16,57 @@
*/
package org.apache.stanbol.enhancer.engines.dbpspotlight.annotate;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
import org.apache.clerezza.rdf.core.Resource;
/**
* Contains a result given by DBPedia Spotlight..
*
+ *
* @author <a href="mailto:iavor.jelev@babelmonkeys.com">Iavor Jelev</a>
*/
public class Annotation {
+ /*
+ * TODO (Note by rwesten 2012-08-22)
+ *
+ * Added here functionality to extract DBpedia
+ * Ontoloty types for Annotations. This is mainly to
+ * choose the best dc:type for fise:TextAnnotations
+ * created for Annotation.
+ *
+ * This is based on the assumption that the most generic
+ * dbpedia type is always the last one in the returned list.
+ *
+ * In addition "DBpedia:TopicalConcept" is ignored first
+ * as it seams not to be used by dbpedia.org and second
+ * because it is always parsed last (even after schema
+ * and freebase types) and would therefore be considered
+ * as the most generic dbpedia type.
+ *
+ * I do not like this solution and would like to find
+ * a better solution for that
+ */
+ /**
+ * Allows to add DBpedia Ontology types that should be
+ * ignored by {@link #getDbpediaTypeNames()}.<p>
+ * Introduced this to ignore the "TopicalConcept"
+ * type.
+ */
+ public static final Set<String> IGNORED_DBP_TYPES;
+ static {
+ Set<String> ignored = new HashSet<String>();
+ ignored.add("DBpedia:TopicalConcept");
+ IGNORED_DBP_TYPES = Collections.unmodifiableSet(ignored);
+ }
+
public Resource uri;
public String types;
public Integer support;
@@ -35,20 +75,40 @@ public class Annotation {
public Double similarityScore;
public Double percentageOfSecondRank;
- public HashSet<String> getTypeNames() {
+ public List<String> getTypeNames() {
if (types != null) {
- HashSet<String> t = new HashSet<String>();
+ List<String> t = new ArrayList<String>();
String[] typex = types.split(",");
for (String type : typex) {
// make the returned types referenceable
- String deref = type.replace("DBpedia:", "dbp-ont:")
+ String deref = type.replace("DBpedia:", "http://dbpedia.org/ontology/")
.replace("Freebase:", "http://www.freebase.com/schema")
.replace("Schema:", "http://www.schema.org/");
- t.add(deref);
+ if(!deref.isEmpty()){
+ t.add(deref);
+ }
+ }
+ return t;
+ }
+ return Collections.emptyList();
+ }
+
+ /**
+ * Getter for the dbpedia ontology types excluding {@link #IGNORED_DBP_TYPES}
+ * @return the types or an empty list if none
+ */
+ public List<String> getDbpediaTypeNames(){
+ if (types != null) {
+ List<String> t = new ArrayList<String>();
+ String[] typex = types.split(",");
+ for (String type : typex) {
+ if(!IGNORED_DBP_TYPES.contains(type) && type.startsWith("DBpedia:")){
+ t.add(type.replace("DBpedia:", "http://dbpedia.org/ontology/"));
+ }
}
return t;
}
- return null;
+ return Collections.emptyList();
}
public String toString() {
@@ -57,4 +117,7 @@ public class Annotation {
uri, support, types, surfaceForm, offset,
similarityScore, percentageOfSecondRank);
}
+ public static void main(String[] args) {
+ System.err.print(Integer.MIN_VALUE);
+ }
}
Modified: incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/DBPSpotlightAnnotateEnhancementEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/DBPSpotlightAnnotateEnhancementEngine.java?rev=1376420&r1=1376419&r2=1376420&view=diff
==============================================================================
--- incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/DBPSpotlightAnnotateEnhancementEngine.java (original)
+++ incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/DBPSpotlightAnnotateEnhancementEngine.java Thu Aug 23 09:57:39 2012
@@ -16,6 +16,8 @@
*/
package org.apache.stanbol.enhancer.engines.dbpspotlight.annotate;
+import static org.apache.stanbol.enhancer.engines.dbpspotlight.annotate.XMLParser.getElementsByTagName;
+import static org.apache.stanbol.enhancer.engines.dbpspotlight.annotate.XMLParser.loadXMLFromInputStream;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_LANGUAGE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_RELATION;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_TYPE;
@@ -24,24 +26,28 @@ import static org.apache.stanbol.enhance
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_REFERENCE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_TYPE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTED_TEXT;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTION_CONTEXT;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_START;
-import java.io.BufferedReader;
+import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
+import java.nio.charset.Charset;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Dictionary;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -57,10 +63,14 @@ import org.apache.clerezza.rdf.core.UriR
import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl;
import org.apache.clerezza.rdf.core.impl.TripleImpl;
import org.apache.clerezza.rdf.core.serializedform.Serializer;
+import org.apache.commons.io.IOUtils;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Properties;
import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
import org.apache.felix.scr.annotations.Service;
+import org.apache.stanbol.commons.stanboltools.offline.OfflineMode;
+import org.apache.stanbol.commons.stanboltools.offline.OnlineMode;
import org.apache.stanbol.enhancer.servicesapi.Blob;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
import org.apache.stanbol.enhancer.servicesapi.EngineException;
@@ -77,6 +87,7 @@ import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
/**
* {@link DBPSpotlightAnnotateEnhancementEngine} provides functionality to
@@ -91,6 +102,8 @@ public class DBPSpotlightAnnotateEnhance
AbstractEnhancementEngine<IOException, RuntimeException> implements
EnhancementEngine, ServiceProperties {
+ private static final Charset UTF8 = Charset.forName("UTF-8");
+
/**
* a configurable value of the text segment length to check
*/
@@ -116,6 +129,13 @@ public class DBPSpotlightAnnotateEnhance
public static final String SL_CONFIDENCE = "stanbol.DBPSpotlightAnnotateEnhancementEngine.confidence";
/**
+ * Ensures this engine is deactivated in {@link OfflineMode}
+ */
+ @SuppressWarnings("unused")
+ @Reference
+ private OnlineMode onlineMode;
+
+ /**
* The default value for the Execution of this Engine.
*/
public static final Integer defaultOrder = ORDERING_CONTENT_EXTRACTION - 27;
@@ -128,13 +148,20 @@ public class DBPSpotlightAnnotateEnhance
/** Set containing the only supported mime type {@link #TEXT_PLAIN_MIMETYPE} */
private static final Set<String> SUPPORTED_MIMTYPES = Collections
.singleton(TEXT_PLAIN_MIMETYPE);
+ /**
+ * This contains a list of languages supported by DBpedia Spotlight. If the
+ * metadata doesn't contain a value for the language as the value of the
+ * {@link Property.DC_LANG property} the content can't be processed.
+ */
+ protected static final Set<String> SUPPORTED_LANGUAGES = Collections
+ .unmodifiableSet(new HashSet<String>(Arrays.asList("en")));
/** holds the logger. */
private static final Logger log = LoggerFactory
.getLogger(DBPSpotlightAnnotateEnhancementEngine.class);
/** holds the url of the Spotlight REST endpoint */
- private String spotlightUrl;
+ private URL spotlightUrl;
/** holds the chosen of spotter to be used */
private String spotlightSpotter;
/** holds the chosen of disambiguator to be used */
@@ -149,6 +176,20 @@ public class DBPSpotlightAnnotateEnhance
private String spotlightSparql;
/**
+ * Default constructor used by OSGI. Expects {@link #activate(ComponentContext)}
+ * to be called before the instance is used.
+ */
+ public DBPSpotlightAnnotateEnhancementEngine(){}
+
+ /**
+ * Constructor intended to be used by unit tests
+ * @param spotlightUrl
+ */
+ protected DBPSpotlightAnnotateEnhancementEngine(URL spotlightUrl){
+ this.spotlightUrl = spotlightUrl;
+ }
+
+ /**
* Initialize all parameters from the configuration panel, or with their
* default values
*
@@ -162,8 +203,20 @@ public class DBPSpotlightAnnotateEnhance
super.activate(ce);
Dictionary<String, Object> properties = ce.getProperties();
- spotlightUrl = properties.get(SL_URL_KEY) == null ? "http://spotlight.dbpedia.org/rest/annotate"
- : (String) properties.get(SL_URL_KEY);
+ Object value = properties.get(SL_URL_KEY);
+ if(value == null || value.toString().isEmpty()){
+ throw new ConfigurationException(SL_URL_KEY, "The URL with the DBpedia "
+ + "Spotlight Annotate RESTful Service MUST NOT be NULL nor empty!");
+ } else {
+ String url = (String) properties.get(SL_URL_KEY);
+ try {
+ this.spotlightUrl = new URL(url);
+ } catch (MalformedURLException e) {
+ throw new ConfigurationException(SL_URL_KEY, "The parsed URL for the "
+ + "DBpedia Spotlight Annotate RESTful Service is illegal formatted!",
+ e);
+ }
+ }
spotlightSpotter = properties.get(SL_SPOTTER) == null ? null
: (String) properties.get(SL_SPOTTER);
spotlightDisambiguator = properties.get(SL_DISAMBIGUATOR) == null ? null
@@ -186,7 +239,14 @@ public class DBPSpotlightAnnotateEnhance
*/
public int canEnhance(ContentItem ci) throws EngineException {
if (ContentItemHelper.getBlob(ci, SUPPORTED_MIMTYPES) != null) {
- return ENHANCE_SYNCHRONOUS;
+ String language = EnhancementEngineHelper.getLanguage(ci);
+ if (!SUPPORTED_LANGUAGES.contains(language)) {
+ log.info("DBpedia Spotlight can not process ContentItem {} "
+ + "because language {} is not supported (supported: {})",
+ new Object[] { ci.getUri(), language, SUPPORTED_LANGUAGES });
+ return CANNOT_ENHANCE;
+ }
+ return ENHANCE_ASYNC;
} else {
return CANNOT_ENHANCE;
}
@@ -200,6 +260,18 @@ public class DBPSpotlightAnnotateEnhance
* the {@link ContentItem}
*/
public void computeEnhancements(ContentItem ci) throws EngineException {
+ Language language;
+ String lang = EnhancementEngineHelper.getLanguage(ci);
+ if(!SUPPORTED_LANGUAGES.contains(lang)){
+ throw new IllegalStateException("Langage '"+lang
+ + "' as annotated for ContentItem "
+ + ci.getUri() + " is not supported by this Engine: "
+ + "This is also checked in the canEnhance method! -> This "
+ + "indicated an Bug in the implementation of the "
+ + "EnhancementJobManager!");
+ } else {
+ language = lang == null || lang.isEmpty() ? null : new Language(lang);
+ }
Entry<UriRef, Blob> contentPart = ContentItemHelper.getBlob(ci,
SUPPORTED_MIMTYPES);
if (contentPart == null) {
@@ -212,20 +284,20 @@ public class DBPSpotlightAnnotateEnhance
+ "indicated an Bug in the implementation of the "
+ "EnhancementJobManager!");
}
- String text = "";
+ String text;
try {
text = ContentItemHelper.getText(contentPart.getValue());
} catch (IOException e) {
throw new InvalidContentException(this, ci, e);
}
- Collection<Annotation> dbpslGraph = doPostRequest(text);
+ Collection<Annotation> dbpslGraph = doPostRequest(text,ci.getUri());
if (dbpslGraph != null) {
// Acquire a write lock on the ContentItem when adding the
// enhancements
ci.getLock().writeLock().lock();
try {
- createEnhancements(dbpslGraph, ci);
+ createEnhancements(dbpslGraph, ci, text, language);
if (log.isDebugEnabled()) {
Serializer serializer = Serializer.getInstance();
ByteArrayOutputStream debugStream = new ByteArrayOutputStream();
@@ -255,18 +327,9 @@ public class DBPSpotlightAnnotateEnhance
* @param ci
* the content item
*/
- public void createEnhancements(Collection<Annotation> occs,
- ContentItem ci) {
+ protected void createEnhancements(Collection<Annotation> occs,
+ ContentItem ci, String text, Language language) {
LiteralFactory literalFactory = LiteralFactory.getInstance();
- final Language language; // used for plain literals representing parts
- // fo the content
- String langString = getMetadataLanguage(ci.getMetadata(), null);
-
- if (langString != null && !langString.isEmpty()) {
- language = new Language(langString);
- } else {
- language = null;
- }
HashMap<Resource, UriRef> entityAnnotationMap = new HashMap<Resource, UriRef>();
@@ -281,17 +344,17 @@ public class DBPSpotlightAnnotateEnhance
model.add(new TripleImpl(textAnnotation, ENHANCER_END,
literalFactory.createTypedLiteral(occ.offset
+ occ.surfaceForm.length())));
- // TODO ################## model.add(new TripleImpl(textAnnotation,
- // ENHANCER_SELECTION_CONTEXT, new
- // PlainLiteralImpl(occ.context,language)));
- // create EntityAnnotation only once but add a reference to the
- // textAnnotation
+ model.add(new TripleImpl(textAnnotation, ENHANCER_SELECTION_CONTEXT,
+ new PlainLiteralImpl(
+ getSelectionContext(text, occ.surfaceForm, occ.offset),
+ language)));
if (entityAnnotationMap.containsKey(occ.uri)) {
model.add(new TripleImpl(entityAnnotationMap.get(occ.uri),
DC_RELATION, textAnnotation));
} else {
UriRef entityAnnotation = EnhancementEngineHelper
.createEntityEnhancement(ci, this);
+ log.info(" annotation: {} {}",occ.uri,occ.surfaceForm);
entityAnnotationMap.put(occ.uri, entityAnnotation);
Literal label = new PlainLiteralImpl(occ.surfaceForm,
new Language("en"));
@@ -299,20 +362,26 @@ public class DBPSpotlightAnnotateEnhance
textAnnotation));
model.add(new TripleImpl(entityAnnotation,
ENHANCER_ENTITY_LABEL, label));
-
- HashSet<String> t = occ.getTypeNames();
- if (t != null) {
- Iterator<String> it = t.iterator();
- while (it.hasNext()) {
- UriRef annotationType = new UriRef(it.next());
- model.add(new TripleImpl(entityAnnotation,
- ENHANCER_ENTITY_TYPE, annotationType));
- model.add(new TripleImpl(textAnnotation, DC_TYPE,
- annotationType));
- }
- }
model.add(new TripleImpl(entityAnnotation,
ENHANCER_ENTITY_REFERENCE, occ.uri));
+ //set the fise:entity-type
+ for(String type : occ.getTypeNames()){
+ UriRef annotationType = new UriRef(type);
+ log.info(" > {}",annotationType);
+ model.add(new TripleImpl(entityAnnotation,
+ ENHANCER_ENTITY_TYPE, annotationType));
+ }
+ //set the dc:type of the fise:TextAnnotation if not yet done
+ List<String> dbpTypes = occ.getDbpediaTypeNames();
+ if(!dbpTypes.isEmpty() && !model.filter(textAnnotation, DC_TYPE, null).hasNext()){
+ //use the last of the dbpedia ontology type as they
+ //are sorted from the most specific to the most
+ //common one - the dc:type should be a common one
+ UriRef dcType = new UriRef(dbpTypes.get(dbpTypes.size()-1));
+ log.info(" dcType={}",dcType);
+ model.add(new TripleImpl(textAnnotation, DC_TYPE,
+ dcType));
+ }
}
}
}
@@ -322,53 +391,19 @@ public class DBPSpotlightAnnotateEnhance
*
* @param text
* a <code>String</code> with the text to be analyzed
+ * @param contentItemUri
+ * the URI of the content item (only used for logging)
* @return a <code>Collection<DBPSLAnnotation></code> with the server
* response
* @throws EngineException
* if the request cannot be sent
*/
- public Collection<Annotation> doPostRequest(String text)
+ protected Collection<Annotation> doPostRequest(String text, UriRef contentItemUri)
throws EngineException {
- StringBuilder data = new StringBuilder();
- try {
- if (spotlightSpotter != null && !spotlightSpotter.isEmpty())
- data.append(URLEncoder.encode("spotter", "UTF-8") + "="
- + URLEncoder.encode(spotlightSpotter, "UTF-8") + "&");
- if (spotlightDisambiguator != null
- && !spotlightDisambiguator.isEmpty())
- data.append(URLEncoder.encode("disambiguator", "UTF-8") + "="
- + URLEncoder.encode(spotlightDisambiguator, "UTF-8")
- + "&");
- if (spotlightTypesRestriction != null
- && !spotlightTypesRestriction.isEmpty())
- data.append(URLEncoder.encode("types", "UTF-8") + "="
- + URLEncoder.encode(spotlightTypesRestriction, "UTF-8")
- + "&");
- if (spotlightSupport != null && !spotlightSupport.isEmpty())
- data.append(URLEncoder.encode("support", "UTF-8") + "="
- + URLEncoder.encode(spotlightSupport, "UTF-8") + "&");
- if (spotlightConfidence != null && !spotlightConfidence.isEmpty())
- data.append(URLEncoder.encode("confidence", "UTF-8") + "="
- + URLEncoder.encode(spotlightConfidence, "UTF-8") + "&");
- if (spotlightSparql != null && !spotlightSparql.isEmpty()
- && spotlightTypesRestriction == null)
- data.append(URLEncoder.encode("sparql", "UTF-8") + "="
- + URLEncoder.encode(spotlightSparql, "UTF-8") + "&");
- data.append(URLEncoder.encode("text", "UTF-8") + "="
- + URLEncoder.encode(text, "UTF-8"));
- } catch (UnsupportedEncodingException e) {
- throw new EngineException(
- "Data for the httprequest could not be converted. Error: "
- + e.getMessage());
- }
-
HttpURLConnection connection = null;
- StringBuffer response = new StringBuffer();
-
+ BufferedWriter wr = null;
try {
- // Create connection
- URL url = new URL(spotlightUrl);
- connection = (HttpURLConnection) url.openConnection();
+ connection = (HttpURLConnection) spotlightUrl.openConnection();
connection.setRequestMethod("POST");
connection.setRequestProperty("Content-Type",
"application/x-www-form-urlencoded");
@@ -379,47 +414,78 @@ public class DBPSpotlightAnnotateEnhance
connection.setDoOutput(true);
// Send request
- DataOutputStream wr = new DataOutputStream(
- connection.getOutputStream());
- wr.writeBytes(data.toString());
- wr.flush();
- wr.close();
-
- // Get Response
- InputStream is = connection.getInputStream();
- BufferedReader rd = new BufferedReader(new InputStreamReader(is));
- String line;
- while ((line = rd.readLine()) != null) {
- response.append(line);
- response.append('\r');
+ wr = new BufferedWriter(new OutputStreamWriter(
+ connection.getOutputStream(),UTF8));
+ } catch (IOException e) {
+ IOUtils.closeQuietly(wr);
+ throw new EngineException("Unable to open connection to "+
+ spotlightUrl,e);
+ }
+ try {
+ if (spotlightSpotter != null && !spotlightSpotter.isEmpty()){
+ wr.write("spotter=");
+ wr.write(URLEncoder.encode(spotlightSpotter, "UTF-8"));
+ wr.write('&');
}
- rd.close();
-
- } catch (Exception e) {
-
- e.printStackTrace();
- return null;
-
- } finally {
-
- if (connection != null) {
- connection.disconnect();
+ if (spotlightDisambiguator != null
+ && !spotlightDisambiguator.isEmpty()){
+ wr.write("disambiguator=");
+ wr.write(URLEncoder.encode(spotlightDisambiguator, "UTF-8"));
+ wr.write('&');
+ }
+ if (spotlightTypesRestriction != null
+ && !spotlightTypesRestriction.isEmpty()) {
+ wr.write("types=");
+ wr.write(URLEncoder.encode(spotlightTypesRestriction, "UTF-8"));
+ wr.write('&');
+ }
+ if (spotlightSupport != null && !spotlightSupport.isEmpty()) {
+ wr.write("support=");
+ wr.write(URLEncoder.encode(spotlightSupport, "UTF-8"));
+ wr.write('&');
+ }
+ if (spotlightConfidence != null && !spotlightConfidence.isEmpty()) {
+ wr.write("confidence=");
+ wr.write(URLEncoder.encode(spotlightConfidence, "UTF-8"));
+ wr.write('&');
}
+ if (spotlightSparql != null && !spotlightSparql.isEmpty()
+ && spotlightTypesRestriction == null) {
+ wr.write("sparql=");
+ wr.write(URLEncoder.encode(spotlightSparql, "UTF-8"));
+ wr.write('&');
+ }
+ wr.write("text=");
+ wr.write(URLEncoder.encode(text, "UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ throw new IllegalStateException(
+ "The platform does not support encoding " + UTF8.name(),e);
+ } catch (IOException e) {
+ throw new EngineException("Unable to write 'plain/text' content "
+ + "for ContentItem "+contentItemUri+" to "
+ + spotlightUrl,e);
+ } finally {
+ IOUtils.closeQuietly(wr);
}
-
- // Parse the response
- XMLParser xmlParser = new XMLParser();
+ InputStream is = null;
+ Document xmlDoc;
try {
- Document xmlDoc = xmlParser.loadXMLFromString(response.toString());
- NodeList nlist = xmlParser.getElementsByTagName(xmlDoc, "Resource");
- Collection<Annotation> annos = this.getAnnotations(nlist);
-
- return annos;
- } catch (Exception e) {
- throw new EngineException(
- "Response XML could not be parsed. Error: "
- + e.getMessage());
+ // Get Response
+ is = connection.getInputStream();
+ xmlDoc = loadXMLFromInputStream(is);
+ } catch (IOException e) {
+ throw new EngineException("Unable to spot Entities with"
+ + "Dbpedia Spotlight Annotate RESTful Serice running at "
+ + spotlightUrl,e);
+ } catch(SAXException e) {
+ throw new EngineException("Unable to parse Response from "
+ + "Dbpedia Spotlight Annotate RESTful Serice running at "
+ + spotlightUrl,e);
+ } finally {
+ IOUtils.closeQuietly(is);
}
+ NodeList nlist = getElementsByTagName(xmlDoc, "Resource");
+ return getAnnotations(nlist);
}
/**
@@ -460,33 +526,38 @@ public class DBPSpotlightAnnotateEnhance
ENHANCEMENT_ENGINE_ORDERING, (Object) defaultOrder));
}
- public String getMetadataLanguage(MGraph model, NonLiteral subj) {
- Iterator<Triple> it = model.filter(subj, DC_LANGUAGE, null);
- if (it.hasNext()) {
- Resource langNode = it.next().getObject();
- return getLexicalForm(langNode);
- }
- return null;
- }
-
- public String getLexicalForm(Resource res) {
- if (res == null) {
- return null;
- } else if (res instanceof Literal) {
- return ((Literal) res).getLexicalForm();
- } else {
- return res.toString();
- }
- }
-
- /**
- * This method is used by the test class to set the endpoint url
- *
- * @param url
- * String the url of the Spotlight endpoint
- */
- public void setEndpointUrl(String url) {
- spotlightUrl = url;
- }
+ private static final int DEFAULT_SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE = 50;
+ /**
+ * Extracts the selection context based on the content, selection and
+ * the start char offset of the selection
+ * @param content the content
+ * @param selection the selected text
+ * @param selectionStartPos the start char position of the selection
+ * @return the context
+ */
+ protected static String getSelectionContext(String content, String selection,int selectionStartPos){
+ //extract the selection context
+ int beginPos;
+ if(selectionStartPos <= DEFAULT_SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE){
+ beginPos = 0;
+ } else {
+ int start = selectionStartPos-DEFAULT_SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE;
+ beginPos = content.indexOf(' ',start);
+ if(beginPos < 0 || beginPos >= selectionStartPos){ //no words
+ beginPos = start; //begin within a word
+ }
+ }
+ int endPos;
+ if(selectionStartPos+selection.length()+DEFAULT_SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE >= content.length()){
+ endPos = content.length();
+ } else {
+ int start = selectionStartPos+selection.length()+DEFAULT_SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE;
+ endPos = content.lastIndexOf(' ', start);
+ if(endPos <= selectionStartPos+selection.length()){
+ endPos = start; //end within a word;
+ }
+ }
+ return content.substring(beginPos, endPos);
+ }
}
Modified: incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/XMLParser.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/XMLParser.java?rev=1376420&r1=1376419&r2=1376420&view=diff
==============================================================================
--- incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/XMLParser.java (original)
+++ incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/XMLParser.java Thu Aug 23 09:57:39 2012
@@ -34,15 +34,19 @@ import org.xml.sax.SAXException;
*
* @author <a href="mailto:iavor.jelev@babelmonkeys.com">Iavor Jelev</a>
*/
+public final class XMLParser {
-public class XMLParser {
-
- public NodeList getElementsByTagName(Document doc, String tagName) {
+ /**
+ * Do not create instances of Utility Classes
+ */
+ private XMLParser(){};
+
+ public static NodeList getElementsByTagName(Document doc, String tagName) {
return doc.getElementsByTagName(tagName);
}
- public Document loadXMLFromString(String xml) throws SAXException,
+ public static Document loadXMLFromString(String xml) throws SAXException,
IOException {
Document doc = loadXMLFromInputStream(new ByteArrayInputStream(
xml.getBytes()));
@@ -51,7 +55,7 @@ public class XMLParser {
return doc;
}
- public Document loadXMLFromInputStream(InputStream is) throws SAXException,
+ public static Document loadXMLFromInputStream(InputStream is) throws SAXException,
IOException {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setNamespaceAware(true);
@@ -67,7 +71,7 @@ public class XMLParser {
return doc;
}
- public Document loadXMLFromFile(String filePath)
+ public static Document loadXMLFromFile(String filePath)
throws ParserConfigurationException, SAXException, IOException {
File fXmlFile = new File(filePath);
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
Copied: incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/DBPSpotlightAnnotateEnhancementTest.java (from r1375110, incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/core/DBPSpotlightAnnotateEnhancementTest.java)
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/DBPSpotlightAnnotateEnhancementTest.java?p2=incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/DBPSpotlightAnnotateEnhancementTest.java&p1=incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/core/DBPSpotlightAnnotateEnhancementTest.java&r1=1375110&r2=1376420&rev=1376420&view=diff
==============================================================================
--- incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/core/DBPSpotlightAnnotateEnhancementTest.java (original)
+++ incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-annotate/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/annotate/DBPSpotlightAnnotateEnhancementTest.java Thu Aug 23 09:57:39 2012
@@ -14,14 +14,38 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.stanbol.enhancer.engines.dbpspotlight.annotate.core;
+package org.apache.stanbol.enhancer.engines.dbpspotlight.annotate;
-import java.util.Collection;
+import static org.apache.stanbol.enhancer.servicesapi.EnhancementEngine.ENHANCE_ASYNC;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map.Entry;
+
+import org.apache.clerezza.rdf.core.LiteralFactory;
+import org.apache.clerezza.rdf.core.Resource;
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl;
+import org.apache.clerezza.rdf.core.impl.TripleImpl;
+import org.apache.stanbol.enhancer.contentitem.inmemory.InMemoryContentItemFactory;
import org.apache.stanbol.enhancer.engines.dbpspotlight.annotate.Annotation;
import org.apache.stanbol.enhancer.engines.dbpspotlight.annotate.DBPSpotlightAnnotateEnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.Blob;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.enhancer.servicesapi.ContentItemFactory;
import org.apache.stanbol.enhancer.servicesapi.EngineException;
+import org.apache.stanbol.enhancer.servicesapi.helper.ContentItemHelper;
+import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
+import org.apache.stanbol.enhancer.servicesapi.impl.StringSource;
+import org.apache.stanbol.enhancer.servicesapi.rdf.Properties;
+import org.apache.stanbol.enhancer.test.helper.EnhancementStructureHelper;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.osgi.service.cm.ConfigurationException;
@@ -48,17 +72,33 @@ public class DBPSpotlightAnnotateEnhance
private static String TEST_TEXT = "President Obama is meeting Angela Merkel in Berlin on Monday";
private static DBPSpotlightAnnotateEnhancementEngine dbpslight;
+ private static ContentItemFactory ciFactory = InMemoryContentItemFactory.getInstance();
+
+ private ContentItem ci;
+ private static Entry<UriRef, Blob> textContentPart;
+
@BeforeClass
- public static void oneTimeSetup() throws ConfigurationException {
- dbpslight = new DBPSpotlightAnnotateEnhancementEngine();
- dbpslight.setEndpointUrl(SPL_URL);
+ public static void oneTimeSetup() throws Exception {
+ dbpslight = new DBPSpotlightAnnotateEnhancementEngine(new URL(SPL_URL));
+ }
+ @Before
+ public void initTest() throws IOException {
+ //create the contentItem for testing
+ ci = ciFactory.createContentItem(new StringSource(TEST_TEXT));
+ assertNotNull(ci);
+ textContentPart = ContentItemHelper.getBlob(ci, Collections.singleton("text/plain"));
+ assertNotNull(textContentPart);
+ //add the language of the text
+ ci.getMetadata().add(new TripleImpl(ci.getUri(), Properties.DC_LANGUAGE,
+ new PlainLiteralImpl("en")));
+ assertEquals("en", EnhancementEngineHelper.getLanguage(ci));
}
@Test
public void testEntityExtraction() {
Collection<Annotation> entities;
try {
- entities = dbpslight.doPostRequest(TEST_TEXT);
+ entities = dbpslight.doPostRequest(TEST_TEXT,ci.getUri());
LOG.info("Found entities: {}", entities.size());
LOG.debug("Entities:\n{}", entities);
Assert.assertFalse("No entities were found!", entities.isEmpty());
@@ -67,5 +107,27 @@ public class DBPSpotlightAnnotateEnhance
+ e.getMessage(), true);
}
}
-
+ @Test
+ public void testCanEnhance() throws EngineException {
+ assertEquals(ENHANCE_ASYNC, dbpslight.canEnhance(ci));
+ }
+
+ /**
+ * Validates the Enhancements created by this engine
+ * @throws EngineException
+ */
+ @Test
+ public void testEnhancement() throws EngineException {
+ dbpslight.computeEnhancements(ci);
+ HashMap<UriRef,Resource> expectedValues = new HashMap<UriRef,Resource>();
+ expectedValues.put(Properties.ENHANCER_EXTRACTED_FROM, ci.getUri());
+ expectedValues.put(Properties.DC_CREATOR, LiteralFactory.getInstance().createTypedLiteral(
+ dbpslight.getClass().getName()));
+ //validate fise:TextAnnotations
+ EnhancementStructureHelper.validateAllTextAnnotations(
+ ci.getMetadata(), TEST_TEXT, expectedValues);
+ //validate fise:EntityAnnotations
+ EnhancementStructureHelper.validateAllEntityAnnotations(
+ ci.getMetadata(), expectedValues);
+ }
}
Modified: incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/pom.xml?rev=1376420&r1=1376419&r2=1376420&view=diff
==============================================================================
--- incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/pom.xml (original)
+++ incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/pom.xml Thu Aug 23 09:57:39 2012
@@ -79,6 +79,12 @@
</properties>
<dependencies>
+ <dependency><!-- to ensure deactivation in offline mode -->
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.commons.stanboltools.offline</artifactId>
+ <version>0.9.0-incubating</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.enhancer.servicesapi</artifactId>
@@ -107,6 +113,18 @@
<artifactId>slf4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.test</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.core</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
Modified: incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/DBPSpotlightCandidatesEnhancementEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/DBPSpotlightCandidatesEnhancementEngine.java?rev=1376420&r1=1376419&r2=1376420&view=diff
==============================================================================
--- incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/DBPSpotlightCandidatesEnhancementEngine.java (original)
+++ incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/DBPSpotlightCandidatesEnhancementEngine.java Thu Aug 23 09:57:39 2012
@@ -16,24 +16,32 @@
*/
package org.apache.stanbol.enhancer.engines.dbpspotlight.candidates;
+import static org.apache.stanbol.enhancer.engines.dbpspotlight.candidates.XMLParser.getElementsByTagName;
+import static org.apache.stanbol.enhancer.engines.dbpspotlight.candidates.XMLParser.loadXMLFromInputStream;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_LANGUAGE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_RELATION;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_END;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_LABEL;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_REFERENCE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTED_TEXT;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTION_CONTEXT;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_START;
import java.io.BufferedReader;
+import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
+import java.nio.charset.Charset;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Dictionary;
@@ -55,10 +63,14 @@ import org.apache.clerezza.rdf.core.UriR
import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl;
import org.apache.clerezza.rdf.core.impl.TripleImpl;
import org.apache.clerezza.rdf.core.serializedform.Serializer;
+import org.apache.commons.io.IOUtils;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Properties;
import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
import org.apache.felix.scr.annotations.Service;
+import org.apache.stanbol.commons.stanboltools.offline.OfflineMode;
+import org.apache.stanbol.commons.stanboltools.offline.OnlineMode;
import org.apache.stanbol.enhancer.servicesapi.Blob;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
import org.apache.stanbol.enhancer.servicesapi.EngineException;
@@ -76,6 +88,7 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
/**
* {@link DBPSpotlightCandidatesEnhancementEngine} provides functionality to
@@ -113,21 +126,52 @@ public class DBPSpotlightCandidatesEnhan
public static final String SL_CONFIDENCE = "stanbol.DBPSpotlightCandidatesEnhancementEngine.confidence";
/**
+ * Ensures this engine is deactivated in {@link OfflineMode}
+ */
+ @SuppressWarnings("unused")
+ @Reference
+ private OnlineMode onlineMode;
+
+ /**
* The default value for the Execution of this Engine. Currently set to
- * {@link ServiceProperties#ORDERING_PRE_PROCESSING}
+ * <code>{@link ServiceProperties#ORDERING_CONTENT_EXTRACTION} -35</code>
*/
public static final Integer defaultOrder = ORDERING_CONTENT_EXTRACTION - 35;
+ private static final Charset UTF8 = Charset.forName("UTF-8");
/**
* This contains the only MIME type directly supported by this enhancement
* engine.
*/
private static final String TEXT_PLAIN_MIMETYPE = "text/plain";
/**
+ * This contains a list of languages supported by DBpedia Spotlight. If the
+ * metadata doesn't contain a value for the language as the value of the
+ * {@link Property.DC_LANG property} the content can't be processed.
+ */
+ protected static final Set<String> SUPPORTED_LANGUAGES = Collections
+ .unmodifiableSet(new HashSet<String>(Arrays.asList("en")));
+ /**
* This contains the only MIME type directly supported by this enhancement
* engine.
*/
private static final String SPOTLIGHT_NAME_SPACE = "http://spotlight.dbpedia.org/ns/";
+
+ /*
+ * Definition of some Spotlight specific properties added to
+ * fise:EntityAnnotations created by this Engine
+ */
+ public static final UriRef SPOTLIGHT_CONTEXTUAL_SCORE = new UriRef(
+ SPOTLIGHT_NAME_SPACE + "contextualScore");
+ public static final UriRef SPOTLIGHT_PERCENTAGE_OF_SECOND_RANK = new UriRef(
+ SPOTLIGHT_NAME_SPACE + "percentageOfSecondRank");
+ public static final UriRef SPOTLIGHT_SUPPORT = new UriRef(
+ SPOTLIGHT_NAME_SPACE + "support");
+ public static final UriRef SPOTLIGHT_PRIOR_SCORE = new UriRef(
+ SPOTLIGHT_NAME_SPACE + "priorScore");
+ public static final UriRef SPOTLIGHT_FINAL_SCORE = new UriRef(
+ SPOTLIGHT_NAME_SPACE + "finalScore");
+
/** Set containing the only supported mime type {@link #TEXT_PLAIN_MIMETYPE} */
private static final Set<String> SUPPORTED_MIMTYPES = Collections
.singleton(TEXT_PLAIN_MIMETYPE);
@@ -136,7 +180,7 @@ public class DBPSpotlightCandidatesEnhan
private static final Logger log = LoggerFactory
.getLogger(DBPSpotlightCandidatesEnhancementEngine.class);
/** holds the url of the Spotlight REST endpoint */
- private String spotlightUrl;
+ private URL spotlightUrl;
/** holds the chosen of spotter to be used */
private String spotlightSpotter;
/** holds the chosen of disambiguator to be used */
@@ -151,6 +195,20 @@ public class DBPSpotlightCandidatesEnhan
private String spotlightSparql;
/**
+ * Used by OSGI to instantiate the engine. Expects
+ * {@link #activate(ComponentContext)} to be called before usage
+ */
+ public DBPSpotlightCandidatesEnhancementEngine(){}
+
+ /**
+ * Used by unit tests
+ * @param spotlightUrl
+ */
+ protected DBPSpotlightCandidatesEnhancementEngine(URL spotlightUrl){
+ this.spotlightUrl = spotlightUrl;
+ }
+
+ /**
* Initialize all parameters from the configuration panel, or with their
* default values
*
@@ -165,8 +223,21 @@ public class DBPSpotlightCandidatesEnhan
// TODO initialize Extractor
Dictionary<String, Object> properties = ce.getProperties();
- spotlightUrl = properties.get(SL_URL_KEY) == null ? "http://spotlight.dbpedia.org/rest/candidates"
- : (String) properties.get(SL_URL_KEY);
+ //parse the URL of the RESTful service
+ Object value = properties.get(SL_URL_KEY);
+ if(value == null || value.toString().isEmpty()){
+ throw new ConfigurationException(SL_URL_KEY, "The URL with the DBpedia "
+ + "Spotlight Spot RESTful Service MUST NOT be NULL nor empty!");
+ } else {
+ String url = (String) properties.get(SL_URL_KEY);
+ try {
+ this.spotlightUrl = new URL(url);
+ } catch (MalformedURLException e) {
+ throw new ConfigurationException(SL_URL_KEY, "The parsed URL for the "
+ + "DBpedia Spotlight Spot RESTful Service is illegal formatted!",
+ e);
+ }
+ }
spotlightSpotter = properties.get(SL_SPOTTER) == null ? null
: (String) properties.get(SL_SPOTTER);
spotlightDisambiguator = properties.get(SL_DISAMBIGUATOR) == null ? null
@@ -189,7 +260,14 @@ public class DBPSpotlightCandidatesEnhan
*/
public int canEnhance(ContentItem ci) throws EngineException {
if (ContentItemHelper.getBlob(ci, SUPPORTED_MIMTYPES) != null) {
- return ENHANCE_SYNCHRONOUS;
+ String language = EnhancementEngineHelper.getLanguage(ci);
+ if (!SUPPORTED_LANGUAGES.contains(language)) {
+ log.info("DBpedia Spotlight can not process ContentItem {} "
+ + "because language {} is not supported (supported: {})",
+ new Object[] { ci.getUri(), language, SUPPORTED_LANGUAGES });
+ return CANNOT_ENHANCE;
+ }
+ return ENHANCE_ASYNC;
} else {
return CANNOT_ENHANCE;
}
@@ -203,6 +281,18 @@ public class DBPSpotlightCandidatesEnhan
* the {@link ContentItem}
*/
public void computeEnhancements(ContentItem ci) throws EngineException {
+ Language language;
+ String lang = EnhancementEngineHelper.getLanguage(ci);
+ if(!SUPPORTED_LANGUAGES.contains(lang)){
+ throw new IllegalStateException("Langage '"+lang
+ + "' as annotated for ContentItem "
+ + ci.getUri() + " is not supported by this Engine: "
+ + "This is also checked in the canEnhance method! -> This "
+ + "indicated an Bug in the implementation of the "
+ + "EnhancementJobManager!");
+ } else {
+ language = lang == null || lang.isEmpty() ? null : new Language(lang);
+ }
Entry<UriRef, Blob> contentPart = ContentItemHelper.getBlob(ci,
SUPPORTED_MIMTYPES);
if (contentPart == null) {
@@ -215,20 +305,20 @@ public class DBPSpotlightCandidatesEnhan
+ "indicated an Bug in the implementation of the "
+ "EnhancementJobManager!");
}
- String text = "";
+ String text;
try {
text = ContentItemHelper.getText(contentPart.getValue());
} catch (IOException e) {
throw new InvalidContentException(this, ci, e);
}
- Collection<SurfaceForm> dbpslGraph = doPostRequest(text);
+ Collection<SurfaceForm> dbpslGraph = doPostRequest(text,ci.getUri());
if (dbpslGraph != null) {
// Acquire a write lock on the ContentItem when adding the
// enhancements
ci.getLock().writeLock().lock();
try {
- createEnhancements(dbpslGraph, ci);
+ createEnhancements(dbpslGraph, ci,text,language);
if (log.isDebugEnabled()) {
Serializer serializer = Serializer.getInstance();
ByteArrayOutputStream debugStream = new ByteArrayOutputStream();
@@ -257,26 +347,17 @@ public class DBPSpotlightCandidatesEnhan
* @param ci
* the content item
*/
- public void createEnhancements(Collection<SurfaceForm> occs,
- ContentItem ci) {
+ protected void createEnhancements(Collection<SurfaceForm> occs,
+ ContentItem ci, String text, Language language) {
LiteralFactory literalFactory = LiteralFactory.getInstance();
- final Language language; // used for plain literals representing parts
- // fo the content
- String langString = getMetadataLanguage(ci.getMetadata(), null);
-
- if (langString != null && !langString.isEmpty()) {
- language = new Language(langString);
- } else {
- language = null;
- }
// TODO create TextEnhancement (form, start, end, type?)
HashMap<String, UriRef> entityAnnotationMap = new HashMap<String, UriRef>();
+ MGraph model = ci.getMetadata();
for (SurfaceForm occ : occs) {
UriRef textAnnotation = EnhancementEngineHelper
.createTextEnhancement(ci, this);
- MGraph model = ci.getMetadata();
// model.add(new TripleImpl(textAnnotation, DC_TYPE, new UriRef(
// occ.types )));
// for autotagger use the name instead of the matched term (that
@@ -288,11 +369,10 @@ public class DBPSpotlightCandidatesEnhan
model.add(new TripleImpl(textAnnotation, ENHANCER_END,
literalFactory.createTypedLiteral(occ.offset
+ occ.name.length())));
- // TODO ################## model.add(new TripleImpl(textAnnotation,
- // ENHANCER_SELECTION_CONTEXT, new
- // PlainLiteralImpl(occ.context,language)));
- // create EntityAnnotation only once but add a reference to the
- // textAnnotation
+ model.add(new TripleImpl(textAnnotation, ENHANCER_SELECTION_CONTEXT,
+ new PlainLiteralImpl(
+ getSelectionContext(text, occ.name, occ.offset),
+ language)));
Iterator<CandidateResource> resources = occ.resources.iterator();
while (resources.hasNext()) {
@@ -308,24 +388,15 @@ public class DBPSpotlightCandidatesEnhan
ENHANCER_ENTITY_LABEL, label));
model.add(new TripleImpl(entityAnnotation,
ENHANCER_ENTITY_REFERENCE, new UriRef(resource.uri)));
- model.add(new TripleImpl(entityAnnotation, new UriRef(
- SPOTLIGHT_NAME_SPACE + "contextualScore"),
- literalFactory
- .createTypedLiteral(resource.contextualScore)));
- model.add(new TripleImpl(
- entityAnnotation,
- new UriRef(SPOTLIGHT_NAME_SPACE
- + "percentageOfSecondRank"),
- literalFactory
- .createTypedLiteral(resource.percentageOfSecondRank)));
- model.add(new TripleImpl(entityAnnotation, new UriRef(
- SPOTLIGHT_NAME_SPACE + "support"), literalFactory
+ model.add(new TripleImpl(entityAnnotation, SPOTLIGHT_CONTEXTUAL_SCORE,
+ literalFactory.createTypedLiteral(resource.contextualScore)));
+ model.add(new TripleImpl(entityAnnotation,SPOTLIGHT_PERCENTAGE_OF_SECOND_RANK,
+ literalFactory.createTypedLiteral(resource.percentageOfSecondRank)));
+ model.add(new TripleImpl(entityAnnotation, SPOTLIGHT_SUPPORT, literalFactory
.createTypedLiteral(resource.support)));
- model.add(new TripleImpl(entityAnnotation, new UriRef(
- SPOTLIGHT_NAME_SPACE + "priorScore"), literalFactory
+ model.add(new TripleImpl(entityAnnotation, SPOTLIGHT_PRIOR_SCORE, literalFactory
.createTypedLiteral(resource.priorScore)));
- model.add(new TripleImpl(entityAnnotation, new UriRef(
- SPOTLIGHT_NAME_SPACE + "finalScore"), literalFactory
+ model.add(new TripleImpl(entityAnnotation, SPOTLIGHT_FINAL_SCORE, literalFactory
.createTypedLiteral(resource.finalScore)));
}
if (entityAnnotationMap.containsKey(occ.name)) {
@@ -342,52 +413,18 @@ public class DBPSpotlightCandidatesEnhan
*
* @param text
* a <code>String</code> with the text to be analyzed
+ * @param contentItemUri
+ * Just used for logging
* @return a <code>String</code> with the server response
* @throws EngineException
* if the request cannot be sent
*/
- public Collection<SurfaceForm> doPostRequest(String text)
+ protected Collection<SurfaceForm> doPostRequest(String text,UriRef contentItemUri)
throws EngineException {
- StringBuilder data = new StringBuilder();
- try {
- if (spotlightSpotter != null && !spotlightSpotter.isEmpty())
- data.append(URLEncoder.encode("spotter", "UTF-8") + "="
- + URLEncoder.encode(spotlightSpotter, "UTF-8") + "&");
- if (spotlightDisambiguator != null
- && !spotlightDisambiguator.isEmpty())
- data.append(URLEncoder.encode("disambiguator", "UTF-8") + "="
- + URLEncoder.encode(spotlightDisambiguator, "UTF-8")
- + "&");
- if (spotlightTypesRestriction != null
- && !spotlightTypesRestriction.isEmpty())
- data.append(URLEncoder.encode("types", "UTF-8") + "="
- + URLEncoder.encode(spotlightTypesRestriction, "UTF-8")
- + "&");
- if (spotlightSupport != null && !spotlightSupport.isEmpty())
- data.append(URLEncoder.encode("support", "UTF-8") + "="
- + URLEncoder.encode(spotlightSupport, "UTF-8") + "&");
- if (spotlightConfidence != null && !spotlightConfidence.isEmpty())
- data.append(URLEncoder.encode("confidence", "UTF-8") + "="
- + URLEncoder.encode(spotlightConfidence, "UTF-8") + "&");
- if (spotlightSparql != null && !spotlightSparql.isEmpty()
- && spotlightTypesRestriction == null)
- data.append(URLEncoder.encode("sparql", "UTF-8") + "="
- + URLEncoder.encode(spotlightSparql, "UTF-8") + "&");
- data.append(URLEncoder.encode("text", "UTF-8") + "="
- + URLEncoder.encode(text, "UTF-8"));
- } catch (UnsupportedEncodingException e) {
- throw new EngineException(
- "Data for the httprequest could not be converted. Error: "
- + e.getMessage());
- }
-
HttpURLConnection connection = null;
- StringBuffer response = new StringBuffer();
-
+ BufferedWriter wr = null;
try {
- // Create connection
- URL url = new URL(spotlightUrl);
- connection = (HttpURLConnection) url.openConnection();
+ connection = (HttpURLConnection) spotlightUrl.openConnection();
connection.setRequestMethod("POST");
connection.setRequestProperty("Content-Type",
"application/x-www-form-urlencoded");
@@ -398,51 +435,79 @@ public class DBPSpotlightCandidatesEnhan
connection.setDoOutput(true);
// Send request
- DataOutputStream wr = new DataOutputStream(
- connection.getOutputStream());
- wr.writeBytes(data.toString());
- wr.flush();
- wr.close();
-
- // Get Response
- InputStream is = connection.getInputStream();
- BufferedReader rd = new BufferedReader(new InputStreamReader(is));
- String line;
- while ((line = rd.readLine()) != null) {
- response.append(line);
- response.append('\r');
- }
- rd.close();
-
- } catch (Exception e) {
-
- log.error("[request] Request could not be made. Error: "
- + e.getMessage());
- e.printStackTrace();
- return null;
-
- } finally {
-
- if (connection != null) {
- connection.disconnect();
+ wr = new BufferedWriter(new OutputStreamWriter(
+ connection.getOutputStream(),UTF8));
+ } catch (IOException e) {
+ IOUtils.closeQuietly(wr);
+ throw new EngineException("Unable to open connection to "+
+ spotlightUrl,e);
+ }
+ try {
+ if (spotlightSpotter != null && !spotlightSpotter.isEmpty()) {
+ wr.write("spotter=");
+ wr.write(URLEncoder.encode(spotlightSpotter, "UTF-8"));
+ wr.write('&');
+ }
+ if (spotlightDisambiguator != null
+ && !spotlightDisambiguator.isEmpty()) {
+ wr.write("disambiguator=");
+ wr.write(URLEncoder.encode(spotlightDisambiguator, "UTF-8"));
+ wr.write('&');
+ }
+ if (spotlightTypesRestriction != null
+ && !spotlightTypesRestriction.isEmpty()) {
+ wr.write("types=");
+ wr.write(URLEncoder.encode(spotlightTypesRestriction, "UTF-8"));
+ wr.write('&');
+ }
+ if (spotlightSupport != null && !spotlightSupport.isEmpty()){
+ wr.write("support=");
+ wr.write(URLEncoder.encode(spotlightSupport, "UTF-8"));
+ wr.write('&');
+ }
+ if (spotlightConfidence != null && !spotlightConfidence.isEmpty()) {
+ wr.write("confidence=");
+ wr.write(URLEncoder.encode(spotlightConfidence, "UTF-8"));
+ wr.write('&');
+ }
+ if (spotlightSparql != null && !spotlightSparql.isEmpty()
+ && spotlightTypesRestriction == null) {
+ wr.write("sparql=");
+ wr.write(URLEncoder.encode(spotlightSparql, "UTF-8"));
+ wr.write('&');
}
+ wr.write("text=");
+ wr.write(URLEncoder.encode(text, "UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ throw new IllegalStateException(
+ "The platform does not support encoding " + UTF8.name(),e);
+ } catch (IOException e) {
+ throw new EngineException("Unable to write 'plain/text' content "
+ + "for ContentItem "+contentItemUri+" to "
+ + spotlightUrl,e);
+ } finally {
+ IOUtils.closeQuietly(wr);
}
-
- XMLParser xmlParser = new XMLParser();
+ InputStream is = null;
+ Document xmlDoc;
try {
- Document xmlDoc = xmlParser.loadXMLFromString(response.toString());
- NodeList nlist = xmlParser.getElementsByTagName(xmlDoc,
- "surfaceForm");
- Collection<SurfaceForm> annos = this.getAnnotations(nlist);
-
- return annos;
- } catch (Exception e) {
- log.error("[response] Response XML could not be parsed. Error: "
- + e.getMessage());
- throw new EngineException(
- "Response XML could not be parsed. Error: "
- + e.getMessage());
+ // Get Response
+ is = connection.getInputStream();
+ xmlDoc = loadXMLFromInputStream(is);
+ } catch (IOException e) {
+ throw new EngineException("Unable to spot Entities with"
+ + "Dbpedia Spotlight Spot RESTful Serice running at "
+ + spotlightUrl,e);
+ } catch(SAXException e) {
+ throw new EngineException("Unable to parse Response from "
+ + "Dbpedia Spotlight Spot RESTful Serice running at "
+ + spotlightUrl,e);
+ } finally {
+ IOUtils.closeQuietly(is);
}
+ NodeList nlist = getElementsByTagName(xmlDoc,"surfaceForm");
+ Collection<SurfaceForm> annos = this.getAnnotations(nlist);
+ return annos;
}
/**
@@ -503,32 +568,38 @@ public class DBPSpotlightCandidatesEnhan
ENHANCEMENT_ENGINE_ORDERING, (Object) defaultOrder));
}
- public String getMetadataLanguage(MGraph model, NonLiteral subj) {
- Iterator<Triple> it = model.filter(subj, DC_LANGUAGE, null);
- if (it.hasNext()) {
- Resource langNode = it.next().getObject();
- return getLexicalForm(langNode);
- }
- return null;
- }
+ private static final int DEFAULT_SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE = 50;
+ /**
+ * Extracts the selection context based on the content, selection and
+ * the start char offset of the selection
+ * @param content the content
+ * @param selection the selected text
+ * @param selectionStartPos the start char position of the selection
+ * @return the context
+ */
+ protected static String getSelectionContext(String content, String selection,int selectionStartPos){
+ //extract the selection context
+ int beginPos;
+ if(selectionStartPos <= DEFAULT_SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE){
+ beginPos = 0;
+ } else {
+ int start = selectionStartPos-DEFAULT_SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE;
+ beginPos = content.indexOf(' ',start);
+ if(beginPos < 0 || beginPos >= selectionStartPos){ //no words
+ beginPos = start; //begin within a word
+ }
+ }
+ int endPos;
+ if(selectionStartPos+selection.length()+DEFAULT_SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE >= content.length()){
+ endPos = content.length();
+ } else {
+ int start = selectionStartPos+selection.length()+DEFAULT_SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE;
+ endPos = content.lastIndexOf(' ', start);
+ if(endPos <= selectionStartPos+selection.length()){
+ endPos = start; //end within a word;
+ }
+ }
+ return content.substring(beginPos, endPos);
+ }
- public String getLexicalForm(Resource res) {
- if (res == null) {
- return null;
- } else if (res instanceof Literal) {
- return ((Literal) res).getLexicalForm();
- } else {
- return res.toString();
- }
- }
-
- /**
- * This method is used by the test class to set the endpoint url
- *
- * @param url
- * String the url of the Spotlight endpoint
- */
- public void setEndpointUrl(String url) {
- spotlightUrl = url;
- }
}
Modified: incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/XMLParser.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/XMLParser.java?rev=1376420&r1=1376419&r2=1376420&view=diff
==============================================================================
--- incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/XMLParser.java (original)
+++ incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/XMLParser.java Thu Aug 23 09:57:39 2012
@@ -34,15 +34,19 @@ import org.xml.sax.SAXException;
*
* @author <a href="mailto:iavor.jelev@babelmonkeys.com">Iavor Jelev</a>
*/
+public final class XMLParser {
-public class XMLParser {
-
- public NodeList getElementsByTagName(Document doc, String tagName) {
+ /**
+ * Do not create instances of Utility Classes
+ */
+ private XMLParser(){};
+
+ public static NodeList getElementsByTagName(Document doc, String tagName) {
return doc.getElementsByTagName(tagName);
}
- public Document loadXMLFromString(String xml) throws SAXException,
+ public static Document loadXMLFromString(String xml) throws SAXException,
IOException {
Document doc = loadXMLFromInputStream(new ByteArrayInputStream(
xml.getBytes()));
@@ -51,7 +55,7 @@ public class XMLParser {
return doc;
}
- public Document loadXMLFromInputStream(InputStream is) throws SAXException,
+ public static Document loadXMLFromInputStream(InputStream is) throws SAXException,
IOException {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setNamespaceAware(true);
@@ -67,7 +71,7 @@ public class XMLParser {
return doc;
}
- public Document loadXMLFromFile(String filePath)
+ public static Document loadXMLFromFile(String filePath)
throws ParserConfigurationException, SAXException, IOException {
File fXmlFile = new File(filePath);
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
Copied: incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/DBPSpotlightCandidatesEnhancementTest.java (from r1375110, incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/core/DBPSpotlightCandidatesEnhancementTest.java)
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/DBPSpotlightCandidatesEnhancementTest.java?p2=incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/DBPSpotlightCandidatesEnhancementTest.java&p1=incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/core/DBPSpotlightCandidatesEnhancementTest.java&r1=1375110&r2=1376420&rev=1376420&view=diff
==============================================================================
--- incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/core/DBPSpotlightCandidatesEnhancementTest.java (original)
+++ incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-candidates/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/candidates/DBPSpotlightCandidatesEnhancementTest.java Thu Aug 23 09:57:39 2012
@@ -14,14 +14,38 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.stanbol.enhancer.engines.dbpspotlight.candidates.core;
+package org.apache.stanbol.enhancer.engines.dbpspotlight.candidates;
-import java.util.Collection;
+import static org.apache.stanbol.enhancer.servicesapi.EnhancementEngine.ENHANCE_ASYNC;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map.Entry;
+
+import org.apache.clerezza.rdf.core.LiteralFactory;
+import org.apache.clerezza.rdf.core.Resource;
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl;
+import org.apache.clerezza.rdf.core.impl.TripleImpl;
+import org.apache.stanbol.enhancer.contentitem.inmemory.InMemoryContentItemFactory;
import org.apache.stanbol.enhancer.engines.dbpspotlight.candidates.DBPSpotlightCandidatesEnhancementEngine;
import org.apache.stanbol.enhancer.engines.dbpspotlight.candidates.SurfaceForm;
+import org.apache.stanbol.enhancer.servicesapi.Blob;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.enhancer.servicesapi.ContentItemFactory;
import org.apache.stanbol.enhancer.servicesapi.EngineException;
+import org.apache.stanbol.enhancer.servicesapi.helper.ContentItemHelper;
+import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
+import org.apache.stanbol.enhancer.servicesapi.impl.StringSource;
+import org.apache.stanbol.enhancer.servicesapi.rdf.Properties;
+import org.apache.stanbol.enhancer.test.helper.EnhancementStructureHelper;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.osgi.service.cm.ConfigurationException;
@@ -48,24 +72,57 @@ public class DBPSpotlightCandidatesEnhan
private static String TEST_TEXT = "President Obama is meeting Angela Merkel in Berlin on Monday.";
private static DBPSpotlightCandidatesEnhancementEngine dbpslight;
+ private static ContentItemFactory ciFactory = InMemoryContentItemFactory.getInstance();
+
+ private ContentItem ci;
+ private static Entry<UriRef, Blob> textContentPart;
+
@BeforeClass
- public static void oneTimeSetup() throws ConfigurationException {
- dbpslight = new DBPSpotlightCandidatesEnhancementEngine();
- dbpslight.setEndpointUrl(SPL_URL);
+ public static void oneTimeSetup() throws Exception {
+ dbpslight = new DBPSpotlightCandidatesEnhancementEngine(new URL(SPL_URL));
+ }
+
+ @Before
+ public void initTest() throws IOException {
+ //create the contentItem for testing
+ ci = ciFactory.createContentItem(new StringSource(TEST_TEXT));
+ assertNotNull(ci);
+ textContentPart = ContentItemHelper.getBlob(ci, Collections.singleton("text/plain"));
+ assertNotNull(textContentPart);
+ //add the language of the text
+ ci.getMetadata().add(new TripleImpl(ci.getUri(), Properties.DC_LANGUAGE,
+ new PlainLiteralImpl("en")));
+ assertEquals("en", EnhancementEngineHelper.getLanguage(ci));
}
-
@Test
- public void testEntityExtraction() {
- Collection<SurfaceForm> entities;
- try {
- entities = dbpslight.doPostRequest(TEST_TEXT);
+ public void testEntityExtraction() throws Exception{
+ Collection<SurfaceForm> entities = dbpslight.doPostRequest(TEST_TEXT,ci.getUri());
LOG.info("Found entities: {}", entities.size());
LOG.debug("Entities:\n{}", entities);
Assert.assertFalse("No entities were found!", entities.isEmpty());
- } catch (EngineException e) {
- Assert.assertFalse("An EngineException occurred! The message was: "
- + e.getMessage(), true);
- }
+ }
+ @Test
+ public void testCanEnhance() throws EngineException {
+ assertEquals(ENHANCE_ASYNC, dbpslight.canEnhance(ci));
+ }
+
+ /**
+ * Validates the Enhancements created by this engine
+ * @throws EngineException
+ */
+ @Test
+ public void testEnhancement() throws EngineException {
+ dbpslight.computeEnhancements(ci);
+ HashMap<UriRef,Resource> expectedValues = new HashMap<UriRef,Resource>();
+ expectedValues.put(Properties.ENHANCER_EXTRACTED_FROM, ci.getUri());
+ expectedValues.put(Properties.DC_CREATOR, LiteralFactory.getInstance().createTypedLiteral(
+ dbpslight.getClass().getName()));
+ //validate fise:TextAnnotations
+ EnhancementStructureHelper.validateAllTextAnnotations(
+ ci.getMetadata(), TEST_TEXT, expectedValues);
+ //validate fise:EntityAnnotations
+ EnhancementStructureHelper.validateAllEntityAnnotations(
+ ci.getMetadata(), expectedValues);
}
}
Modified: incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-spot/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/spot/DBPSpotlightSpotEnhancementEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-spot/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/spot/DBPSpotlightSpotEnhancementEngine.java?rev=1376420&r1=1376419&r2=1376420&view=diff
==============================================================================
--- incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-spot/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/spot/DBPSpotlightSpotEnhancementEngine.java (original)
+++ incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-spot/src/main/java/org/apache/stanbol/enhancer/engines/dbpspotlight/spot/DBPSpotlightSpotEnhancementEngine.java Thu Aug 23 09:57:39 2012
@@ -101,8 +101,9 @@ public class DBPSpotlightSpotEnhancement
/**
* Ensures this engine is deactivated in {@link OfflineMode}
*/
+ @SuppressWarnings("unused")
@Reference
- OnlineMode onlineMode;
+ private OnlineMode onlineMode;
/**
* a configurable value of the text segment length to check
@@ -451,15 +452,16 @@ public class DBPSpotlightSpotEnhancement
// return null;
// }
- public String getLexicalForm(Resource res) {
- if (res == null) {
- return null;
- } else if (res instanceof Literal) {
- return ((Literal) res).getLexicalForm();
- } else {
- return res.toString();
- }
- }
+// rwesten: unused
+// public String getLexicalForm(Resource res) {
+// if (res == null) {
+// return null;
+// } else if (res instanceof Literal) {
+// return ((Literal) res).getLexicalForm();
+// } else {
+// return res.toString();
+// }
+// }
private static final int DEFAULT_SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE = 50;
/**
@@ -470,7 +472,7 @@ public class DBPSpotlightSpotEnhancement
* @param selectionStartPos the start char position of the selection
* @return the context
*/
- public static String getSelectionContext(String content, String selection,int selectionStartPos){
+ protected static String getSelectionContext(String content, String selection,int selectionStartPos){
//extract the selection context
int beginPos;
if(selectionStartPos <= DEFAULT_SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE){
Modified: incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-spot/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/spot/DBPSpotlightSpotEnhancementTest.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-spot/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/spot/DBPSpotlightSpotEnhancementTest.java?rev=1376420&r1=1376419&r2=1376420&view=diff
==============================================================================
--- incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-spot/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/spot/DBPSpotlightSpotEnhancementTest.java (original)
+++ incubator/stanbol/branches/dbpedia-spotlight-engines/engines/dbpedia-spotlight-spot/src/test/java/org/apache/stanbol/enhancer/engines/dbpspotlight/spot/DBPSpotlightSpotEnhancementTest.java Thu Aug 23 09:57:39 2012
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import java.io.IOException;
import java.net.URL;
import java.util.Collection;
import java.util.Collections;
@@ -47,6 +48,7 @@ import org.apache.stanbol.enhancer.servi
import org.apache.stanbol.enhancer.servicesapi.rdf.Properties;
import org.apache.stanbol.enhancer.test.helper.EnhancementStructureHelper;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.osgi.service.cm.ConfigurationException;
@@ -75,11 +77,17 @@ public class DBPSpotlightSpotEnhancement
private static ContentItemFactory ciFactory = InMemoryContentItemFactory.getInstance();
- private static ContentItem ci;
+ private ContentItem ci;
private static Entry<UriRef, Blob> textContentPart;
@BeforeClass
public static void oneTimeSetup() throws Exception {
+ //and the enhancement engine instance
+ dbpslight = new DBPSpotlightSpotEnhancementEngine(new URL(SPL_URL),null);
+ }
+
+ @Before
+ public void initTest() throws IOException {
//create the contentItem for testing
ci = ciFactory.createContentItem(new StringSource(TEST_TEXT));
assertNotNull(ci);
@@ -89,10 +97,8 @@ public class DBPSpotlightSpotEnhancement
ci.getMetadata().add(new TripleImpl(ci.getUri(), Properties.DC_LANGUAGE,
new PlainLiteralImpl("en")));
assertEquals("en", EnhancementEngineHelper.getLanguage(ci));
- //and the enhancement engine instance
- dbpslight = new DBPSpotlightSpotEnhancementEngine(new URL(SPL_URL),null);
}
-
+
@Test
public void testEntityExtraction() throws Exception {
Collection<SurfaceForm> entities;