You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by sf...@apache.org on 2010/12/10 16:00:14 UTC
svn commit: r1044386 [1/3] - in /incubator/stanbol/trunk/fise:
engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/
engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/impl/
engines/autotagging/src/test/java/eu...
Author: sfermigier
Date: Fri Dec 10 15:00:13 2010
New Revision: 1044386
URL: http://svn.apache.org/viewvc?rev=1044386&view=rev
Log:
Cleanup and format a few files.
Modified:
incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/AutotaggerProvider.java
incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/impl/EnhancementRDFUtils.java
incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/impl/EntityMentionEnhancementEngine.java
incubator/stanbol/trunk/fise/engines/autotagging/src/test/java/eu/iksproject/fise/engines/autotagging/impl/TestEntityMentionEnhancementEngine.java
incubator/stanbol/trunk/fise/engines/geonames/src/main/java/eu/iksproject/fise/engines/geonames/impl/LocationEnhancementEngine.java
incubator/stanbol/trunk/fise/engines/geonames/src/test/java/eu/iksproject/fise/engines/geonames/impl/TestLocationEnhancementEngine.java
incubator/stanbol/trunk/fise/engines/langid/pom.xml
incubator/stanbol/trunk/fise/engines/langid/src/main/java/eu/iksproject/fise/engines/langid/LangIdEnhancementEngine.java
incubator/stanbol/trunk/fise/engines/langid/src/test/java/eu/iksproject/fise/engines/langid/core/TextCatTest.java
incubator/stanbol/trunk/fise/engines/metaxa/src/main/java/eu/iksproject/fise/engines/metaxa/MetaxaEngine.java
incubator/stanbol/trunk/fise/engines/metaxa/src/main/java/eu/iksproject/fise/engines/metaxa/core/MetaxaCore.java
incubator/stanbol/trunk/fise/engines/metaxa/src/main/java/eu/iksproject/fise/engines/metaxa/core/RDF2GoUtils.java
incubator/stanbol/trunk/fise/engines/metaxa/src/main/java/eu/iksproject/fise/engines/metaxa/core/html/BundleURIResolver.java
incubator/stanbol/trunk/fise/engines/metaxa/src/main/java/eu/iksproject/fise/engines/metaxa/core/html/DOMUtils.java
incubator/stanbol/trunk/fise/engines/metaxa/src/main/java/eu/iksproject/fise/engines/metaxa/core/html/HtmlExtractionComponent.java
incubator/stanbol/trunk/fise/engines/metaxa/src/main/java/eu/iksproject/fise/engines/metaxa/core/html/HtmlExtractionRegistry.java
incubator/stanbol/trunk/fise/engines/metaxa/src/main/java/eu/iksproject/fise/engines/metaxa/core/html/HtmlExtractorFactory.java
incubator/stanbol/trunk/fise/engines/metaxa/src/main/java/eu/iksproject/fise/engines/metaxa/core/html/HtmlParser.java
incubator/stanbol/trunk/fise/engines/metaxa/src/main/java/eu/iksproject/fise/engines/metaxa/core/html/IksHtmlExtractor.java
incubator/stanbol/trunk/fise/engines/metaxa/src/main/java/eu/iksproject/fise/engines/metaxa/core/html/InitializationException.java
incubator/stanbol/trunk/fise/engines/metaxa/src/main/java/eu/iksproject/fise/engines/metaxa/core/html/XsltExtractor.java
incubator/stanbol/trunk/fise/engines/opennlp-ner/src/main/java/eu/iksproject/fise/engines/opennlp/impl/NamedEntityExtractionEnhancementEngine.java
incubator/stanbol/trunk/fise/engines/zemanta/src/main/java/eu/iksproject/engines/zemanta/ZemantaOntologyEnum.java
incubator/stanbol/trunk/fise/engines/zemanta/src/main/java/eu/iksproject/fise/engines/zemanta/impl/ZemantaEnhancementEngine.java
incubator/stanbol/trunk/fise/engines/zemanta/src/main/java/eu/iksproject/fise/engines/zemanta/impl/ZemantaPropertyEnum.java
incubator/stanbol/trunk/fise/generic/servicesapi/src/main/java/eu/iksproject/fise/servicesapi/EnhancementEngine.java
incubator/stanbol/trunk/fise/generic/servicesapi/src/main/java/eu/iksproject/fise/servicesapi/ServiceProperties.java
incubator/stanbol/trunk/fise/generic/servicesapi/src/main/java/eu/iksproject/fise/servicesapi/TextAnnotation.java
incubator/stanbol/trunk/fise/generic/standalone/src/main/java/eu/iksproject/fise/standalone/impl/MainServlet.java
incubator/stanbol/trunk/fise/iks-autotagging/src/main/java/eu/iksproject/autotagging/Autotagger.java
Modified: incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/AutotaggerProvider.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/AutotaggerProvider.java?rev=1044386&r1=1044385&r2=1044386&view=diff
==============================================================================
--- incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/AutotaggerProvider.java (original)
+++ incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/AutotaggerProvider.java Fri Dec 10 15:00:13 2010
@@ -12,8 +12,8 @@ public interface AutotaggerProvider {
/**
* @return the autotagger instance or null if none is configured
*/
- public Autotagger getAutotagger();
+ Autotagger getAutotagger();
- public BundleContext getBundleContext();
+ BundleContext getBundleContext();
}
Modified: incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/impl/EnhancementRDFUtils.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/impl/EnhancementRDFUtils.java?rev=1044386&r1=1044385&r2=1044386&view=diff
==============================================================================
--- incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/impl/EnhancementRDFUtils.java (original)
+++ incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/impl/EnhancementRDFUtils.java Fri Dec 10 15:00:13 2010
@@ -14,7 +14,11 @@ import eu.iksproject.fise.servicesapi.he
import eu.iksproject.fise.servicesapi.rdf.Properties;
public class EnhancementRDFUtils {
- /**
+
+ private EnhancementRDFUtils() {
+ }
+
+ /**
* @param literalFactory the LiteralFactory to use
* @param graph the MGraph to use
* @param contentItemId the contentItemId the enhancement is extracted from
Modified: incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/impl/EntityMentionEnhancementEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/impl/EntityMentionEnhancementEngine.java?rev=1044386&r1=1044385&r2=1044386&view=diff
==============================================================================
--- incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/impl/EntityMentionEnhancementEngine.java (original)
+++ incubator/stanbol/trunk/fise/engines/autotagging/src/main/java/eu/iksproject/fise/engines/autotagging/impl/EntityMentionEnhancementEngine.java Fri Dec 10 15:00:13 2010
@@ -52,7 +52,7 @@ public class EntityMentionEnhancementEng
* The default value for the Execution of this Engine. Currently set to
* {@link EnhancementJobManager#DEFAULT_ORDER}
*/
- public static final Integer defaultOrder = ServiceProperties.ORDERING_EXTRACTION_ENHANCEMENT;
+ public static final Integer defaultOrder = ORDERING_EXTRACTION_ENHANCEMENT;
public void computeEnhancements(ContentItem ci) throws EngineException {
Autotagger autotagger = autotaggerProvider.getAutotagger();
@@ -173,7 +173,7 @@ public class EntityMentionEnhancementEng
@Override
public Map<String, Object> getServiceProperties() {
return Collections.unmodifiableMap(Collections.singletonMap(
- ServiceProperties.ENHANCEMENT_ENGINE_ORDERING,
+ ENHANCEMENT_ENGINE_ORDERING,
(Object) defaultOrder));
}
}
Modified: incubator/stanbol/trunk/fise/engines/autotagging/src/test/java/eu/iksproject/fise/engines/autotagging/impl/TestEntityMentionEnhancementEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/fise/engines/autotagging/src/test/java/eu/iksproject/fise/engines/autotagging/impl/TestEntityMentionEnhancementEngine.java?rev=1044386&r1=1044385&r2=1044386&view=diff
==============================================================================
--- incubator/stanbol/trunk/fise/engines/autotagging/src/test/java/eu/iksproject/fise/engines/autotagging/impl/TestEntityMentionEnhancementEngine.java (original)
+++ incubator/stanbol/trunk/fise/engines/autotagging/src/test/java/eu/iksproject/fise/engines/autotagging/impl/TestEntityMentionEnhancementEngine.java Fri Dec 10 15:00:13 2010
@@ -19,6 +19,12 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import static eu.iksproject.fise.servicesapi.rdf.OntologicalClasses.DBPEDIA_ORGANISATION;
+import static eu.iksproject.fise.servicesapi.rdf.OntologicalClasses.DBPEDIA_PERSON;
+import static eu.iksproject.fise.servicesapi.rdf.OntologicalClasses.DBPEDIA_PLACE;
+import static eu.iksproject.fise.servicesapi.rdf.Properties.*;
+import static eu.iksproject.fise.servicesapi.rdf.TechnicalClasses.FISE_ENTITYANNOTATION;
+import static eu.iksproject.fise.servicesapi.rdf.TechnicalClasses.FISE_TEXTANNOTATION;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
@@ -32,27 +38,27 @@ import eu.iksproject.fise.servicesapi.rd
public class TestEntityMentionEnhancementEngine {
- /**
- * The context for the tests (same as in TestOpenNLPEnhancementEngine)
- */
- public static final String CONTEXT = "Dr. Patrick Marshall (1869 - November 1950) was a"
- + " geologist who lived in New Zealand and worked at the University of Otago.";
- /**
- * The person for the tests (same as in TestOpenNLPEnhancementEngine)
- */
- public static final String PERSON = "Patrick Marshall";
- /**
- * The organisation for the tests (same as in TestOpenNLPEnhancementEngine)
- */
- public static final String ORGANISATION ="University of Otago";
- /**
- * The place for the tests (same as in TestOpenNLPEnhancementEngine)
- */
- public static final String PLACE = "New Zealand";
-
- static ConfiguredAutotaggerProvider autotaggerProvider = new ConfiguredAutotaggerProvider();
-
- static EntityMentionEnhancementEngine entityMentionEngine = new EntityMentionEnhancementEngine();
+ /**
+ * The context for the tests (same as in TestOpenNLPEnhancementEngine)
+ */
+ public static final String CONTEXT = "Dr. Patrick Marshall (1869 - November 1950) was a"
+ + " geologist who lived in New Zealand and worked at the University of Otago.";
+ /**
+ * The person for the tests (same as in TestOpenNLPEnhancementEngine)
+ */
+ public static final String PERSON = "Patrick Marshall";
+ /**
+ * The organisation for the tests (same as in TestOpenNLPEnhancementEngine)
+ */
+ public static final String ORGANISATION = "University of Otago";
+ /**
+ * The place for the tests (same as in TestOpenNLPEnhancementEngine)
+ */
+ public static final String PLACE = "New Zealand";
+
+ static ConfiguredAutotaggerProvider autotaggerProvider = new ConfiguredAutotaggerProvider();
+
+ static EntityMentionEnhancementEngine entityMentionEngine = new EntityMentionEnhancementEngine();
@BeforeClass
public static void setUpServices() throws IOException {
@@ -61,22 +67,23 @@ public class TestEntityMentionEnhancemen
MockComponentContext context = new MockComponentContext(properties);
autotaggerProvider.activate(context);
}
- @Before
- public void bindServices() throws IOException {
- entityMentionEngine.bindAutotaggerProvider(autotaggerProvider);
- }
-
- @After
- public void unbindServices() {
- entityMentionEngine.unbindAutotaggerProvider(autotaggerProvider);
- }
+
+ @Before
+ public void bindServices() throws IOException {
+ entityMentionEngine.bindAutotaggerProvider(autotaggerProvider);
+ }
+
+ @After
+ public void unbindServices() {
+ entityMentionEngine.unbindAutotaggerProvider(autotaggerProvider);
+ }
@AfterClass
public static void shutdownServices() {
autotaggerProvider.deactivate(null);
}
- public static ContentItem getContentItem(final String id,
- final String text) {
+
+ public static ContentItem getContentItem(final String id, final String text) {
return new ContentItem() {
SimpleMGraph metadata = new SimpleMGraph();
@@ -98,64 +105,65 @@ public class TestEntityMentionEnhancemen
}
};
}
-
- public static void getTextAnnotation(ContentItem ci, String name,String context,UriRef type){
- String content;
- try {
- content = IOUtils.toString(ci.getStream());
- } catch (IOException e) {
- //should never happen anyway!
- content = "";
- }
- RdfEntityFactory factory = RdfEntityFactory.createInstance(ci.getMetadata());
- TextAnnotation testAnnotation = factory.getProxy(new UriRef("urn:iks-project:fise:test:text-annotation:person"), TextAnnotation.class);
- testAnnotation.setCreator(new UriRef("urn:iks-project:fise:test:dummyEngine"));
- testAnnotation.setCreated(new Date());
- testAnnotation.setSelectedText(name);
- testAnnotation.setSelectionContext(context);
- testAnnotation.getDcType().add(type);
- Integer start = content.indexOf(name);
- if(start < 0){ //if not found in the content
- //set some random numbers for start/end
- start = (int)Math.random()*100;
- }
- testAnnotation.setStart(start);
- testAnnotation.setEnd(start+name.length());
+
+ public static void getTextAnnotation(ContentItem ci, String name, String context, UriRef type) {
+ String content;
+ try {
+ content = IOUtils.toString(ci.getStream());
+ } catch (IOException e) {
+ //should never happen anyway!
+ content = "";
+ }
+ RdfEntityFactory factory = RdfEntityFactory.createInstance(ci.getMetadata());
+ TextAnnotation testAnnotation = factory.getProxy(
+ new UriRef("urn:iks-project:fise:test:text-annotation:person"), TextAnnotation.class);
+ testAnnotation.setCreator(new UriRef("urn:iks-project:fise:test:dummyEngine"));
+ testAnnotation.setCreated(new Date());
+ testAnnotation.setSelectedText(name);
+ testAnnotation.setSelectionContext(context);
+ testAnnotation.getDcType().add(type);
+ Integer start = content.indexOf(name);
+ if (start < 0) { //if not found in the content
+ //set some random numbers for start/end
+ start = (int) Math.random() * 100;
+ }
+ testAnnotation.setStart(start);
+ testAnnotation.setEnd(start + name.length());
}
+
@Test
- public void testEntityMentionEnhancementEngine() throws Exception{
- //create a content item
- ContentItem ci = getContentItem("urn:iks-project:fise:text:content-item:person", CONTEXT);
- //add three text annotations to be consumed by this test
- getTextAnnotation(ci, PERSON, CONTEXT, OntologicalClasses.DBPEDIA_PERSON);
- getTextAnnotation(ci, ORGANISATION, CONTEXT, OntologicalClasses.DBPEDIA_ORGANISATION);
- getTextAnnotation(ci, PLACE, CONTEXT, OntologicalClasses.DBPEDIA_PLACE);
- //perform the computation of the enhancements
- entityMentionEngine.computeEnhancements(ci);
- // ... and test the results
- /*
- * TODO: rw 20100617
- * - Expected results depend on the used Index.
- * - Use an example where the Organisation, Person and Place is part
- * of the index
- */
- int entityAnnotationCount = checkAllEntityAnnotations(ci.getMetadata());
- assertEquals(2, entityAnnotationCount);
+ public void testEntityMentionEnhancementEngine() throws Exception {
+ //create a content item
+ ContentItem ci = getContentItem("urn:iks-project:fise:text:content-item:person", CONTEXT);
+ //add three text annotations to be consumed by this test
+ getTextAnnotation(ci, PERSON, CONTEXT, DBPEDIA_PERSON);
+ getTextAnnotation(ci, ORGANISATION, CONTEXT, DBPEDIA_ORGANISATION);
+ getTextAnnotation(ci, PLACE, CONTEXT, DBPEDIA_PLACE);
+ //perform the computation of the enhancements
+ entityMentionEngine.computeEnhancements(ci);
+ // ... and test the results
+ /*
+ * TODO: rw 20100617
+ * - Expected results depend on the used Index.
+ * - Use an example where the Organisation, Person and Place is part
+ * of the index
+ */
+ int entityAnnotationCount = checkAllEntityAnnotations(ci.getMetadata());
+ assertEquals(2, entityAnnotationCount);
}
-
+
/*
- * -----------------------------------------------------------------------
- * Helper Methods to check Text and EntityAnnotations
- * -----------------------------------------------------------------------
- */
-
+ * -----------------------------------------------------------------------
+ * Helper Methods to check Text and EntityAnnotations
+ * -----------------------------------------------------------------------
+ */
+
/**
* @param g
* @return
*/
private int checkAllEntityAnnotations(MGraph g) {
- Iterator<Triple> entityAnnotationIterator = g.filter(null,
- Properties.RDF_TYPE, TechnicalClasses.FISE_ENTITYANNOTATION);
+ Iterator<Triple> entityAnnotationIterator = g.filter(null, RDF_TYPE, FISE_ENTITYANNOTATION);
int entityAnnotationCount = 0;
while (entityAnnotationIterator.hasNext()) {
UriRef entityAnnotation = (UriRef) entityAnnotationIterator.next().getSubject();
@@ -165,6 +173,7 @@ public class TestEntityMentionEnhancemen
}
return entityAnnotationCount;
}
+
/**
* Checks if an entity annotation is valid
*
@@ -172,20 +181,17 @@ public class TestEntityMentionEnhancemen
* @param textAnnotation
*/
private void checkEntityAnnotation(MGraph g, UriRef entityAnnotation) {
- Iterator<Triple> relationToTextAnnotationIterator = g.filter(
- entityAnnotation, Properties.DC_RELATION, null);
+ Iterator<Triple> relationToTextAnnotationIterator = g.filter(entityAnnotation, DC_RELATION, null);
// check if the relation to the text annotation is set
assertTrue(relationToTextAnnotationIterator.hasNext());
while (relationToTextAnnotationIterator.hasNext()) {
// test if the referred annotations are text annotations
UriRef referredTextAnnotation = (UriRef) relationToTextAnnotationIterator.next().getObject();
- assertTrue(g.filter(referredTextAnnotation, Properties.RDF_TYPE,
- TechnicalClasses.FISE_TEXTANNOTATION).hasNext());
+ assertTrue(g.filter(referredTextAnnotation, RDF_TYPE, FISE_TEXTANNOTATION).hasNext());
}
// test if an entity is referred
- Iterator<Triple> entityReferenceIterator = g.filter(entityAnnotation,
- Properties.FISE_ENTITY_REFERENCE, null);
+ Iterator<Triple> entityReferenceIterator = g.filter(entityAnnotation, FISE_ENTITY_REFERENCE, null);
assertTrue(entityReferenceIterator.hasNext());
// test if the reference is an URI
assertTrue(entityReferenceIterator.next().getObject() instanceof UriRef);
@@ -193,8 +199,7 @@ public class TestEntityMentionEnhancemen
assertFalse(entityReferenceIterator.hasNext());
// finally test if the entity label is set
- Iterator<Triple> entityLabelIterator = g.filter(entityAnnotation,
- Properties.FISE_ENTITY_LABEL, null);
+ Iterator<Triple> entityLabelIterator = g.filter(entityAnnotation, FISE_ENTITY_LABEL, null);
assertTrue(entityLabelIterator.hasNext());
}
Modified: incubator/stanbol/trunk/fise/engines/geonames/src/main/java/eu/iksproject/fise/engines/geonames/impl/LocationEnhancementEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/fise/engines/geonames/src/main/java/eu/iksproject/fise/engines/geonames/impl/LocationEnhancementEngine.java?rev=1044386&r1=1044385&r2=1044386&view=diff
==============================================================================
--- incubator/stanbol/trunk/fise/engines/geonames/src/main/java/eu/iksproject/fise/engines/geonames/impl/LocationEnhancementEngine.java (original)
+++ incubator/stanbol/trunk/fise/engines/geonames/src/main/java/eu/iksproject/fise/engines/geonames/impl/LocationEnhancementEngine.java Fri Dec 10 15:00:13 2010
@@ -6,6 +6,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Dictionary;
+import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -49,199 +50,191 @@ import eu.iksproject.fise.servicesapi.rd
public class LocationEnhancementEngine implements EnhancementEngine, ServiceProperties {
/**
- * The default value for the Execution of this Engine. Currently set to
+ * The default value for the Execution of this Engine. Currently set to
* {@link ServiceProperties#ORDERING_EXTRACTION_ENHANCEMENT}
*/
- public static final Integer defaultOrder = ServiceProperties.ORDERING_EXTRACTION_ENHANCEMENT;
- private static final Logger log = LoggerFactory.getLogger(EnhancementEngineHelper.class);
- /**
- * This maps geonames.org feature classes to dbPedia.org ontology classes
- */
- public static final Map<FeatureClass,Collection<UriRef>> FEATURE_CLASS_CONCEPT_MAPPINGS;
- public static final Map<String,Collection<UriRef>> FEATURE_TYPE_CONCEPT_MAPPINGS;
- /**
- * Default value for minimum scores of search results are added to the
- * metadata of the parsed ContentItem
- */
- private static final Double DEFAULT_MIN_SCORE = 0.33;
- @Property
- public static final String MIN_SCORE = "eu.iksproject.fise.engines.geonames.locationEnhancementEngine.min-score";
- /**
- * Default values for the number of results returned by search requests
- * to the geonames.org web service
- */
- private static final Integer DEFAULT_MAX_LOCATION_ENHANCEMENTS = 5;
- @Property
- public static final String MAX_LOCATION_ENHANCEMENTS = "eu.iksproject.fise.engines.geonames.locationEnhancementEngine.max-location-enhancements";
- /**
- * Default value for the minimum score of search results used to also add
- * the hierarchy
- */
- private static final Double DEFAULT_MIN_HIERARCHY_SCORE = 0.70;
- @Property
- public static final String MIN_HIERARCHY_SCORE = "eu.iksproject.fise.engines.geonames.locationEnhancementEngine.min-hierarchy-score";
-
- public static final UriRef CONCEPT_GEONAMES_FEATURE = new UriRef(NamespaceEnum.geonames.toString()+"Feature");
-
- static {
- Map<FeatureClass,Collection<UriRef>> mappings = new java.util.EnumMap<FeatureClass,Collection<UriRef>>(FeatureClass.class);
- //first add the concepts of the geonames ontology
- for(FeatureClass fc : FeatureClass.values()){
- List<UriRef> conceptMappings = new ArrayList<UriRef>();
- conceptMappings.add(CONCEPT_GEONAMES_FEATURE); //all things are features
- conceptMappings.add(OntologicalClasses.DBPEDIA_PLACE); //all things are dbPedia places
- mappings.put(fc, conceptMappings);
- }
- //now add additional mappings to the dbPedia Ontology
- UriRef populatedPlace = new UriRef(NamespaceEnum.dbpedia_ont+"PopulatedPlace");
- mappings.get(FeatureClass.P).addAll(Arrays.asList(
- populatedPlace,
- new UriRef(NamespaceEnum.dbpedia_ont+"Settlement")));
- mappings.get(FeatureClass.A).addAll(Arrays.asList(
- populatedPlace,
- new UriRef(NamespaceEnum.dbpedia_ont+"AdministrativeRegion")));
- mappings.get(FeatureClass.H).add(
- new UriRef(NamespaceEnum.dbpedia_ont+"BodyOfWater"));
- mappings.get(FeatureClass.R).add(
- new UriRef(NamespaceEnum.dbpedia_ont+"Infrastructure"));
- mappings.get(FeatureClass.S).add(
- new UriRef(NamespaceEnum.dbpedia_ont+"Building"));
- mappings.get(FeatureClass.T).add(
- new UriRef(NamespaceEnum.dbpedia_ont+"Mountain"));
- //now write the unmodifiable static final constant
- FEATURE_CLASS_CONCEPT_MAPPINGS = Collections.unmodifiableMap(mappings);
-
- //Mappings for known FeatureTypes
- Map<String,Collection<UriRef>> typeMappings = new HashMap<String, Collection<UriRef>>();
- Collection<UriRef> lakeTypes = Arrays.asList(
- new UriRef(NamespaceEnum.dbpedia_ont+"Lake"));
- typeMappings.put("H.LK", lakeTypes);
- typeMappings.put("H.LKS", lakeTypes);
- typeMappings.put("H.LKI", lakeTypes);
- typeMappings.put("H.LKN", lakeTypes);
- typeMappings.put("H.LK", lakeTypes);
- typeMappings.put("H.LKO", lakeTypes);
- typeMappings.put("H.LKX", lakeTypes);
- typeMappings.put("H.LKC", lakeTypes);
- typeMappings.put("H.LKNI", lakeTypes);
- typeMappings.put("H.LKSI", lakeTypes);
- typeMappings.put("H.LKOI", lakeTypes);
- typeMappings.put("H.LKSN", lakeTypes);
- typeMappings.put("H.LKSC", lakeTypes);
- typeMappings.put("H.LKSB", lakeTypes);
- typeMappings.put("H.LKSNI", lakeTypes);
- typeMappings.put("H.RSV", lakeTypes);
-
- UriRef stream = new UriRef(NamespaceEnum.dbpedia_ont+" Stream");
- Collection<UriRef> canalTypes = Arrays.asList(
- stream,
- new UriRef(NamespaceEnum.dbpedia_ont+"Canal"));
- typeMappings.put("H.CNL", canalTypes);
- typeMappings.put("H.CNLA", canalTypes);
- typeMappings.put("H.CNLB", canalTypes);
- typeMappings.put("H.CNLI", canalTypes);
- typeMappings.put("H.CNLD", canalTypes);
- typeMappings.put("H.CNLSB", canalTypes);
- typeMappings.put("H.CNLN", canalTypes);
- typeMappings.put("H.CNLQ", canalTypes);
- typeMappings.put("H.CNLX", canalTypes);
-
- Collection<UriRef> riverTypes = Arrays.asList(
- stream,
- new UriRef(NamespaceEnum.dbpedia_ont+"River"));
- typeMappings.put("H.STM", riverTypes);
- typeMappings.put("H.STMI", riverTypes);
- typeMappings.put("H.STMB", riverTypes);
- typeMappings.put("H.STMD", riverTypes);
- typeMappings.put("H.STMM", riverTypes);
- typeMappings.put("H.STMA", riverTypes);
- typeMappings.put("H.STMC", riverTypes);
- typeMappings.put("H.STMX", riverTypes);
- typeMappings.put("H.STMIX", riverTypes);
- typeMappings.put("H.STMH", riverTypes);
- typeMappings.put("H.STMSB", riverTypes);
- typeMappings.put("H.STMQ", riverTypes);
- typeMappings.put("H.STMS", riverTypes);
- typeMappings.put("H.STM", riverTypes);
- typeMappings.put("H.STM", riverTypes);
- typeMappings.put("H.STM", riverTypes);
-
- Collection<UriRef> caveTypes = Arrays.asList(
- new UriRef(NamespaceEnum.dbpedia_ont+"Cave"));
- typeMappings.put("H.LKSB", caveTypes);
- typeMappings.put("R.TNLN", caveTypes);
- typeMappings.put("S.CAVE", caveTypes);
- typeMappings.put("S.BUR", caveTypes);
-
- Collection<UriRef> countryTypes = Arrays.asList(
- new UriRef(NamespaceEnum.dbpedia_ont+"Country"));
- typeMappings.put("A.PCLI", countryTypes);
-
- UriRef settlement = new UriRef(NamespaceEnum.dbpedia_ont+"Settlement");
- Collection<UriRef> cityTypes = Arrays.asList(
- settlement,
- new UriRef(NamespaceEnum.dbpedia_ont+"City"));
- Collection<UriRef> villageTypes = Arrays.asList(
- settlement,
- new UriRef(NamespaceEnum.dbpedia_ont+"Village"));
- typeMappings.put("P.PPLG", cityTypes);
- typeMappings.put("P.PPLC", cityTypes);
- typeMappings.put("P.PPLF", villageTypes);
- typeMappings.put("P.PPLA", cityTypes);
- //write the mappings as unmodifiable map the the static final constant
- FEATURE_TYPE_CONCEPT_MAPPINGS = Collections.unmodifiableMap(typeMappings);
- }
-
-
- private Integer maxLocationEnhancements;
- private Double minScore;
- private Double minHierarchyScore;
+ public static final Integer defaultOrder = ORDERING_EXTRACTION_ENHANCEMENT;
+
+ private static final Logger log = LoggerFactory.getLogger(EnhancementEngineHelper.class);
+
+ /**
+ * This maps geonames.org feature classes to dbPedia.org ontology classes
+ */
+ public static final Map<FeatureClass, Collection<UriRef>> FEATURE_CLASS_CONCEPT_MAPPINGS;
+
+ public static final Map<String, Collection<UriRef>> FEATURE_TYPE_CONCEPT_MAPPINGS;
+
+ /**
+ * Default value for minimum scores of search results are added to the
+ * metadata of the parsed ContentItem
+ */
+ private static final Double DEFAULT_MIN_SCORE = 0.33;
+
+ @Property
+ public static final String MIN_SCORE = "eu.iksproject.fise.engines.geonames.locationEnhancementEngine.min-score";
+
+ /**
+ * Default values for the number of results returned by search requests
+ * to the geonames.org web service
+ */
+ private static final Integer DEFAULT_MAX_LOCATION_ENHANCEMENTS = 5;
+
+ @Property
+ public static final String MAX_LOCATION_ENHANCEMENTS = "eu.iksproject.fise.engines.geonames.locationEnhancementEngine.max-location-enhancements";
+
+ /**
+ * Default value for the minimum score of search results used to also add
+ * the hierarchy
+ */
+ private static final Double DEFAULT_MIN_HIERARCHY_SCORE = 0.70;
+
+ @Property
+ public static final String MIN_HIERARCHY_SCORE = "eu.iksproject.fise.engines.geonames.locationEnhancementEngine.min-hierarchy-score";
+
+ public static final UriRef CONCEPT_GEONAMES_FEATURE = new UriRef(NamespaceEnum.geonames.toString() + "Feature");
+
+ static {
+ Map<FeatureClass, Collection<UriRef>> mappings = new EnumMap<FeatureClass, Collection<UriRef>>(FeatureClass.class);
+ //first add the concepts of the geonames ontology
+ for (FeatureClass fc : FeatureClass.values()) {
+ List<UriRef> conceptMappings = new ArrayList<UriRef>();
+ conceptMappings.add(CONCEPT_GEONAMES_FEATURE); //all things are features
+ conceptMappings.add(OntologicalClasses.DBPEDIA_PLACE); //all things are dbPedia places
+ mappings.put(fc, conceptMappings);
+ }
+ //now add additional mappings to the dbPedia Ontology
+ UriRef populatedPlace = new UriRef(NamespaceEnum.dbpedia_ont + "PopulatedPlace");
+ mappings.get(FeatureClass.P).addAll(Arrays.asList(populatedPlace, new UriRef(NamespaceEnum.dbpedia_ont + "Settlement")));
+ mappings.get(FeatureClass.A).addAll(Arrays.asList(populatedPlace, new UriRef(NamespaceEnum.dbpedia_ont + "AdministrativeRegion")));
+ mappings.get(FeatureClass.H).add(new UriRef(NamespaceEnum.dbpedia_ont + "BodyOfWater"));
+ mappings.get(FeatureClass.R).add(new UriRef(NamespaceEnum.dbpedia_ont + "Infrastructure"));
+ mappings.get(FeatureClass.S).add(new UriRef(NamespaceEnum.dbpedia_ont + "Building"));
+ mappings.get(FeatureClass.T).add(new UriRef(NamespaceEnum.dbpedia_ont + "Mountain"));
+ //now write the unmodifiable static final constant
+ FEATURE_CLASS_CONCEPT_MAPPINGS = Collections.unmodifiableMap(mappings);
+
+ //Mappings for known FeatureTypes
+ Map<String, Collection<UriRef>> typeMappings = new HashMap<String, Collection<UriRef>>();
+ Collection<UriRef> lakeTypes = Arrays.asList(new UriRef(NamespaceEnum.dbpedia_ont + "Lake"));
+ typeMappings.put("H.LK", lakeTypes);
+ typeMappings.put("H.LKS", lakeTypes);
+ typeMappings.put("H.LKI", lakeTypes);
+ typeMappings.put("H.LKN", lakeTypes);
+ typeMappings.put("H.LK", lakeTypes);
+ typeMappings.put("H.LKO", lakeTypes);
+ typeMappings.put("H.LKX", lakeTypes);
+ typeMappings.put("H.LKC", lakeTypes);
+ typeMappings.put("H.LKNI", lakeTypes);
+ typeMappings.put("H.LKSI", lakeTypes);
+ typeMappings.put("H.LKOI", lakeTypes);
+ typeMappings.put("H.LKSN", lakeTypes);
+ typeMappings.put("H.LKSC", lakeTypes);
+ typeMappings.put("H.LKSB", lakeTypes);
+ typeMappings.put("H.LKSNI", lakeTypes);
+ typeMappings.put("H.RSV", lakeTypes);
+
+ UriRef stream = new UriRef(NamespaceEnum.dbpedia_ont + " Stream");
+ Collection<UriRef> canalTypes = Arrays.asList(stream, new UriRef(NamespaceEnum.dbpedia_ont + "Canal"));
+ typeMappings.put("H.CNL", canalTypes);
+ typeMappings.put("H.CNLA", canalTypes);
+ typeMappings.put("H.CNLB", canalTypes);
+ typeMappings.put("H.CNLI", canalTypes);
+ typeMappings.put("H.CNLD", canalTypes);
+ typeMappings.put("H.CNLSB", canalTypes);
+ typeMappings.put("H.CNLN", canalTypes);
+ typeMappings.put("H.CNLQ", canalTypes);
+ typeMappings.put("H.CNLX", canalTypes);
+
+ Collection<UriRef> riverTypes = Arrays.asList(stream, new UriRef(NamespaceEnum.dbpedia_ont + "River"));
+ typeMappings.put("H.STM", riverTypes);
+ typeMappings.put("H.STMI", riverTypes);
+ typeMappings.put("H.STMB", riverTypes);
+ typeMappings.put("H.STMD", riverTypes);
+ typeMappings.put("H.STMM", riverTypes);
+ typeMappings.put("H.STMA", riverTypes);
+ typeMappings.put("H.STMC", riverTypes);
+ typeMappings.put("H.STMX", riverTypes);
+ typeMappings.put("H.STMIX", riverTypes);
+ typeMappings.put("H.STMH", riverTypes);
+ typeMappings.put("H.STMSB", riverTypes);
+ typeMappings.put("H.STMQ", riverTypes);
+ typeMappings.put("H.STMS", riverTypes);
+ typeMappings.put("H.STM", riverTypes);
+ typeMappings.put("H.STM", riverTypes);
+ typeMappings.put("H.STM", riverTypes);
+
+ Collection<UriRef> caveTypes = Arrays.asList(new UriRef(NamespaceEnum.dbpedia_ont + "Cave"));
+ typeMappings.put("H.LKSB", caveTypes);
+ typeMappings.put("R.TNLN", caveTypes);
+ typeMappings.put("S.CAVE", caveTypes);
+ typeMappings.put("S.BUR", caveTypes);
+
+ Collection<UriRef> countryTypes = Arrays.asList(new UriRef(NamespaceEnum.dbpedia_ont + "Country"));
+ typeMappings.put("A.PCLI", countryTypes);
+
+ UriRef settlement = new UriRef(NamespaceEnum.dbpedia_ont + "Settlement");
+ Collection<UriRef> cityTypes = Arrays.asList(settlement, new UriRef(NamespaceEnum.dbpedia_ont + "City"));
+ Collection<UriRef> villageTypes = Arrays.asList(settlement, new UriRef(NamespaceEnum.dbpedia_ont + "Village"));
+ typeMappings.put("P.PPLG", cityTypes);
+ typeMappings.put("P.PPLC", cityTypes);
+ typeMappings.put("P.PPLF", villageTypes);
+ typeMappings.put("P.PPLA", cityTypes);
+ //write the mappings as unmodifiable map the the static final constant
+ FEATURE_TYPE_CONCEPT_MAPPINGS = Collections.unmodifiableMap(typeMappings);
+ }
+
+
+ private Integer maxLocationEnhancements;
+ private Double minScore;
+ private Double minHierarchyScore;
@SuppressWarnings("unchecked")
- protected void activate(ComponentContext ce) throws IOException {
+ protected void activate(ComponentContext ce) throws IOException {
Dictionary<String, String> properties = ce.getProperties();
String value;
log.debug("activating ...");
- try{
- value = properties.get(MIN_SCORE);
- log.debug(" -> "+MIN_SCORE+"="+value);
- setMinScore(value == null?null:Double.valueOf(value));
- log.debug(" <- "+MIN_SCORE+"="+getMinScore());
+ try {
+ value = properties.get(MIN_SCORE);
+ log.debug(" -> " + MIN_SCORE + "=" + value);
+ setMinScore(value == null ? null : Double.valueOf(value));
+ log.debug(" <- " + MIN_SCORE + "=" + getMinScore());
} catch (NumberFormatException e) {
- log.warn("Unable to parse MIN_SCORE",e);
- setMinScore(null);
- }
- try{
- value = properties.get(MAX_LOCATION_ENHANCEMENTS);
- log.debug(" -> "+MAX_LOCATION_ENHANCEMENTS+"="+value);
- setMaxLocationEnhancements(value == null?null:Integer.valueOf(value));
- log.debug(" <- "+MAX_LOCATION_ENHANCEMENTS+"="+getMaxLocationEnhancements());
+ log.warn("Unable to parse MIN_SCORE", e);
+ setMinScore(null);
+ }
+ try {
+ value = properties.get(MAX_LOCATION_ENHANCEMENTS);
+ log.debug(" -> " + MAX_LOCATION_ENHANCEMENTS + "=" + value);
+ setMaxLocationEnhancements(value == null ? null : Integer.valueOf(value));
+ log.debug(" <- " + MAX_LOCATION_ENHANCEMENTS + "=" + getMaxLocationEnhancements());
} catch (NumberFormatException e) {
- log.warn("Unable to parse MAX_LOCATION_ENHANCEMENTS",e);
- setMaxLocationEnhancements(null);
- }
- try{
- value = properties.get(MIN_HIERARCHY_SCORE);
- log.debug(" -> "+MIN_HIERARCHY_SCORE+"="+value);
- setMinHierarchyScore(value == null?null:Double.valueOf(value));
- log.debug(" <- "+MIN_HIERARCHY_SCORE+"="+getMinHierarchyScore());
+ log.warn("Unable to parse MAX_LOCATION_ENHANCEMENTS", e);
+ setMaxLocationEnhancements(null);
+ }
+ try {
+ value = properties.get(MIN_HIERARCHY_SCORE);
+ log.debug(" -> " + MIN_HIERARCHY_SCORE + "=" + value);
+ setMinHierarchyScore(value == null ? null : Double.valueOf(value));
+ log.debug(" <- " + MIN_HIERARCHY_SCORE + "=" + getMinHierarchyScore());
} catch (NumberFormatException e) {
- log.warn("Unable to parse MIN_HIERARCHY_SCORE",e);
- setMinHierarchyScore(null);
- }
+ log.warn("Unable to parse MIN_HIERARCHY_SCORE", e);
+ setMinHierarchyScore(null);
+ }
}
+
protected void deactivate(ComponentContext ce) {
setMinScore(null);
setMaxLocationEnhancements(null);
setMinHierarchyScore(null);
}
+
@Override
- public int canEnhance(ContentItem ci) throws EngineException {
- return EnhancementEngine.ENHANCE_SYNCHRONOUS;
- }
+ public int canEnhance(ContentItem ci) throws EngineException {
+ return EnhancementEngine.ENHANCE_SYNCHRONOUS;
+ }
- @Override
- public void computeEnhancements(ContentItem ci) throws EngineException {
+ @Override
+ public void computeEnhancements(ContentItem ci) throws EngineException {
UriRef contentItemId = new UriRef(ci.getId());
MGraph graph = ci.getMetadata();
LiteralFactory literalFactory = LiteralFactory.getInstance();
@@ -253,305 +246,305 @@ public class LocationEnhancementEngine i
* selecting the same name.
*/
Map<String, Collection<NonLiteral>> name2placeEnhancementMap = new HashMap<String, Collection<NonLiteral>>();
- for(Iterator<Triple> it = graph.filter(null, Properties.DC_TYPE, OntologicalClasses.DBPEDIA_PLACE); it.hasNext();){
- NonLiteral placeEnhancement = it.next().getSubject(); //the enhancement annotating an place
- //this can still be an TextAnnotation of an EntityAnnotation
- //so we need to filter TextAnnotation
- Triple isTextAnnotation = new TripleImpl(placeEnhancement, Properties.RDF_TYPE, TechnicalClasses.FISE_TEXTANNOTATION);
- if(graph.contains(isTextAnnotation)){
- //now get the name
- String name = EnhancementEngineHelper.getString(graph, placeEnhancement, Properties.FISE_SELECTED_TEXT);
- if(name == null){
- log.warn("Unable to process TextAnnotation "+placeEnhancement+" because property"+
- Properties.FISE_SELECTED_TEXT+ " is not present");
- } else {
- Collection<NonLiteral> placeEnhancements = name2placeEnhancementMap.get(name);
- if(placeEnhancements == null){
- placeEnhancements = new ArrayList<NonLiteral>();
- name2placeEnhancementMap.put(name, placeEnhancements);
- }
- placeEnhancements.add(placeEnhancement);
- }
- } else {
- //TODO: if we also ant to process EntityAnnotations with the dc:type dbPedia:Place
- // than we need to parse the name based on the fise:entity-name property
- }
+ for (Iterator<Triple> it = graph.filter(null, Properties.DC_TYPE, OntologicalClasses.DBPEDIA_PLACE); it.hasNext();) {
+ NonLiteral placeEnhancement = it.next().getSubject(); //the enhancement annotating an place
+ //this can still be an TextAnnotation of an EntityAnnotation
+ //so we need to filter TextAnnotation
+ Triple isTextAnnotation = new TripleImpl(placeEnhancement, Properties.RDF_TYPE, TechnicalClasses.FISE_TEXTANNOTATION);
+ if (graph.contains(isTextAnnotation)) {
+ //now get the name
+ String name = EnhancementEngineHelper.getString(graph, placeEnhancement, Properties.FISE_SELECTED_TEXT);
+ if (name == null) {
+ log.warn("Unable to process TextAnnotation " + placeEnhancement + " because property" + Properties.FISE_SELECTED_TEXT + " is not present");
+ } else {
+ Collection<NonLiteral> placeEnhancements = name2placeEnhancementMap.get(name);
+ if (placeEnhancements == null) {
+ placeEnhancements = new ArrayList<NonLiteral>();
+ name2placeEnhancementMap.put(name, placeEnhancements);
+ }
+ placeEnhancements.add(placeEnhancement);
+ }
+ } else {
+ //TODO: if we also ant to process EntityAnnotations with the dc:type dbPedia:Place
+ // than we need to parse the name based on the fise:entity-name property
+ }
}
//Now we do have all the names we need to lookup
- for(Map.Entry<String,Collection<NonLiteral>> entry : name2placeEnhancementMap.entrySet()){
- ToponymSearchResult results;
- try {
- results = lookupLocation(entry.getKey());
- } catch (Exception e) {
- /*
- * TODO: Review if it makes sense to catch here for each name, or
- * to catch the whole loop.
- * This depends if single requests can result in Exceptions
- * (e.g. because of encoding problems) or if usually Exceptions
- * are thrown because of general things like connection issues
- * or service unavailability.
- */
+ for (Map.Entry<String, Collection<NonLiteral>> entry : name2placeEnhancementMap.entrySet()) {
+ ToponymSearchResult results;
+ try {
+ results = lookupLocation(entry.getKey());
+ } catch (Exception e) {
+ /*
+ * TODO: Review if it makes sense to catch here for each name, or
+ * to catch the whole loop.
+ * This depends if single requests can result in Exceptions
+ * (e.g. because of encoding problems) or if usually Exceptions
+ * are thrown because of general things like connection issues
+ * or service unavailability.
+ */
throw new EngineException(this, ci, e);
- }
- if(results != null){
- for(Toponym result : results.getToponyms()){
- log.info("process result "+result.getGeoNameId()+" "+result.getName());
- Double score;
- try {
- score = getToponymScore(result);
- } catch (InsufficientStyleException e) {
- throw new IllegalStateException("Change style of this implementation to get the score values",e);
- }
- log.info(" > score "+score);
- if(score != null){
- if(score.doubleValue() < minScore){
- //if score is lower than the under bound, than stop
- break;
- }
- } else {
- log.warn("NULL returned as Score for "+result.getGeoNameId()+" "+result.getName());
- /*
- * NOTE: If score is not present all suggestions are
- * added as enhancements to the metadata of the content
- * item.
- */
- }
- //write the enhancement!
- NonLiteral locationEnhancement = writeEntityEnhancement(contentItemId, graph, literalFactory, result, entry.getValue(),null);
- log.info(" > "+score+ " >= "+minHierarchyScore);
- if(score != null && score >= minHierarchyScore){
- log.info(" > getHierarchy for "+result.getGeoNameId()+" "+result.getName());
- //get the hierarchy
- try {
- Iterator<Toponym> hierarchy = getHierarchy(result).iterator();
- for(int level = 0;hierarchy.hasNext();level++){
- Toponym hierarchyEntry = hierarchy.next();
- //TODO: filter the interesting entries
- // maybe add an configuration
- if(level == 0){
- //Mother earth -> ignore
- continue;
- }
- //write it as dependent to the locationEnhancement
- if(result.getGeoNameId() != hierarchyEntry.getGeoNameId()){
- //TODO: add additional checks based on possible
- // configuration here!
- log.info(" - write hierarchy "+hierarchyEntry.getGeoNameId()+" "+hierarchyEntry.getName());
- /*
- * The hierarchy service dose not provide a score, because it would be 1.0
- * so we need to set the score to this value.
- * Currently is is set to the value of the suggested entry
- */
- hierarchyEntry.setScore(score);
- writeEntityEnhancement(contentItemId, graph, literalFactory,
- hierarchyEntry, null, Collections.singletonList(locationEnhancement));
- }
- }
- } catch (Exception e) {
- log.warn("Unable to get Hierarchy for "+result.getGeoNameId()+" "+result.getName(),e);
- }
- }
- }
- }
- }
- }
- /**
- * Getter for the socre in a range from [0..1]<p>
- * NOTE (2010.11.16, rw): GeoNames previously returned the score in the
- * range from [0..1]. It looks like that up from now they use the
- * range [0..100]. Therefore I created this method to make the necessary
- * adaptation.
- * see also http://code.google.com/p/iks-project/issues/detail?id=89
- * @param toponym the toponym
- * @return the score in a range [0..1]
- * @throws InsufficientStyleException
- */
- private Double getToponymScore(Toponym toponym) throws InsufficientStyleException {
- return toponym.getScore()/100;
- }
- /**
- * Returns the hierarchy for the parsed toponym. The planet Earth will be
- * at the first position of the list and the parsed toponum represents the
- * last element of the list
- * @param toponym the toponym
- * @return The list containing the hierarchy
- * @throws Exception on any error while accessing the webservice
- */
- protected Collection<Toponym> getHierarchy(Toponym toponym) throws Exception{
- return WebService.hierarchy(toponym.getGeoNameId(), null, Style.FULL);
- }
- /**
- * Writes an entity enhancement for the content item in the parsed graph
- * based on the parsed toponym.
- * @param contentItemId The id of the contentItem
- * @param graph The graph used to write the triples
- * @param literalFactory the literal factory used to create literals
- * @param toponym the toponym
- * @param relatedEnhancements related enhancements
- * @param requiresEnhancements required enhancements
- * @return The UriRef of the created entity enhancement
- */
- private UriRef writeEntityEnhancement(UriRef contentItemId, MGraph graph, LiteralFactory literalFactory, Toponym toponym, Collection<NonLiteral> relatedEnhancements, Collection<NonLiteral> requiresEnhancements) {
- UriRef entityRef = new UriRef("http://sws.geonames.org/"+toponym.getGeoNameId()+'/');
- FeatureClass featureClass = toponym.getFeatureClass();
- log.debug(" > featureClass "+featureClass);
- UriRef entityAnnotation = EnhancementEngineHelper.createEntityEnhancement(
- graph, this, contentItemId);
- // first relate this entity annotation to the text annotation(s)
- if(relatedEnhancements != null){
- for(NonLiteral related : relatedEnhancements){
- graph.add(new TripleImpl(entityAnnotation,
- Properties.DC_RELATION, related));
- }
- }
- if(requiresEnhancements != null){
- for(NonLiteral requires : requiresEnhancements){
- graph.add(new TripleImpl(entityAnnotation,
- Properties.DC_REQUIRES, requires));
- }
- }
- graph.add(new TripleImpl(entityAnnotation,
- Properties.FISE_ENTITY_REFERENCE, entityRef));
- log.debug(" > name "+toponym.getName());
- graph.add(new TripleImpl(entityAnnotation,
- Properties.FISE_ENTITY_LABEL,
- literalFactory.createTypedLiteral(toponym.getName())));
- Double score;
- try {
- score = getToponymScore(toponym);
- if(score != null){
- graph.add(new TripleImpl(entityAnnotation,
- Properties.FISE_CONFIDENCE,
- literalFactory.createTypedLiteral(score)));
- }
- } catch (InsufficientStyleException e) {
- //ignore -> don not write the score
- }
- //now get all the entity types for the results
- Set<UriRef> entityTypes = new HashSet<UriRef>();
- //first based on the feature class
- Collection<UriRef> featureClassTypes = FEATURE_CLASS_CONCEPT_MAPPINGS.get(featureClass);
- if(featureClassTypes != null){
- entityTypes.addAll(featureClassTypes);
- }
- //second for the feature Code
- String featureCode = toponym.getFeatureCode();
- Collection<UriRef> featureCodeTypes = FEATURE_TYPE_CONCEPT_MAPPINGS.get(featureCode);
- if(featureCodeTypes != null){
- entityTypes.addAll(featureCodeTypes);
- }
- //third add the feature Code as additional type
- entityTypes.add(new UriRef(NamespaceEnum.geonames+featureClass.name()+'.'+featureCode));
- //finally add the type triples to the enhancement
- for (UriRef entityType : entityTypes) {
- graph.add(new TripleImpl(entityAnnotation,
- Properties.FISE_ENTITY_TYPE, entityType));
- }
- return entityAnnotation;
- }
-
- protected ToponymSearchResult lookupLocation(String name) throws Exception {
- ToponymSearchCriteria criteria = new ToponymSearchCriteria();
- criteria.setName(name);
- criteria.setMaxRows(maxLocationEnhancements);
- criteria.setStyle(Style.FULL);
- criteria.setMaxRows(5);
- // criteria.setLanguage("language") TODO: search for the language of the document!
- return WebService.search(criteria);
-
- }
- /**
- * Getter for the maximum number of enhancements added for a single
- * TextAnnoation.
- * @return The maximum number of location enhancements added for a single
- * text annotation
- */
- public final Integer getMaxLocationEnhancements() {
- return maxLocationEnhancements;
- }
- /**
- * Setter for the maximum number of enhancements added for a single
- * TextAnnoation. If the parsed value is <code>null</code> or < 1, than the
- * value is set to {@link LocationEnhancementEngine#DEFAULT_MAX_LOCATION_ENHANCEMENTS}.
- * @param maxNumber the maximum number of enhancements added to a singel
- * text annotation
- */
- public final void setMaxLocationEnhancements(Integer maxNumber) {
- if(maxNumber == null){
- maxNumber = DEFAULT_MAX_LOCATION_ENHANCEMENTS;
- }
- if(maxNumber < 1){
- maxNumber = DEFAULT_MAX_LOCATION_ENHANCEMENTS;
- }
- this.maxLocationEnhancements = maxNumber;
- }
-
- @Override
- public Map<String, Object> getServiceProperties() {
- return Collections.unmodifiableMap(
- Collections.singletonMap(
- ServiceProperties.ENHANCEMENT_ENGINE_ORDERING,
- (Object)defaultOrder));
- }
-
- /**
- * Setter for the minimum score used to decide if results of geoname.org
- * search results are added as EntityEnhancements to the ContentItem.
- * If <code>null</code> is parsed the value is set to
- * {@link LocationEnhancementEngine#DEFAULT_MIN_SCORE}. For values > 1 the
- * value is set to 1 and for values < 0 the value is set to 0.
- * @param minScore the minScore to set
- */
- public void setMinScore(Double minScore) {
- if(minScore == null){
- minScore = DEFAULT_MIN_SCORE;
- } else if(minScore > 1){
- minScore = Double.valueOf(1);
- } else if(minScore < 0){
- minScore = Double.valueOf(0);
- }
- this.minScore = minScore;
- }
-
- /**
- * Getter for the minimum score used to decide if results of geoname.org
- * search results are added as EntityEnhancements to the ContentItem
- * @return the minScore
- */
- public Double getMinScore() {
- return minScore;
- }
-
- /**
- * Setter for the minimum value used to decide based on the score of
- * locations returned by the geonames.org web service if also the hierarchy
- * of that point should be added as enhancements to the analysed
- * content item. <br>
- * If <code>null</code> is parsed the value is set to
- * {@link LocationEnhancementEngine#DEFAULT_MIN_HIERARCHY_SCORE}.
- * For values > 1 the value is set to 1 and for values < 0 the value
- * is set to 0.
- * @param minHierarchyScore the minHierarchyScore to set
- */
- public void setMinHierarchyScore(Double minHierarchyScore) {
- if(minHierarchyScore == null){
- minHierarchyScore = DEFAULT_MIN_HIERARCHY_SCORE;
- } else if (minHierarchyScore > 1){
- minHierarchyScore = Double.valueOf(1);
- } else if(minScore < 0){
- minHierarchyScore = Double.valueOf(0);
- }
- this.minHierarchyScore = minHierarchyScore;
- }
-
- /**
- * Setter for the minimum value used to decide based on the score of
- * locations returned by the geonames.org web service if also the hierarchy
- * of that point should be added as enhancements to the analysed
- * content item.
- * @return the minHierarchyScore
- */
- public Double getMinHierarchyScore() {
- return minHierarchyScore;
- }
+ }
+ if (results != null) {
+ for (Toponym result : results.getToponyms()) {
+ log.info("process result " + result.getGeoNameId() + " " + result.getName());
+ Double score;
+ try {
+ score = getToponymScore(result);
+ } catch (InsufficientStyleException e) {
+ throw new IllegalStateException("Change style of this implementation to get the score values", e);
+ }
+ log.info(" > score " + score);
+ if (score != null) {
+ if (score.doubleValue() < minScore) {
+ //if score is lower than the under bound, than stop
+ break;
+ }
+ } else {
+ log.warn("NULL returned as Score for " + result.getGeoNameId() + " " + result.getName());
+ /*
+ * NOTE: If score is not present all suggestions are
+ * added as enhancements to the metadata of the content
+ * item.
+ */
+ }
+ //write the enhancement!
+ NonLiteral locationEnhancement = writeEntityEnhancement(contentItemId, graph, literalFactory, result, entry.getValue(), null);
+ log.info(" > " + score + " >= " + minHierarchyScore);
+ if (score != null && score >= minHierarchyScore) {
+ log.info(" > getHierarchy for " + result.getGeoNameId() + " " + result.getName());
+ //get the hierarchy
+ try {
+ Iterator<Toponym> hierarchy = getHierarchy(result).iterator();
+ for (int level = 0; hierarchy.hasNext(); level++) {
+ Toponym hierarchyEntry = hierarchy.next();
+ //TODO: filter the interesting entries
+ // maybe add an configuration
+ if (level == 0) {
+ //Mother earth -> ignore
+ continue;
+ }
+ //write it as dependent to the locationEnhancement
+ if (result.getGeoNameId() != hierarchyEntry.getGeoNameId()) {
+ //TODO: add additional checks based on possible
+ // configuration here!
+ log.info(" - write hierarchy " + hierarchyEntry.getGeoNameId() + " " + hierarchyEntry.getName());
+ /*
+ * The hierarchy service dose not provide a score, because it would be 1.0
+ * so we need to set the score to this value.
+ * Currently is is set to the value of the suggested entry
+ */
+ hierarchyEntry.setScore(score);
+ writeEntityEnhancement(contentItemId, graph, literalFactory, hierarchyEntry, null, Collections.singletonList(locationEnhancement));
+ }
+ }
+ } catch (Exception e) {
+ log.warn("Unable to get Hierarchy for " + result.getGeoNameId() + " " + result.getName(), e);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Getter for the socre in a range from [0..1]<p>
+ * NOTE (2010.11.16, rw): GeoNames previously returned the score in the
+ * range from [0..1]. It looks like that up from now they use the
+ * range [0..100]. Therefore I created this method to make the necessary
+ * adaptation.
+ * see also http://code.google.com/p/iks-project/issues/detail?id=89
+ *
+ * @param toponym the toponym
+ * @return the score in a range [0..1]
+ * @throws InsufficientStyleException
+ */
+ private Double getToponymScore(Toponym toponym) throws InsufficientStyleException {
+ return toponym.getScore() / 100;
+ }
+
+ /**
+ * Returns the hierarchy for the parsed toponym. The planet Earth will be
+ * at the first position of the list and the parsed toponum represents the
+ * last element of the list
+ *
+ * @param toponym the toponym
+ * @return The list containing the hierarchy
+ * @throws Exception on any error while accessing the webservice
+ */
+ protected Collection<Toponym> getHierarchy(Toponym toponym) throws Exception {
+ return WebService.hierarchy(toponym.getGeoNameId(), null, Style.FULL);
+ }
+
+ /**
+ * Writes an entity enhancement for the content item in the parsed graph
+ * based on the parsed toponym.
+ *
+ * @param contentItemId The id of the contentItem
+ * @param graph The graph used to write the triples
+ * @param literalFactory the literal factory used to create literals
+ * @param toponym the toponym
+ * @param relatedEnhancements related enhancements
+ * @param requiresEnhancements required enhancements
+ * @return The UriRef of the created entity enhancement
+ */
+ private UriRef writeEntityEnhancement(UriRef contentItemId, MGraph graph, LiteralFactory literalFactory, Toponym toponym, Collection<NonLiteral> relatedEnhancements, Collection<NonLiteral> requiresEnhancements) {
+ UriRef entityRef = new UriRef("http://sws.geonames.org/" + toponym.getGeoNameId() + '/');
+ FeatureClass featureClass = toponym.getFeatureClass();
+ log.debug(" > featureClass " + featureClass);
+ UriRef entityAnnotation = EnhancementEngineHelper.createEntityEnhancement(graph, this, contentItemId);
+ // first relate this entity annotation to the text annotation(s)
+ if (relatedEnhancements != null) {
+ for (NonLiteral related : relatedEnhancements) {
+ graph.add(new TripleImpl(entityAnnotation, Properties.DC_RELATION, related));
+ }
+ }
+ if (requiresEnhancements != null) {
+ for (NonLiteral requires : requiresEnhancements) {
+ graph.add(new TripleImpl(entityAnnotation, Properties.DC_REQUIRES, requires));
+ }
+ }
+ graph.add(new TripleImpl(entityAnnotation, Properties.FISE_ENTITY_REFERENCE, entityRef));
+ log.debug(" > name " + toponym.getName());
+ graph.add(new TripleImpl(entityAnnotation, Properties.FISE_ENTITY_LABEL, literalFactory.createTypedLiteral(toponym.getName())));
+ Double score;
+ try {
+ score = getToponymScore(toponym);
+ if (score != null) {
+ graph.add(new TripleImpl(entityAnnotation, Properties.FISE_CONFIDENCE, literalFactory.createTypedLiteral(score)));
+ }
+ } catch (InsufficientStyleException e) {
+ //ignore -> don not write the score
+ }
+ //now get all the entity types for the results
+ Set<UriRef> entityTypes = new HashSet<UriRef>();
+ //first based on the feature class
+ Collection<UriRef> featureClassTypes = FEATURE_CLASS_CONCEPT_MAPPINGS.get(featureClass);
+ if (featureClassTypes != null) {
+ entityTypes.addAll(featureClassTypes);
+ }
+ //second for the feature Code
+ String featureCode = toponym.getFeatureCode();
+ Collection<UriRef> featureCodeTypes = FEATURE_TYPE_CONCEPT_MAPPINGS.get(featureCode);
+ if (featureCodeTypes != null) {
+ entityTypes.addAll(featureCodeTypes);
+ }
+ //third add the feature Code as additional type
+ entityTypes.add(new UriRef(NamespaceEnum.geonames + featureClass.name() + '.' + featureCode));
+ //finally add the type triples to the enhancement
+ for (UriRef entityType : entityTypes) {
+ graph.add(new TripleImpl(entityAnnotation, Properties.FISE_ENTITY_TYPE, entityType));
+ }
+ return entityAnnotation;
+ }
+
+ protected ToponymSearchResult lookupLocation(String name) throws Exception {
+ ToponymSearchCriteria criteria = new ToponymSearchCriteria();
+ criteria.setName(name);
+ criteria.setMaxRows(maxLocationEnhancements);
+ criteria.setStyle(Style.FULL);
+ criteria.setMaxRows(5);
+ // criteria.setLanguage("language") TODO: search for the language of the document!
+ return WebService.search(criteria);
+
+ }
+
+ /**
+ * Getter for the maximum number of enhancements added for a single
+ * TextAnnotation.
+ *
+ * @return The maximum number of location enhancements added for a single
+ * text annotation
+ */
+ public final Integer getMaxLocationEnhancements() {
+ return maxLocationEnhancements;
+ }
+
+ /**
+ * Setter for the maximum number of enhancements added for a single
+ * TextAnnotation. If the parsed value is <code>null</code> or < 1, than the
+ * value is set to {@link LocationEnhancementEngine#DEFAULT_MAX_LOCATION_ENHANCEMENTS}.
+ *
+ * @param maxNumber the maximum number of enhancements added to a singel
+ * text annotation
+ */
+ public final void setMaxLocationEnhancements(Integer maxNumber) {
+ if (maxNumber == null) {
+ maxNumber = DEFAULT_MAX_LOCATION_ENHANCEMENTS;
+ }
+ if (maxNumber < 1) {
+ maxNumber = DEFAULT_MAX_LOCATION_ENHANCEMENTS;
+ }
+ this.maxLocationEnhancements = maxNumber;
+ }
+
+ @Override
+ public Map<String, Object> getServiceProperties() {
+ return Collections.unmodifiableMap(Collections.singletonMap(ENHANCEMENT_ENGINE_ORDERING, (Object) defaultOrder));
+ }
+
+ /**
+ * Setter for the minimum score used to decide if results of geoname.org
+ * search results are added as EntityEnhancements to the ContentItem.
+ * If <code>null</code> is parsed the value is set to
+ * {@link LocationEnhancementEngine#DEFAULT_MIN_SCORE}. For values > 1 the
+ * value is set to 1 and for values < 0 the value is set to 0.
+ *
+ * @param minScore the minScore to set
+ */
+ public void setMinScore(Double minScore) {
+ if (minScore == null) {
+ minScore = DEFAULT_MIN_SCORE;
+ } else if (minScore > 1) {
+ minScore = Double.valueOf(1);
+ } else if (minScore < 0) {
+ minScore = Double.valueOf(0);
+ }
+ this.minScore = minScore;
+ }
+
+ /**
+ * Getter for the minimum score used to decide if results of geoname.org
+ * search results are added as EntityEnhancements to the ContentItem
+ *
+ * @return the minScore
+ */
+ public Double getMinScore() {
+ return minScore;
+ }
+
+ /**
+ * Setter for the minimum value used to decide based on the score of
+ * locations returned by the geonames.org web service if also the hierarchy
+ * of that point should be added as enhancements to the analysed
+ * content item. <br>
+ * If <code>null</code> is parsed the value is set to
+ * {@link LocationEnhancementEngine#DEFAULT_MIN_HIERARCHY_SCORE}.
+ * For values > 1 the value is set to 1 and for values < 0 the value
+ * is set to 0.
+ *
+ * @param minHierarchyScore the minHierarchyScore to set
+ */
+ public void setMinHierarchyScore(Double minHierarchyScore) {
+ if (minHierarchyScore == null) {
+ minHierarchyScore = DEFAULT_MIN_HIERARCHY_SCORE;
+ } else if (minHierarchyScore > 1) {
+ minHierarchyScore = Double.valueOf(1);
+ } else if (minScore < 0) {
+ minHierarchyScore = Double.valueOf(0);
+ }
+ this.minHierarchyScore = minHierarchyScore;
+ }
+
+ /**
+ * Setter for the minimum value used to decide based on the score of
+ * locations returned by the geonames.org web service if also the hierarchy
+ * of that point should be added as enhancements to the analysed
+ * content item.
+ *
+ * @return the minHierarchyScore
+ */
+ public Double getMinHierarchyScore() {
+ return minHierarchyScore;
+ }
}
Modified: incubator/stanbol/trunk/fise/engines/geonames/src/test/java/eu/iksproject/fise/engines/geonames/impl/TestLocationEnhancementEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/fise/engines/geonames/src/test/java/eu/iksproject/fise/engines/geonames/impl/TestLocationEnhancementEngine.java?rev=1044386&r1=1044385&r2=1044386&view=diff
==============================================================================
--- incubator/stanbol/trunk/fise/engines/geonames/src/test/java/eu/iksproject/fise/engines/geonames/impl/TestLocationEnhancementEngine.java (original)
+++ incubator/stanbol/trunk/fise/engines/geonames/src/test/java/eu/iksproject/fise/engines/geonames/impl/TestLocationEnhancementEngine.java Fri Dec 10 15:00:13 2010
@@ -36,19 +36,23 @@ import eu.iksproject.fise.servicesapi.rd
public class TestLocationEnhancementEngine {
private Logger log = LoggerFactory.getLogger(TestLocationEnhancementEngine.class);
+
/**
* The context for the tests (same as in TestOpenNLPEnhancementEngine)
*/
public static final String CONTEXT = "Dr. Patrick Marshall (1869 - November 1950) was a"
+ " geologist who lived in New Zealand and worked at the University of Otago.";
+
/**
* The person for the tests (same as in TestOpenNLPEnhancementEngine)
*/
public static final String PERSON = "Patrick Marshall";
+
/**
* The organisation for the tests (same as in TestOpenNLPEnhancementEngine)
*/
public static final String ORGANISATION ="University of Otago";
+
/**
* The place for the tests (same as in TestOpenNLPEnhancementEngine)
*/
@@ -56,12 +60,14 @@ public class TestLocationEnhancementEngi
static LocationEnhancementEngine locationEnhancementEngine = new LocationEnhancementEngine();
+
@BeforeClass
public static void setUpServices() throws IOException {
Dictionary<String, Object> properties = new Hashtable<String, Object>();
MockComponentContext context = new MockComponentContext(properties);
locationEnhancementEngine.activate(context);
}
+
@AfterClass
public static void shutdownServices() {
locationEnhancementEngine.deactivate(null);
@@ -114,6 +120,7 @@ public class TestLocationEnhancementEngi
testAnnotation.setStart(start);
testAnnotation.setEnd(start+name.length());
}
+
@Test
public void testLocationEnhancementEngine() {//throws Exception{
//create a content item
@@ -169,6 +176,7 @@ public class TestLocationEnhancementEngi
}
return entityAnnotationCount;
}
+
/**
* Checks if an entity annotation is valid
*
Modified: incubator/stanbol/trunk/fise/engines/langid/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/fise/engines/langid/pom.xml?rev=1044386&r1=1044385&r2=1044386&view=diff
==============================================================================
--- incubator/stanbol/trunk/fise/engines/langid/pom.xml (original)
+++ incubator/stanbol/trunk/fise/engines/langid/pom.xml Fri Dec 10 15:00:13 2010
@@ -1,113 +1,115 @@
<?xml version="1.0" encoding="UTF-8"?>
- <!--
- IKS FISE software is licensed under the Apache License, Version 2.0,
- see http://www.apache.org/licenses/LICENSE-2.0 Unless required by
- applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
- CONDITIONS OF ANY KIND, either express or implied. See the License for
- the specific language governing permissions and limitations under the
- License.
- -->
+ <!--
+ IKS FISE software is licensed under the Apache License, Version 2.0,
+ see http://www.apache.org/licenses/LICENSE-2.0 Unless required by
+ applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the
+ License.
+ -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
+ <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>eu.iksproject.fise.parent</artifactId>
- <groupId>eu.iksproject</groupId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>../../parent</relativePath>
- </parent>
-
- <groupId>eu.iksproject</groupId>
- <artifactId>eu.iksproject.fise.engines.langid</artifactId>
- <version>${fise-version}</version>
- <packaging>bundle</packaging>
-
- <name>IKS FISE Enhancement Engine : Language Identifier</name>
- <description>identifies a text as: German(de), English(en), French(fr), Spanish(es), Italian(it), Swedish(sv), Polish(pl), Dutch(nl), Norwegian(no), Finnish(fi), Albanian(sq), Slovakian(sk), Slovenian(sl), Danish(da), Hungarian(hu)</description>
-
- <inceptionYear>2010</inceptionYear>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.felix</groupId>
- <artifactId>maven-bundle-plugin</artifactId>
- <extensions>true</extensions>
- <configuration>
- <instructions>
- <Export-Package>
- eu.iksproject.fise.engines.langid;version=${project.version}
- </Export-Package>
- <Embed-Dependency>
- textcat
- </Embed-Dependency>
- </instructions>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.felix</groupId>
- <artifactId>maven-scr-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>eu.iksproject</groupId>
- <artifactId>eu.iksproject.fise.servicesapi</artifactId>
- <version>0.9-SNAPSHOT</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>net.sourceforge</groupId>
- <artifactId>textcat</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.felix</groupId>
- <artifactId>org.apache.felix.scr.annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.clerezza</groupId>
- <artifactId>org.apache.clerezza.rdf.core</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <repositories>
- <repository>
- <releases>
- <enabled>true</enabled>
- <updatePolicy>never</updatePolicy>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- <id>dfki-lt-repo</id>
- <name>DFKI LT Repository</name>
- <url>http://www.dfki.de/~kasper/IKS/maven</url>
- </repository>
- </repositories>
+ <parent>
+ <artifactId>eu.iksproject.fise.parent</artifactId>
+ <groupId>eu.iksproject</groupId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>../../parent</relativePath>
+ </parent>
+
+ <groupId>eu.iksproject</groupId>
+ <artifactId>eu.iksproject.fise.engines.langid</artifactId>
+ <version>${fise-version}</version>
+ <packaging>bundle</packaging>
+
+ <name>IKS FISE Enhancement Engine : Language Identifier</name>
+ <description>identifies a text as: German(de), English(en), French(fr), Spanish(es), Italian(it),
+ Swedish(sv), Polish(pl), Dutch(nl), Norwegian(no), Finnish(fi), Albanian(sq), Slovakian(sk),
+ Slovenian(sl), Danish(da), Hungarian(hu)</description>
+
+ <inceptionYear>2010</inceptionYear>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <Export-Package>
+ eu.iksproject.fise.engines.langid;version=${project.version}
+ </Export-Package>
+ <Embed-Dependency>
+ textcat
+ </Embed-Dependency>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-scr-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>eu.iksproject</groupId>
+ <artifactId>eu.iksproject.fise.servicesapi</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>net.sourceforge</groupId>
+ <artifactId>textcat</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>org.apache.felix.scr.annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.clerezza</groupId>
+ <artifactId>org.apache.clerezza.rdf.core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <repositories>
+ <repository>
+ <releases>
+ <enabled>true</enabled>
+ <updatePolicy>never</updatePolicy>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ <id>dfki-lt-repo</id>
+ <name>DFKI LT Repository</name>
+ <url>http://www.dfki.de/~kasper/IKS/maven</url>
+ </repository>
+ </repositories>
</project>
Modified: incubator/stanbol/trunk/fise/engines/langid/src/main/java/eu/iksproject/fise/engines/langid/LangIdEnhancementEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/fise/engines/langid/src/main/java/eu/iksproject/fise/engines/langid/LangIdEnhancementEngine.java?rev=1044386&r1=1044385&r2=1044386&view=diff
==============================================================================
--- incubator/stanbol/trunk/fise/engines/langid/src/main/java/eu/iksproject/fise/engines/langid/LangIdEnhancementEngine.java (original)
+++ incubator/stanbol/trunk/fise/engines/langid/src/main/java/eu/iksproject/fise/engines/langid/LangIdEnhancementEngine.java Fri Dec 10 15:00:13 2010
@@ -26,12 +26,14 @@ import eu.iksproject.fise.servicesapi.En
import eu.iksproject.fise.servicesapi.InvalidContentException;
import eu.iksproject.fise.servicesapi.ServiceProperties;
import eu.iksproject.fise.servicesapi.helper.EnhancementEngineHelper;
-import eu.iksproject.fise.servicesapi.rdf.Properties;
+
+import static eu.iksproject.fise.servicesapi.rdf.Properties.DC_LANGUAGE;
+import static eu.iksproject.fise.servicesapi.rdf.Properties.NIE_PLAINTEXTCONTENT;
/**
* {@link LangIdEnhancementEngine} provides functionality to enhance document
* with their language.
- *
+ *
* @author Joerg Steffen, DFKI
* @version $Id$
*/
@@ -39,11 +41,11 @@ import eu.iksproject.fise.servicesapi.rd
@Service
public class LangIdEnhancementEngine implements EnhancementEngine, ServiceProperties {
- /**
- * The default value for the Execution of this Engine. Currently set to
- * {@link ServiceProperties#ORDERING_PRE_PROCESSING}
- */
- public static final Integer defaultOrder = ServiceProperties.ORDERING_PRE_PROCESSING - 2;
+ /**
+ * The default value for the Execution of this Engine. Currently set to
+ * {@link ServiceProperties#ORDERING_PRE_PROCESSING}
+ */
+ public static final Integer defaultOrder = ORDERING_PRE_PROCESSING - 2;
/**
* This contains the only supported MIME type of this enhancement engine.
@@ -53,8 +55,7 @@ public class LangIdEnhancementEngine imp
/**
* This contains the logger.
*/
- private static Logger log =
- LoggerFactory.getLogger(LangIdEnhancementEngine.class);
+ private static final Logger log = LoggerFactory.getLogger(LangIdEnhancementEngine.class);
/**
* This contains the language identifier.
@@ -62,7 +63,7 @@ public class LangIdEnhancementEngine imp
private TextCategorizer languageIdentifier;
private static final int PROBE_LENGTH_DEFAULT = 400;
-
+
@Property
public static final String PROBE_LENGTH_PROP = "eu.iksproject.fise.engines.langid.probe-length";
@@ -70,123 +71,88 @@ public class LangIdEnhancementEngine imp
* How much text should be used for testing: If the value is 0 or smaller, the complete text will be used. Otherwise a text probe of the given length is taken from the middle of the text. The default length is 400 characters.
*/
private int probeLength = PROBE_LENGTH_DEFAULT;
-
+
/**
* The activate method.
- *
- * @param ce
- * the {@link ComponentContext}
+ *
+ * @param ce the {@link ComponentContext}
*/
protected void activate(@SuppressWarnings("unused") ComponentContext ce) {
- if (ce != null) {
- Dictionary<String,String> properties = ce.getProperties();
- String lengthVal = properties.get(PROBE_LENGTH_PROP);
- setProbeLength(lengthVal==null?PROBE_LENGTH_DEFAULT:Integer.parseInt(lengthVal));
- }
- this.languageIdentifier = new TextCategorizer();
+ if (ce != null) {
+ Dictionary<String, String> properties = ce.getProperties();
+ String lengthVal = properties.get(PROBE_LENGTH_PROP);
+ probeLength = lengthVal == null ? PROBE_LENGTH_DEFAULT : Integer.parseInt(lengthVal);
+ }
+ languageIdentifier = new TextCategorizer();
}
-
/**
* The deactivate method.
- *
- * @param ce
- * the {@link ComponentContext}
+ *
+ * @param ce the {@link ComponentContext}
*/
protected void deactivate(@SuppressWarnings("unused") ComponentContext ce) {
-
- this.languageIdentifier = null;
+ languageIdentifier = null;
}
-
- /**
- * {@inheritDoc}
- */
- public int canEnhance(ContentItem ci)
- throws EngineException {
- String mimeType = ci.getMimeType().split(";",2)[0];
+ public int canEnhance(ContentItem ci) throws EngineException {
+ String mimeType = ci.getMimeType().split(";", 2)[0];
if (TEXT_PLAIN_MIMETYPE.equalsIgnoreCase(mimeType)) {
return ENHANCE_SYNCHRONOUS;
}
- else {
- // TODO: check whether there is the graph contains the text
- UriRef subj = new UriRef(ci.getId());
- Iterator<Triple> it = ci.getMetadata().filter(subj, Properties.NIE_PLAINTEXTCONTENT, null);
- if (it.hasNext()) {
+
+ // TODO: check whether there is the graph contains the text
+ UriRef subj = new UriRef(ci.getId());
+ Iterator<Triple> it = ci.getMetadata().filter(subj, NIE_PLAINTEXTCONTENT, null);
+ if (it.hasNext()) {
return ENHANCE_SYNCHRONOUS;
- }
}
return CANNOT_ENHANCE;
}
-
- /**
- * {@inheritDoc}
- */
- public void computeEnhancements(ContentItem ci)
- throws EngineException {
-
+ public void computeEnhancements(ContentItem ci) throws EngineException {
String text = "";
if (TEXT_PLAIN_MIMETYPE.equals(ci.getMimeType())) {
- try {
- text = IOUtils.toString(ci.getStream());
- } catch (IOException e) {
- throw new InvalidContentException(this, ci, e);
- }
- }
- else {
- for (Iterator<Triple> it = ci.getMetadata().filter(new UriRef(ci.getId()), Properties.NIE_PLAINTEXTCONTENT, null); it.hasNext();) {
- text += it.next().getObject();
- }
+ try {
+ text = IOUtils.toString(ci.getStream());
+ } catch (IOException e) {
+ throw new InvalidContentException(this, ci, e);
+ }
+ } else {
+ Iterator<Triple> it = ci.getMetadata().filter(new UriRef(ci.getId()), NIE_PLAINTEXTCONTENT, null);
+ while (it.hasNext()) {
+ text += it.next().getObject();
+ }
}
if (text.trim().length() == 0) {
log.warn("no text found");
return;
}
- // truncate text to some piece from the middle if probeLenght > 0
- int checkLength = getProbeLength();
+ // truncate text to some piece from the middle if probeLength > 0
+ int checkLength = probeLength;
if (checkLength > 0 && text.length() > checkLength) {
- text =
- text.substring(text.length() / 2 - checkLength / 2, text
- .length()
- / 2
- + checkLength / 2);
+ text = text.substring(text.length() / 2 - checkLength / 2, text.length() / 2 + checkLength / 2);
}
- String language = this.languageIdentifier.categorize(text);
+ String language = languageIdentifier.categorize(text);
log.info("language identified as " + language);
-
+
// add language to metadata
MGraph g = ci.getMetadata();
- UriRef textEnhancement =
- EnhancementEngineHelper.createTextEnhancement(ci, this);
- g.add(
- new TripleImpl(
- textEnhancement, Properties.DC_LANGUAGE, new PlainLiteralImpl(
- language)));
-
+ UriRef textEnhancement = EnhancementEngineHelper.createTextEnhancement(ci, this);
+ g.add(new TripleImpl(textEnhancement, DC_LANGUAGE, new PlainLiteralImpl(language)));
}
- /**
- * @return the probeLength
- */
public int getProbeLength() {
- return probeLength;
+ return probeLength;
}
- /**
- * @param probeLength the probeLength to set
- */
public void setProbeLength(int probeLength) {
- this.probeLength = probeLength;
+ this.probeLength = probeLength;
}
-
public Map<String, Object> getServiceProperties() {
- return Collections.unmodifiableMap(Collections.singletonMap(
- ServiceProperties.ENHANCEMENT_ENGINE_ORDERING,
- (Object) defaultOrder));
+ return Collections.unmodifiableMap(Collections.singletonMap(ENHANCEMENT_ENGINE_ORDERING, (Object) defaultOrder));
}
-
-
+
}
Modified: incubator/stanbol/trunk/fise/engines/langid/src/test/java/eu/iksproject/fise/engines/langid/core/TextCatTest.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/fise/engines/langid/src/test/java/eu/iksproject/fise/engines/langid/core/TextCatTest.java?rev=1044386&r1=1044385&r2=1044386&view=diff
==============================================================================
--- incubator/stanbol/trunk/fise/engines/langid/src/test/java/eu/iksproject/fise/engines/langid/core/TextCatTest.java (original)
+++ incubator/stanbol/trunk/fise/engines/langid/src/test/java/eu/iksproject/fise/engines/langid/core/TextCatTest.java Fri Dec 10 15:00:13 2010
@@ -9,6 +9,9 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.knallgrau.utils.textcat.TextCategorizer;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
/**
* {@link TextCatTest} is a test class for {@link TextCategorizer}.
*
@@ -22,16 +25,13 @@ public class TextCatTest {
*/
private static TextCategorizer tc;
-
/**
* This initializes the text categorizer.
*/
@BeforeClass
public static void oneTimeSetUp() {
-
tc = new TextCategorizer();
}
-
/**
* This test the language identification.
@@ -41,14 +41,14 @@ public class TextCatTest {
*/
@Test
public void testTextCat() throws IOException {
-
String testFileName = "en.txt";
InputStream in =
this.getClass().getClassLoader().getResourceAsStream(
testFileName);
- Assert.assertNotNull("failed to load resource " + testFileName, in);
+ assertNotNull("failed to load resource " + testFileName, in);
String text = IOUtils.toString(in);
- Assert.assertEquals("en", tc.categorize(text));
+ assertEquals("en", tc.categorize(text));
}
+
}