You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2012/05/14 21:43:24 UTC
svn commit: r1338363 [1/2] - in /incubator/stanbol/trunk:
enhancer/engines/entitytagging/
enhancer/engines/entitytagging/src/main/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/
enhancer/engines/entitytagging/src/test/java/org/apache/stanb...
Author: rwesten
Date: Mon May 14 19:43:22 2012
New Revision: 1338363
URL: http://svn.apache.org/viewvc?rev=1338363&view=rev
Log:
STANBOL-612: UnitTest helper for the Stanbol Enhancement Structure
* added tests for TextAnnotations, EntityAnnotations and Enhancements mainly by collection/extending tests already done by unit tests of different engines
* applied tests to Engines
* re-enabled testing for the Named Entity Tagging Engine
* corrected two small issues with the Zemanta Engine (TypedLiteral for labels and missing selection context)
Missing:
* I would like to add tests for TopicEnhancements
Note: The added tests do break the the CELI NER engine (STANBOL-583) as of the latest patch. While I have (locally) already fixed some of the issues (missing selection-context, PlainLiterals for labels, xsd:int for start/end positions) there are still some remaining issues that might be XML entity and/or char encoding related (wrong selected texts, wrong start/end indexes).
Added:
incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/MockEntityhub.java (with props)
incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/TestDataFileProvider.java (with props)
incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/resources/
incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/resources/META-INF/
incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/resources/META-INF/services/
incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/resources/META-INF/services/org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider
incubator/stanbol/trunk/enhancer/generic/test/src/main/java/org/apache/stanbol/enhancer/test/helper/
incubator/stanbol/trunk/enhancer/generic/test/src/main/java/org/apache/stanbol/enhancer/test/helper/EnhancementStructureHelper.java (with props)
Removed:
incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/MockComponentContext.java
Modified:
incubator/stanbol/trunk/enhancer/engines/entitytagging/pom.xml
incubator/stanbol/trunk/enhancer/engines/entitytagging/src/main/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/NamedEntityTaggingEngine.java
incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/TestEntityLinkingEnhancementEngine.java
incubator/stanbol/trunk/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/TestTaxonomyLinker.java
incubator/stanbol/trunk/enhancer/engines/opencalais/pom.xml
incubator/stanbol/trunk/enhancer/engines/opencalais/src/test/java/org/apache/stanbol/enhancer/engines/opencalais/impl/TestOpenCalaisEngine.java
incubator/stanbol/trunk/enhancer/engines/opennlp-ner/pom.xml
incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java
incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/test/java/org/apache/stanbol/enhancer/engines/opennlp/impl/TestNamedEntityExtractionEnhancementEngine.java
incubator/stanbol/trunk/enhancer/engines/zemanta/pom.xml
incubator/stanbol/trunk/enhancer/engines/zemanta/src/main/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngine.java
incubator/stanbol/trunk/enhancer/engines/zemanta/src/test/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngineTest.java
incubator/stanbol/trunk/enhancer/generic/test/pom.xml
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexerFactory.java
Modified: incubator/stanbol/trunk/enhancer/engines/entitytagging/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/entitytagging/pom.xml?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/entitytagging/pom.xml (original)
+++ incubator/stanbol/trunk/enhancer/engines/entitytagging/pom.xml Mon May 14 19:43:22 2012
@@ -124,12 +124,36 @@
<!-- Testing -->
<dependency>
<groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.test</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.enhancer.core</artifactId>
<version>0.10.0-incubating-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.entityhub.core</artifactId>
+ <version>0.9.0-incubating</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.entityhub.yard.solr</artifactId>
+ <version>0.9.0-incubating</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.data.sites.dbpedia</artifactId>
+ <version>1.0.4-incubating-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.enhancer.rdfentities</artifactId>
<version>0.10.0-incubating-SNAPSHOT</version>
<scope>test</scope>
Modified: incubator/stanbol/trunk/enhancer/engines/entitytagging/src/main/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/NamedEntityTaggingEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/entitytagging/src/main/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/NamedEntityTaggingEngine.java?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/entitytagging/src/main/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/NamedEntityTaggingEngine.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/entitytagging/src/main/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/NamedEntityTaggingEngine.java Mon May 14 19:43:22 2012
@@ -191,14 +191,14 @@ public class NamedEntityTaggingEngine
/**
* The field used to search for the selected text of the TextAnnotation.
*/
- private String nameField;
+ protected String nameField;
/**
* The number of Suggestions to be added
*/
- public Integer numSuggestions = 3;
+ protected Integer numSuggestions = 3;
- public boolean dereferenceEntities = true;
+ protected boolean dereferenceEntities = true;
/**
* The {@link OfflineMode} is used by Stanbol to indicate that no external service should be referenced.
Added: incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/MockEntityhub.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/MockEntityhub.java?rev=1338363&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/MockEntityhub.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/MockEntityhub.java Mon May 14 19:43:22 2012
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitytagging.impl;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+
+import org.apache.stanbol.entityhub.core.model.EntityImpl;
+import org.apache.stanbol.entityhub.core.query.DefaultQueryFactory;
+import org.apache.stanbol.entityhub.core.query.QueryResultListImpl;
+import org.apache.stanbol.entityhub.servicesapi.Entityhub;
+import org.apache.stanbol.entityhub.servicesapi.EntityhubConfiguration;
+import org.apache.stanbol.entityhub.servicesapi.EntityhubException;
+import org.apache.stanbol.entityhub.servicesapi.mapping.FieldMapper;
+import org.apache.stanbol.entityhub.servicesapi.model.Entity;
+import org.apache.stanbol.entityhub.servicesapi.model.ManagedEntityState;
+import org.apache.stanbol.entityhub.servicesapi.model.Representation;
+import org.apache.stanbol.entityhub.servicesapi.query.FieldQuery;
+import org.apache.stanbol.entityhub.servicesapi.query.FieldQueryFactory;
+import org.apache.stanbol.entityhub.servicesapi.query.QueryResultList;
+import org.apache.stanbol.entityhub.servicesapi.yard.Yard;
+import org.apache.stanbol.entityhub.servicesapi.yard.YardException;
+import org.apache.stanbol.entityhub.yard.solr.impl.SolrYard;
+import org.apache.stanbol.entityhub.yard.solr.impl.SolrYardConfig;
+
+/**
+ * Mocks an Entityhub for the {@link NamedEntityTaggingEngine} for Unit Testing<p>
+ * This loads the dbpedia default data and wraps it as entityhub.
+ * @author Rupert Westenthaler
+ *
+ */
+class MockEntityhub implements Entityhub {
+
+ protected SolrYard yard;
+
+ protected MockEntityhub(){
+ SolrYardConfig config = new SolrYardConfig("dbpedia", "dbpedia_43k");
+ try {
+ yard = new SolrYard(config);
+ } catch (YardException e) {
+ throw new IllegalStateException("Unable to init Yard!",e);
+ }
+ }
+ /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Only the following two Methods are used by the EntityLinkingengine
+ * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ */
+
+ @Override
+ public FieldQueryFactory getQueryFactory() {
+ return DefaultQueryFactory.getInstance();
+ }
+ @Override
+ public QueryResultList<Entity> findEntities(FieldQuery query) throws EntityhubException {
+ QueryResultList<Representation> results = yard.findRepresentation(query);
+ Collection<Entity> entities = new ArrayList<Entity>(results.size());
+ for(Representation r : results){
+ entities.add(new EntityImpl("dbpedia", r, null));
+ }
+ return new QueryResultListImpl<Entity>(results.getQuery(),entities,Entity.class);
+ }
+
+ // UNUSED METHODS
+
+ @Override
+ public Yard getYard() {
+ return yard;
+ }
+
+ @Override
+ public Entity lookupLocalEntity(String reference) throws EntityhubException {
+ return null;
+ }
+
+ @Override
+ public Entity lookupLocalEntity(String reference, boolean create) throws IllegalArgumentException,
+ EntityhubException {
+ return null;
+ }
+
+ @Override
+ public Entity getEntity(String entityId) throws IllegalArgumentException, EntityhubException {
+ return null;
+ }
+
+ @Override
+ public Entity importEntity(String reference) throws IllegalStateException,
+ IllegalArgumentException,
+ EntityhubException {
+ return null;
+ }
+
+ @Override
+ public Entity getMappingById(String id) throws EntityhubException, IllegalArgumentException {
+ return null;
+ }
+
+ @Override
+ public Entity getMappingBySource(String source) throws EntityhubException {
+ return null;
+ }
+
+ @Override
+ public FieldMapper getFieldMappings() {
+ return null;
+ }
+
+ @Override
+ public Collection<Entity> getMappingsByTarget(String entityId) throws EntityhubException {
+ return null;
+ }
+
+ @Override
+ public QueryResultList<String> findEntityReferences(FieldQuery query) throws EntityhubException {
+ return null;
+ }
+
+ @Override
+ public QueryResultList<Representation> find(FieldQuery query) throws EntityhubException {
+ return null;
+ }
+
+ @Override
+ public boolean isRepresentation(String id) throws EntityhubException, IllegalArgumentException {
+ return false;
+ }
+
+ @Override
+ public Entity store(Representation representation) throws EntityhubException, IllegalArgumentException {
+ return null;
+ }
+
+ @Override
+ public Entity delete(String id) throws EntityhubException, IllegalArgumentException {
+ return null;
+ }
+
+ @Override
+ public Entity setState(String id, ManagedEntityState state) throws EntityhubException,
+ IllegalArgumentException {
+ return null;
+ }
+
+ @Override
+ public EntityhubConfiguration getConfig() {
+ return null;
+ }
+
+}
Propchange: incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/MockEntityhub.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/TestDataFileProvider.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/TestDataFileProvider.java?rev=1338363&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/TestDataFileProvider.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/TestDataFileProvider.java Mon May 14 19:43:22 2012
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitytagging.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Map;
+import java.util.ServiceLoader;
+
+import org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider;
+
+/**
+ *
+ * The Standalone implementation of the ManagedSolrServer uses
+ * {@link ServiceLoader} to search for {@link DataFileProvider}.
+ * This implementation ensures that the DBpedia default data index can be loaded
+ * and initialised by the StandaloneManagedSolrServer.
+ *
+ * @author Rupert Westenthaler
+ *
+ */
+public class TestDataFileProvider implements DataFileProvider {
+
+ private static String DBPEDIA_PREFIX = "org/apache/stanbol/data/site/dbpedia/default/index/";
+
+ @Override
+ public InputStream getInputStream(String bundleSymbolicName, String filename, Map<String,String> comments) throws IOException {
+ String resource = DBPEDIA_PREFIX+filename;
+ URL resourceUri = lookupResource(resource);
+ if(resourceUri != null){
+ return resourceUri.openStream();
+ } else {
+ throw new IOException("Resource '"+resource+"' not found");
+ }
+ }
+
+ @Override
+ public boolean isAvailable(String bundleSymbolicName, String filename, Map<String,String> comments) {
+ return lookupResource(DBPEDIA_PREFIX+filename) != null;
+ }
+ /**
+ * @param resource
+ * @return
+ */
+ private URL lookupResource(String resource) {
+ ClassLoader cl = TestDataFileProvider.class.getClassLoader();
+ URL resourceUri = cl.getResource(resource);
+ if(resourceUri == null){
+ cl = Thread.currentThread().getContextClassLoader();
+ resourceUri = cl.getResource(resource);
+ }
+ if(resourceUri == null){
+ resourceUri = ClassLoader.getSystemResource(resource);
+ }
+ return resourceUri;
+ }
+
+}
Propchange: incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/TestDataFileProvider.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/TestEntityLinkingEnhancementEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/TestEntityLinkingEnhancementEngine.java?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/TestEntityLinkingEnhancementEngine.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/java/org/apache/stanbol/enhancer/engines/entitytagging/impl/TestEntityLinkingEnhancementEngine.java Mon May 14 19:43:22 2012
@@ -16,20 +16,32 @@
*/
package org.apache.stanbol.enhancer.engines.entitytagging.impl;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.OntologicalClasses.DBPEDIA_ORGANISATION;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.OntologicalClasses.DBPEDIA_PERSON;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.OntologicalClasses.DBPEDIA_PLACE;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_CREATOR;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_RELATION;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_LABEL;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_REFERENCE;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_EXTRACTED_FROM;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.RDF_TYPE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_ENTITYANNOTATION;
import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_TEXTANNOTATION;
+import static org.apache.stanbol.enhancer.test.helper.EnhancementStructureHelper.validateAllEntityAnnotations;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.io.File;
import java.io.IOException;
import java.util.Date;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
+import org.apache.clerezza.rdf.core.LiteralFactory;
import org.apache.clerezza.rdf.core.MGraph;
+import org.apache.clerezza.rdf.core.Resource;
import org.apache.clerezza.rdf.core.Triple;
import org.apache.clerezza.rdf.core.UriRef;
import org.apache.commons.io.IOUtils;
@@ -39,8 +51,12 @@ import org.apache.stanbol.enhancer.rdfen
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
import org.apache.stanbol.enhancer.servicesapi.ContentItemFactory;
import org.apache.stanbol.enhancer.servicesapi.impl.StringSource;
+import org.apache.stanbol.enhancer.servicesapi.rdf.OntologicalClasses;
+import org.apache.stanbol.enhancer.servicesapi.rdf.Properties;
+import org.apache.stanbol.enhancer.test.helper.EnhancementStructureHelper;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -68,11 +84,34 @@ public class TestEntityLinkingEnhancemen
private static final ContentItemFactory ciFactory = InMemoryContentItemFactory.getInstance();
- static NamedEntityTaggingEngine entityLinkingEngine
- = new NamedEntityTaggingEngine();
+ static NamedEntityTaggingEngine entityLinkingEngine;
+ private static String userDir = System.getProperty("user.dir");
+
@BeforeClass
public static void setUpServices() throws IOException {
+ //TODO: set user.dir to /target/test-files
+ File testFiles = new File("./target/test-files");
+ if(!testFiles.isDirectory()){
+ if(!testFiles.mkdirs()){
+ throw new IOException("Unable to create directory for test files "+testFiles);
+ }
+ }
+
+ System.getProperties().setProperty("user.dir", testFiles.getCanonicalPath());
+ entityLinkingEngine = new NamedEntityTaggingEngine();
+ //instead of calling activate we directly set the required fields
+ //we need a data source for linking
+ entityLinkingEngine.entityhub = new MockEntityhub();
+ entityLinkingEngine.personState = true;
+ entityLinkingEngine.personType = OntologicalClasses.DBPEDIA_PERSON.getUnicodeString();
+ entityLinkingEngine.orgState = true;
+ entityLinkingEngine.orgType = OntologicalClasses.DBPEDIA_ORGANISATION.getUnicodeString();
+ entityLinkingEngine.placeState = true;
+ entityLinkingEngine.placeType = OntologicalClasses.DBPEDIA_PLACE.getUnicodeString();
+ entityLinkingEngine.nameField = Properties.RDFS_LABEL.getUnicodeString();
+ //not implemented
+ entityLinkingEngine.dereferenceEntities = false;
}
@Before
@@ -85,6 +124,7 @@ public class TestEntityLinkingEnhancemen
@AfterClass
public static void shutdownServices() {
+ System.getProperties().setProperty("user.dir", userDir);
}
public static ContentItem getContentItem(final String id, final String text) throws IOException {
@@ -118,68 +158,22 @@ public class TestEntityLinkingEnhancemen
@Test
public void testEntityLinkingEnhancementEngine() throws Exception{
- //TODO: adapt this test to work with this engine
- // -> here the problem is mainly to fake the needed infrastructure
- return;
-// //create a content item
-// ContentItem ci = getContentItem("urn:iks-project:enhancer:text:content-item:person", CONTEXT);
-// //add three text annotations to be consumed by this test
-// getTextAnnotation(ci, PERSON, CONTEXT, OntologicalClasses.DBPEDIA_PERSON);
-// getTextAnnotation(ci, ORGANISATION, CONTEXT, OntologicalClasses.DBPEDIA_ORGANISATION);
-// getTextAnnotation(ci, PLACE, CONTEXT, OntologicalClasses.DBPEDIA_PLACE);
-// //perform the computation of the enhancements
-// entityLinkingEngine.computeEnhancements(ci);
-// int entityAnnotationCount = checkAllEntityAnnotations(ci.getMetadata());
-// assertEquals(2, entityAnnotationCount);
- }
-
- /*
- * -----------------------------------------------------------------------
- * Helper Methods to check Text and EntityAnnotations
- * -----------------------------------------------------------------------
- */
-
- private int checkAllEntityAnnotations(MGraph g) {
- Iterator<Triple> entityAnnotationIterator = g.filter(null,
- RDF_TYPE, ENHANCER_ENTITYANNOTATION);
- int entityAnnotationCount = 0;
- while (entityAnnotationIterator.hasNext()) {
- UriRef entityAnnotation = (UriRef) entityAnnotationIterator.next().getSubject();
- // test if selected Text is added
- checkEntityAnnotation(g, entityAnnotation);
- entityAnnotationCount++;
- }
- return entityAnnotationCount;
+ //create a content item
+ ContentItem ci = getContentItem("urn:iks-project:enhancer:text:content-item:person", CONTEXT);
+ //add three text annotations to be consumed by this test
+ getTextAnnotation(ci, PERSON, CONTEXT, DBPEDIA_PERSON);
+ getTextAnnotation(ci, ORGANISATION, CONTEXT, DBPEDIA_ORGANISATION);
+ getTextAnnotation(ci, PLACE, CONTEXT, DBPEDIA_PLACE);
+ //perform the computation of the enhancements
+ entityLinkingEngine.computeEnhancements(ci);
+ Map<UriRef,Resource> expectedValues = new HashMap<UriRef,Resource>();
+ expectedValues.put(ENHANCER_EXTRACTED_FROM, ci.getUri());
+ expectedValues.put(DC_CREATOR,LiteralFactory.getInstance().createTypedLiteral(
+ entityLinkingEngine.getClass().getName()));
+ int entityAnnotationCount = validateAllEntityAnnotations(ci.getMetadata(),expectedValues);
+ assertEquals(3, entityAnnotationCount);
}
- /**
- * Checks if an entity annotation is valid.
- */
- private void checkEntityAnnotation(MGraph g, UriRef entityAnnotation) {
- Iterator<Triple> relationToTextAnnotationIterator = g.filter(
- entityAnnotation, DC_RELATION, null);
- // check if the relation to the text annotation is set
- assertTrue(relationToTextAnnotationIterator.hasNext());
- while (relationToTextAnnotationIterator.hasNext()) {
- // test if the referred annotations are text annotations
- UriRef referredTextAnnotation = (UriRef) relationToTextAnnotationIterator.next().getObject();
- assertTrue(g.filter(referredTextAnnotation, RDF_TYPE,
- ENHANCER_TEXTANNOTATION).hasNext());
- }
- // test if an entity is referred
- Iterator<Triple> entityReferenceIterator = g.filter(entityAnnotation,
- ENHANCER_ENTITY_REFERENCE, null);
- assertTrue(entityReferenceIterator.hasNext());
- // test if the reference is an URI
- assertTrue(entityReferenceIterator.next().getObject() instanceof UriRef);
- // test if there is only one entity referred
- assertFalse(entityReferenceIterator.hasNext());
-
- // finally test if the entity label is set
- Iterator<Triple> entityLabelIterator = g.filter(entityAnnotation,
- ENHANCER_ENTITY_LABEL, null);
- assertTrue(entityLabelIterator.hasNext());
- }
}
Added: incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/resources/META-INF/services/org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/resources/META-INF/services/org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider?rev=1338363&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/resources/META-INF/services/org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider (added)
+++ incubator/stanbol/trunk/enhancer/engines/entitytagging/src/test/resources/META-INF/services/org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider Mon May 14 19:43:22 2012
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.stanbol.enhancer.engines.entitytagging.impl.TestDataFileProvider
\ No newline at end of file
Modified: incubator/stanbol/trunk/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/TestTaxonomyLinker.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/TestTaxonomyLinker.java?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/TestTaxonomyLinker.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/TestTaxonomyLinker.java Mon May 14 19:43:22 2012
@@ -134,7 +134,7 @@ public class TestTaxonomyLinker {
}
@Test
- public void testTaxonomyLinker() throws Exception{
+ public void testTaxonomyLinker() throws Exception {
OpenNlpAnalysedContentFactory acf = OpenNlpAnalysedContentFactory.getInstance(openNLP,
new TextAnalyzerConfig());
EntityLinkerConfig config = new EntityLinkerConfig();
Modified: incubator/stanbol/trunk/enhancer/engines/opencalais/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/opencalais/pom.xml?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/opencalais/pom.xml (original)
+++ incubator/stanbol/trunk/enhancer/engines/opencalais/pom.xml Mon May 14 19:43:22 2012
@@ -130,6 +130,12 @@
<!-- Test dependencies -->
<dependency>
<groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.test</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.enhancer.core</artifactId>
<version>0.10.0-incubating-SNAPSHOT</version>
<scope>test</scope>
Modified: incubator/stanbol/trunk/enhancer/engines/opencalais/src/test/java/org/apache/stanbol/enhancer/engines/opencalais/impl/TestOpenCalaisEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/opencalais/src/test/java/org/apache/stanbol/enhancer/engines/opencalais/impl/TestOpenCalaisEngine.java?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/opencalais/src/test/java/org/apache/stanbol/enhancer/engines/opencalais/impl/TestOpenCalaisEngine.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/opencalais/src/test/java/org/apache/stanbol/enhancer/engines/opencalais/impl/TestOpenCalaisEngine.java Mon May 14 19:43:22 2012
@@ -16,13 +16,18 @@
*/
package org.apache.stanbol.enhancer.engines.opencalais.impl;
+import static org.apache.stanbol.enhancer.test.helper.EnhancementStructureHelper.validateAllEntityAnnotations;
+import static org.apache.stanbol.enhancer.test.helper.EnhancementStructureHelper.validateAllTextAnnotations;
+
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.HashMap;
+import java.util.Map;
import org.apache.clerezza.rdf.core.LiteralFactory;
import org.apache.clerezza.rdf.core.MGraph;
+import org.apache.clerezza.rdf.core.Resource;
import org.apache.clerezza.rdf.core.UriRef;
import org.apache.clerezza.rdf.core.access.TcManager;
import org.apache.clerezza.rdf.core.impl.TripleImpl;
@@ -32,6 +37,7 @@ import org.apache.stanbol.enhancer.servi
import org.apache.stanbol.enhancer.servicesapi.EngineException;
import org.apache.stanbol.enhancer.servicesapi.impl.StringSource;
import org.apache.stanbol.enhancer.servicesapi.rdf.Properties;
+import org.apache.stanbol.enhancer.test.helper.EnhancementStructureHelper;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.BeforeClass;
@@ -40,73 +46,94 @@ import org.osgi.service.cm.Configuration
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
-/** This class provides JUnit tests for OpenCalaisEngine.
+/**
+ * This class provides JUnit tests for OpenCalaisEngine.
*
* @author <a href="mailto:kasper@dfki.de">Walter Kasper</a>
*/
public class TestOpenCalaisEngine {
- /**
- * This contains the logger.
- */
- private static final Logger LOG = LoggerFactory.getLogger(TestOpenCalaisEngine.class);
-
- private static OpenCalaisEngine calaisExtractor;
-
- private static final ContentItemFactory ciFactory = InMemoryContentItemFactory.getInstance();
- private static String TEST_LICENSE_KEY = System.getProperty(OpenCalaisEngine.LICENSE_KEY);
- private static String TEST_TEXT = "Israeli PM Netanyahu pulls out of US nuclear summit\nIsraeli PM Benjamin Netanyahu has cancelled a visit to the US where he was to attend a summit on nuclear security, Israeli officials say. Mr Netanyahu made the decision after learning that Egypt and Turkey intended to raise the issue of Israel's presumed nuclear arsenal, the officials said. Mr Obama is due to host dozens of world leaders at the two-day conference, which begins in Washington on Monday. Israel has never confirmed or denied that it possesses atomic weapons. Israel's Intelligence and Atomic Energy Minister Dan Meridor will take Netanyahu's place in the nuclear summit, Israeli radio said. More than 40 countries are expected at the meeting, which will focus on preventing the spread of nuclear weapons to militant groups.";
-
- @BeforeClass
- public static void oneTimeSetup() throws ConfigurationException {
- calaisExtractor = new OpenCalaisEngine();
- calaisExtractor.setCalaisTypeMap(new HashMap<UriRef,UriRef>());
- calaisExtractor.tcManager = TcManager.getInstance();
- if (TEST_LICENSE_KEY != null && TEST_LICENSE_KEY.matches("\\w+")) {
- calaisExtractor.setLicenseKey(TEST_LICENSE_KEY);
+ /**
+ * This contains the logger.
+ */
+ private static final Logger LOG = LoggerFactory.getLogger(TestOpenCalaisEngine.class);
+
+ private static OpenCalaisEngine calaisExtractor;
+
+ private static final ContentItemFactory ciFactory = InMemoryContentItemFactory.getInstance();
+ private static String TEST_LICENSE_KEY = System.getProperty(OpenCalaisEngine.LICENSE_KEY);
+ private static String TEST_TEXT = "Israeli PM Netanyahu pulls out of US nuclear summit\n"
+ + "Israeli PM Benjamin Netanyahu has cancelled a visit to the US where he was to "
+ + "attend a summit on nuclear security, Israeli officials say. Mr Netanyahu made "
+ + "the decision after learning that Egypt and Turkey intended to raise the issue "
+ + "of Israel's presumed nuclear arsenal, the officials said. Mr Obama is due to "
+ + "host dozens of world leaders at the two-day conference, which begins in "
+ + "Washington on Monday. Israel has never confirmed or denied that it possesses "
+ + "atomic weapons. Israel's Intelligence and Atomic Energy Minister Dan Meridor "
+ + "will take Netanyahu's place in the nuclear summit, Israeli radio said. More "
+ + "than 40 countries are expected at the meeting, which will focus on preventing "
+ + "the spread of nuclear weapons to militant groups.";
+
+ @BeforeClass
+ public static void oneTimeSetup() throws ConfigurationException {
+ calaisExtractor = new OpenCalaisEngine();
+ calaisExtractor.setCalaisTypeMap(new HashMap<UriRef,UriRef>());
+ calaisExtractor.tcManager = TcManager.getInstance();
+ if (TEST_LICENSE_KEY != null && TEST_LICENSE_KEY.matches("\\w+")) {
+ calaisExtractor.setLicenseKey(TEST_LICENSE_KEY);
+ }
}
- }
- public static ContentItem wrapAsContentItem(final String text) throws IOException {
- return ciFactory.createContentItem(new StringSource(text));
- }
-
- @Test
- public void testEntityExtraction() {
- String testFile = "calaisresult.owl";
- String format = "application/rdf+xml";
- InputStream in =
- this.getClass().getClassLoader().getResourceAsStream(
- testFile);
- Assert.assertNotNull("failed to load resource " + testFile, in);
- MGraph model = calaisExtractor.readModel(in, format);
- Assert.assertNotNull("model reader failed with format: "+format, model);
- Collection<CalaisEntityOccurrence> entities = calaisExtractor.queryModel(model);
- LOG.info("Found entities: {}",entities.size());
- LOG.debug("Entities:\n{}",entities);
- Assert.assertFalse("No entities found!", entities.isEmpty());
- }
-
- @Test
- public void testCalaisConnection() throws IOException {
- Assume.assumeNotNull(calaisExtractor.getLicenseKey());
- try {
- ContentItem ci = wrapAsContentItem(TEST_TEXT);
- ci.getMetadata().add(new TripleImpl(ci.getUri(), Properties.DC_LANGUAGE, LiteralFactory.getInstance().createTypedLiteral("en")));
- MGraph model = calaisExtractor.getCalaisAnalysis(TEST_TEXT,"text/plain");
- Assert.assertNotNull("No model", model);
- Collection<CalaisEntityOccurrence> entities = calaisExtractor.queryModel(model);
- LOG.info("Found entities: {}",entities.size());
- LOG.debug("Entities:\n{}",entities);
- Assert.assertFalse("No entities found!", entities.isEmpty());
- } catch (EngineException e) {
- Assert.fail("Connection problem: "+e.getMessage());
- }
- }
-
- // problem with license keys for testing?
- // ask user to supply it as system property or whatever?
-
+ public static ContentItem wrapAsContentItem(final String text) throws IOException {
+ return ciFactory.createContentItem(new StringSource(text));
+ }
+
+ @Test
+ public void testEntityExtraction() throws IOException {
+ String testFile = "calaisresult.owl";
+ String format = "application/rdf+xml";
+ InputStream in = this.getClass().getClassLoader().getResourceAsStream(testFile);
+ Assert.assertNotNull("failed to load resource " + testFile, in);
+ MGraph model = calaisExtractor.readModel(in, format);
+ Assert.assertNotNull("model reader failed with format: " + format, model);
+ Collection<CalaisEntityOccurrence> entities = calaisExtractor.queryModel(model);
+ LOG.info("Found entities: {}", entities.size());
+ LOG.debug("Entities:\n{}", entities);
+ Assert.assertFalse("No entities found!", entities.isEmpty());
+ //test the generation of the Enhancements
+ ContentItem ci = wrapAsContentItem(TEST_TEXT);
+ calaisExtractor.createEnhancements(entities, ci);
+ Map<UriRef,Resource> expectedValues = new HashMap<UriRef,Resource>();
+ expectedValues.put(Properties.ENHANCER_EXTRACTED_FROM, ci.getUri());
+ expectedValues.put(Properties.DC_CREATOR,
+ LiteralFactory.getInstance().createTypedLiteral(
+ calaisExtractor.getClass().getName()));
+ validateAllTextAnnotations(ci.getMetadata(),
+ TEST_TEXT, expectedValues);
+ validateAllEntityAnnotations(ci.getMetadata(), expectedValues);
+ }
+
+ @Test
+ public void testCalaisConnection() throws IOException {
+ Assume.assumeNotNull(calaisExtractor.getLicenseKey());
+ try {
+ ContentItem ci = wrapAsContentItem(TEST_TEXT);
+ ci.getMetadata().add(
+ new TripleImpl(ci.getUri(), Properties.DC_LANGUAGE, LiteralFactory.getInstance()
+ .createTypedLiteral("en")));
+ MGraph model = calaisExtractor.getCalaisAnalysis(TEST_TEXT, "text/plain");
+ Assert.assertNotNull("No model", model);
+ Collection<CalaisEntityOccurrence> entities = calaisExtractor.queryModel(model);
+ LOG.info("Found entities: {}", entities.size());
+ LOG.debug("Entities:\n{}", entities);
+ Assert.assertFalse("No entities found!", entities.isEmpty());
+ } catch (EngineException e) {
+ Assert.fail("Connection problem: " + e.getMessage());
+ }
+ }
+
+ // problem with license keys for testing?
+ // ask user to supply it as system property or whatever?
+
}
Modified: incubator/stanbol/trunk/enhancer/engines/opennlp-ner/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/opennlp-ner/pom.xml?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/opennlp-ner/pom.xml (original)
+++ incubator/stanbol/trunk/enhancer/engines/opennlp-ner/pom.xml Mon May 14 19:43:22 2012
@@ -123,6 +123,12 @@
<!-- for tests -->
<dependency>
<groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.test</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.enhancer.core</artifactId>
<version>0.10.0-incubating-SNAPSHOT</version>
<scope>test</scope>
Modified: incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/main/java/org/apache/stanbol/enhancer/engines/opennlp/impl/NEREngineCore.java Mon May 14 19:43:22 2012
@@ -494,9 +494,9 @@ public class NEREngineCore implements En
return defaultLang;
}
} else {
- log.warn("Unable to extract language for ContentItem %s! Is the %s active?",
+ log.info("Unable to extract language for ContentItem {}! Is the {} active?",
ci.getUri().getUnicodeString(),LANG_ID_ENGINE_NAME.getLexicalForm());
- log.warn(" ... return '{}' as default",defaultLang);
+ log.info(" ... return '{}' as default",defaultLang);
return defaultLang;
}
}
Modified: incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/test/java/org/apache/stanbol/enhancer/engines/opennlp/impl/TestNamedEntityExtractionEnhancementEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/test/java/org/apache/stanbol/enhancer/engines/opennlp/impl/TestNamedEntityExtractionEnhancementEngine.java?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/test/java/org/apache/stanbol/enhancer/engines/opennlp/impl/TestNamedEntityExtractionEnhancementEngine.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/opennlp-ner/src/test/java/org/apache/stanbol/enhancer/engines/opennlp/impl/TestNamedEntityExtractionEnhancementEngine.java Mon May 14 19:43:22 2012
@@ -16,33 +16,25 @@
*/
package org.apache.stanbol.enhancer.engines.opennlp.impl;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_END;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTED_TEXT;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTION_CONTEXT;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_START;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.RDF_TYPE;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_TEXTANNOTATION;
+import static org.apache.stanbol.enhancer.test.helper.EnhancementStructureHelper.validateAllTextAnnotations;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
-import java.util.Iterator;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.clerezza.rdf.core.Literal;
import org.apache.clerezza.rdf.core.LiteralFactory;
import org.apache.clerezza.rdf.core.MGraph;
import org.apache.clerezza.rdf.core.Resource;
-import org.apache.clerezza.rdf.core.Triple;
-import org.apache.clerezza.rdf.core.TypedLiteral;
import org.apache.clerezza.rdf.core.UriRef;
-import org.apache.stanbol.enhancer.contentitem.inmemory.InMemoryContentItem;
import org.apache.stanbol.enhancer.contentitem.inmemory.InMemoryContentItemFactory;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
import org.apache.stanbol.enhancer.servicesapi.ContentItemFactory;
import org.apache.stanbol.enhancer.servicesapi.EngineException;
import org.apache.stanbol.enhancer.servicesapi.impl.StringSource;
+import org.apache.stanbol.enhancer.servicesapi.rdf.Properties;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -134,91 +126,12 @@ public class TestNamedEntityExtractionEn
throws EngineException, IOException {
ContentItem ci = wrapAsContentItem("my doc id", SINGLE_SENTENCE);
nerEngine.computeEnhancements(ci);
+ Map<UriRef,Resource> expectedValues = new HashMap<UriRef,Resource>();
+ expectedValues.put(Properties.ENHANCER_EXTRACTED_FROM, ci.getUri());
+ expectedValues.put(Properties.DC_CREATOR, LiteralFactory.getInstance().createTypedLiteral(nerEngine.getClass().getName()));
MGraph g = ci.getMetadata();
- int textAnnotationCount = checkAllTextAnnotations(g,SINGLE_SENTENCE);
+ int textAnnotationCount = validateAllTextAnnotations(g,SINGLE_SENTENCE,expectedValues);
assertEquals(3, textAnnotationCount);
-
- //This Engine dose no longer create entityAnnotations
-// int entityAnnotationCount = checkAllEntityAnnotations(g);
-// assertEquals(2, entityAnnotationCount);
- }
-
- /*
- * -----------------------------------------------------------------------
- * Helper Methods to check Text and EntityAnnotations
- * -----------------------------------------------------------------------
- */
-
- private int checkAllTextAnnotations(MGraph g, String content) {
- Iterator<Triple> textAnnotationIterator = g.filter(null,
- RDF_TYPE, ENHANCER_TEXTANNOTATION);
- // test if a textAnnotation is present
- assertTrue(textAnnotationIterator.hasNext());
- int textAnnotationCount = 0;
- while (textAnnotationIterator.hasNext()) {
- UriRef textAnnotation = (UriRef) textAnnotationIterator.next().getSubject();
- // test if selected Text is added
- checkTextAnnotation(g, textAnnotation,content);
- textAnnotationCount++;
- }
- return textAnnotationCount;
- }
-
- /**
- * Checks if a text annotation is valid
- */
- private void checkTextAnnotation(MGraph g, UriRef textAnnotation, String content) {
- Iterator<Triple> selectedTextIterator = g.filter(textAnnotation,
- ENHANCER_SELECTED_TEXT, null);
- // check if the selected text is added
- assertTrue(selectedTextIterator.hasNext());
- // test if the selected text is part of the TEXT_TO_TEST
- Resource object = selectedTextIterator.next().getObject();
- assertTrue(object instanceof Literal);
- Literal selectedText = (Literal)object;
- object = null;
- assertTrue(SINGLE_SENTENCE.contains(selectedText.getLexicalForm()));
- // test if context is added
- Iterator<Triple> selectionContextIterator = g.filter(textAnnotation,
- ENHANCER_SELECTION_CONTEXT, null);
- assertTrue(selectionContextIterator.hasNext());
- // test if the selected text is part of the TEXT_TO_TEST
- object = selectionContextIterator.next().getObject();
- assertTrue(object instanceof Literal);
- assertTrue(SINGLE_SENTENCE.contains(((Literal) object).getLexicalForm()));
- object = null;
- //test start/end if present
- Iterator<Triple> startPosIterator = g.filter(textAnnotation,
- ENHANCER_START, null);
- Iterator<Triple> endPosIterator = g.filter(textAnnotation,
- ENHANCER_END, null);
- //start end is optional, but if start is present, that also end needs to be set
- if(startPosIterator.hasNext()){
- Resource resource = startPosIterator.next().getObject();
- //only a single start position is supported
- assertTrue(!startPosIterator.hasNext());
- assertTrue(resource instanceof TypedLiteral);
- TypedLiteral startPosLiteral = (TypedLiteral) resource;
- resource = null;
- int start = LiteralFactory.getInstance().createObject(Integer.class, startPosLiteral);
- startPosLiteral = null;
- //now get the end
- //end must be defined if start is present
- assertTrue(endPosIterator.hasNext());
- resource = endPosIterator.next().getObject();
- //only a single end position is supported
- assertTrue(!endPosIterator.hasNext());
- assertTrue(resource instanceof TypedLiteral);
- TypedLiteral endPosLiteral = (TypedLiteral) resource;
- resource = null;
- int end = LiteralFactory.getInstance().createObject(Integer.class, endPosLiteral);
- endPosLiteral = null;
- //check for equality of the selected text and the text on the selected position in the content
- //System.out.println("TA ["+start+"|"+end+"]"+selectedText.getLexicalForm()+"<->"+content.substring(start,end));
- assertEquals(content.substring(start, end), selectedText.getLexicalForm());
- } else {
- //if no start position is present, there must also be no end position defined
- assertTrue(!endPosIterator.hasNext());
- }
}
+
}
\ No newline at end of file
Modified: incubator/stanbol/trunk/enhancer/engines/zemanta/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/zemanta/pom.xml?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/zemanta/pom.xml (original)
+++ incubator/stanbol/trunk/enhancer/engines/zemanta/pom.xml Mon May 14 19:43:22 2012
@@ -118,6 +118,12 @@
<!-- for tests -->
<dependency>
<groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.test</artifactId>
+ <version>0.10.0-incubating-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.enhancer.core</artifactId>
<version>0.10.0-incubating-SNAPSHOT</version>
<scope>test</scope>
Modified: incubator/stanbol/trunk/enhancer/engines/zemanta/src/main/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/zemanta/src/main/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngine.java?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/zemanta/src/main/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngine.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/zemanta/src/main/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngine.java Mon May 14 19:43:22 2012
@@ -25,6 +25,7 @@ import static org.apache.stanbol.enhance
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_REFERENCE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_TYPE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTED_TEXT;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTION_CONTEXT;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_START;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.RDF_TYPE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_CATEGORY;
@@ -46,6 +47,7 @@ import org.apache.clerezza.rdf.core.Lite
import org.apache.clerezza.rdf.core.LiteralFactory;
import org.apache.clerezza.rdf.core.MGraph;
import org.apache.clerezza.rdf.core.NonLiteral;
+import org.apache.clerezza.rdf.core.PlainLiteral;
import org.apache.clerezza.rdf.core.Triple;
import org.apache.clerezza.rdf.core.TripleCollection;
import org.apache.clerezza.rdf.core.UriRef;
@@ -103,7 +105,7 @@ import org.slf4j.LoggerFactory;
public class ZemantaEnhancementEngine
extends AbstractEnhancementEngine<IOException,RuntimeException>
implements EnhancementEngine, ServiceProperties {
-
+
@Property
public static final String API_KEY_PROPERTY = "org.apache.stanbol.enhancer.engines.zemanta.key";
@@ -114,6 +116,15 @@ public class ZemantaEnhancementEngine
Collections.unmodifiableSet(new HashSet<String>(
Arrays.asList("text/plain","text/html")));
+ /**
+ * The maximal prefix/suffix size used for the selection context. This is
+ * required, because Zemanta does only provide the Anchor text, but not the
+ * exact position within the text. So this engine creates a TextAnnotation
+ * for each occurrence of the Anchor within the text and uses the surrounding
+ * as context.
+ */
+ private static final int SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE = 50;
+
private static final Logger log = LoggerFactory.getLogger(ZemantaEnhancementEngine.class);
/**
@@ -223,13 +234,13 @@ public class ZemantaEnhancementEngine
Iterator<Triple> categories = results.filter(null, RDF_TYPE, ZemantaOntologyEnum.Category.getUri());
while (categories.hasNext()) {
NonLiteral category = categories.next().getSubject();
- log.info("process category " + category);
+ log.debug("process category " + category);
Double confidence = parseConfidence(results, category);
- log.info(" > confidence :" + confidence);
+ log.debug(" > confidence :" + confidence);
//now we need to follow the Target link
UriRef target = EnhancementEngineHelper.getReference(results, category, ZemantaOntologyEnum.target.getUri());
if (target != null) {
- //first check the the used categorisation
+ //first check the used categorisation
UriRef categorisationScheme = EnhancementEngineHelper.getReference(results, target, ZemantaOntologyEnum.categorization.getUri());
if (categorisationScheme != null && categorisationScheme.equals(ZemantaOntologyEnum.categorization_DMOZ.getUri())) {
String categoryTitle = EnhancementEngineHelper.getString(results, target, ZemantaOntologyEnum.title.getUri());
@@ -237,7 +248,7 @@ public class ZemantaEnhancementEngine
//now write the Stanbol Enhancer entity enhancement
UriRef categoryEnhancement = EnhancementEngineHelper.createEntityEnhancement(enhancements, this, ciId);
//write the title
- enhancements.add(new TripleImpl(categoryEnhancement, ENHANCER_ENTITY_LABEL, literalFactory.createTypedLiteral(categoryTitle)));
+ enhancements.add(new TripleImpl(categoryEnhancement, ENHANCER_ENTITY_LABEL, new PlainLiteralImpl(categoryTitle)));
//write the reference
if (categoryTitle.startsWith(ZEMANTA_DMOZ_PREFIX)) {
enhancements.add(
@@ -417,6 +428,28 @@ public class ZemantaEnhancementEngine
new TripleImpl(textAnnotation, ENHANCER_END, literalFactory.createTypedLiteral(current + anchorLength)));
enhancements.add(
new TripleImpl(textAnnotation, ENHANCER_SELECTED_TEXT, anchorLiteral));
+ //extract the selection context
+ int beginPos;
+ if(current <= SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE){
+ beginPos = 0;
+ } else {
+ int start = current-SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE;
+ beginPos = text.indexOf(' ',start);
+ if(beginPos < 0 || beginPos >= current){ //no words
+ beginPos = start; //begin within a word
+ }
+ }
+ int endPos;
+ if(current+anchorLength+SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE >= text.length()){
+ endPos = text.length();
+ } else {
+ int start = current+anchorLength+SELECTION_CONTEXT_PREFIX_SUFFIX_SIZE;
+ endPos = text.lastIndexOf(' ', start);
+ if(endPos <= current+anchorLength){
+ endPos = start; //end within a word;
+ }
+ }
+ enhancements.add(new TripleImpl(textAnnotation,ENHANCER_SELECTION_CONTEXT,new PlainLiteralImpl(text.substring(beginPos, endPos))));
//TODO: Currently I use the confidence of the extraction, but I think this is more
// related to the annotated Entity rather to the selected text.
if (confidence != null) {
Modified: incubator/stanbol/trunk/enhancer/engines/zemanta/src/test/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngineTest.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/zemanta/src/test/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngineTest.java?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/zemanta/src/test/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngineTest.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/zemanta/src/test/java/org/apache/stanbol/enhancer/engines/zemanta/impl/ZemantaEnhancementEngineTest.java Mon May 14 19:43:22 2012
@@ -18,31 +18,17 @@ package org.apache.stanbol.enhancer.engi
import static org.apache.clerezza.rdf.core.serializedform.SupportedFormat.TURTLE;
import static org.apache.stanbol.enhancer.engines.zemanta.impl.ZemantaEnhancementEngine.API_KEY_PROPERTY;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_RELATION;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_REQUIRES;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_TYPE;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_END;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_LABEL;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_REFERENCE;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_START;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.RDF_TYPE;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_CATEGORY;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_ENTITYANNOTATION;
-import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_TEXTANNOTATION;
-import static org.junit.Assert.assertTrue;
+import static org.apache.stanbol.enhancer.test.helper.EnhancementStructureHelper.validateAllTextAnnotations;
import java.io.IOException;
import java.net.UnknownHostException;
import java.util.Dictionary;
+import java.util.HashMap;
import java.util.Hashtable;
-import java.util.Iterator;
+import java.util.Map;
-import org.apache.clerezza.rdf.core.Literal;
import org.apache.clerezza.rdf.core.LiteralFactory;
-import org.apache.clerezza.rdf.core.MGraph;
import org.apache.clerezza.rdf.core.Resource;
-import org.apache.clerezza.rdf.core.Triple;
-import org.apache.clerezza.rdf.core.TypedLiteral;
import org.apache.clerezza.rdf.core.UriRef;
import org.apache.clerezza.rdf.jena.serializer.JenaSerializerProvider;
import org.apache.stanbol.enhancer.contentitem.inmemory.InMemoryContentItemFactory;
@@ -53,6 +39,7 @@ import org.apache.stanbol.enhancer.servi
import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
import org.apache.stanbol.enhancer.servicesapi.impl.StringSource;
import org.apache.stanbol.enhancer.servicesapi.rdf.Properties;
+import org.apache.stanbol.enhancer.test.helper.EnhancementStructureHelper;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -124,147 +111,21 @@ public class ZemantaEnhancementEngineTes
}
JenaSerializerProvider serializer = new JenaSerializerProvider();
serializer.serialize(System.out, ci.getMetadata(), TURTLE);
- int textAnnoNum = checkAllTextAnnotations(ci.getMetadata(), BIO_DOMAIN_TEXT);
+ Map<UriRef,Resource> expectedValues = new HashMap<UriRef,Resource>();
+ expectedValues.put(Properties.ENHANCER_EXTRACTED_FROM, ci.getUri());
+ expectedValues.put(Properties.DC_CREATOR, LiteralFactory.getInstance().createTypedLiteral(
+ zemantaEngine.getClass().getName()));
+
+ int textAnnoNum = validateAllTextAnnotations(ci.getMetadata(), BIO_DOMAIN_TEXT, expectedValues);
log.info(textAnnoNum + " TextAnnotations found ...");
- int entityAnnoNum = checkAllEntityAnnotations(ci.getMetadata());
+ int entityAnnoNum = EnhancementStructureHelper.validateAllEntityAnnotations(ci.getMetadata(),expectedValues);
log.info(entityAnnoNum + " EntityAnnotations found ...");
}
- /*
- * -----------------------------------------------------------------------
- * Helper Methods to check Text and EntityAnnotations
- * -----------------------------------------------------------------------
- */
-
- private int checkAllEntityAnnotations(MGraph g) {
- Iterator<Triple> entityAnnotationIterator = g.filter(null,
- RDF_TYPE, ENHANCER_ENTITYANNOTATION);
- int entityAnnotationCount = 0;
- while (entityAnnotationIterator.hasNext()) {
- UriRef entityAnnotation = (UriRef) entityAnnotationIterator.next().getSubject();
- // test if selected Text is added
- checkEntityAnnotation(g, entityAnnotation);
- entityAnnotationCount++;
- }
- return entityAnnotationCount;
- }
-
- /**
- * Checks if an entity annotation is valid.
- */
- private void checkEntityAnnotation(MGraph g, UriRef entityAnnotation) {
- Iterator<Triple> relationIterator = g.filter(
- entityAnnotation, DC_RELATION, null);
- Iterator<Triple> requiresIterator = g.filter(
- entityAnnotation, DC_REQUIRES, null);
- Iterator<Triple> dcTypeCategory = g.filter(
- entityAnnotation, DC_TYPE, ENHANCER_CATEGORY);
- // check if the relation or an requires annotation set
- // also include the DC_TYPE ENHANCER_CATEGORY, because such entityEnhancements
- // do not need to have any values for DC_RELATION nor DC_REQUIRES
- assertTrue(relationIterator.hasNext() || requiresIterator.hasNext() || dcTypeCategory.hasNext());
- while (relationIterator.hasNext()) {
- // test if the referred annotations are text annotations
- UriRef referredTextAnnotation = (UriRef) relationIterator.next().getObject();
- assertTrue(g.filter(referredTextAnnotation, RDF_TYPE,
- ENHANCER_TEXTANNOTATION).hasNext());
- }
-
- // test if an entity is referred
- Iterator<Triple> entityReferenceIterator = g.filter(entityAnnotation,
- ENHANCER_ENTITY_REFERENCE, null);
- assertTrue(entityReferenceIterator.hasNext());
- // test if the reference is an URI
- assertTrue(entityReferenceIterator.next().getObject() instanceof UriRef);
- // test if there is only one entity referred
- //NOTE: The Zemanta Engine referrs several entities if they are marked as
- // owl:sameAs by Zemanta
- //assertFalse(entityReferenceIterator.hasNext());
-
- // finally test if the entity label is set
- Iterator<Triple> entityLabelIterator = g.filter(entityAnnotation,
- ENHANCER_ENTITY_LABEL, null);
- assertTrue(entityLabelIterator.hasNext());
- }
-
- private int checkAllTextAnnotations(MGraph g, String content) {
- Iterator<Triple> textAnnotationIterator = g.filter(null,
- RDF_TYPE, ENHANCER_TEXTANNOTATION);
- // test if a textAnnotation is present
- assertTrue(textAnnotationIterator.hasNext());
- int textAnnotationCount = 0;
- while (textAnnotationIterator.hasNext()) {
- UriRef textAnnotation = (UriRef) textAnnotationIterator.next().getSubject();
- // test if selected Text is added
- checkTextAnnotation(g, textAnnotation,content);
- textAnnotationCount++;
- }
- return textAnnotationCount;
- }
-
- /**
- * Checks if a text annotation is valid.
- */
- private void checkTextAnnotation(MGraph g, UriRef textAnnotation, String content) {
- Iterator<Triple> selectedTextIterator = g.filter(textAnnotation,
- Properties.ENHANCER_SELECTED_TEXT, null);
- // check if the selected text is added
- assertTrue(selectedTextIterator.hasNext());
- // test if the selected text is part of the TEXT_TO_TEST
- Resource object = selectedTextIterator.next().getObject();
- assertTrue(object instanceof Literal);
- Literal selectedText = (Literal)object;
- object = null;
- assertTrue(content.indexOf(selectedText.getLexicalForm()) >= 0);
- // test if context is added
- //context not present for Zemanta
-// Iterator<Triple> selectionContextIterator = g.filter(textAnnotation,
-// Properties.ENHANCER_SELECTION_CONTEXT, null);
-// assertTrue(selectionContextIterator.hasNext());
-// // test if the selected text is part of the TEXT_TO_TEST
-// object = selectionContextIterator.next().getObject();
-// assertTrue(object instanceof Literal);
-// assertTrue(content.indexOf(((Literal)object).getLexicalForm()) >= 0);
-// object = null;
- //test start/end if present
- Iterator<Triple> startPosIterator = g.filter(textAnnotation,
- ENHANCER_START, null);
- Iterator<Triple> endPosIterator = g.filter(textAnnotation,
- ENHANCER_END, null);
- //start end is optional, but if start is present, that also end needs to be set
- if(startPosIterator.hasNext()){
- Resource resource = startPosIterator.next().getObject();
- //only a single start position is supported
- assertTrue(!startPosIterator.hasNext());
- assertTrue(resource instanceof TypedLiteral);
- TypedLiteral startPosLiteral = (TypedLiteral) resource;
- resource = null;
- int start = LiteralFactory.getInstance().createObject(Integer.class, startPosLiteral);
- startPosLiteral = null;
- //now get the end
- //end must be defined if start is present
- assertTrue(endPosIterator.hasNext());
- resource = endPosIterator.next().getObject();
- //only a single end position is supported
- assertTrue(!endPosIterator.hasNext());
- assertTrue(resource instanceof TypedLiteral);
- TypedLiteral endPosLiteral = (TypedLiteral) resource;
- resource = null;
- int end = LiteralFactory.getInstance().createObject(Integer.class, endPosLiteral);
- endPosLiteral = null;
- //check for equality of the selected text and the text on the selected position in the content
- //System.out.println("TA ["+start+"|"+end+"]"+selectedText.getLexicalForm()+"<->"+content.substring(start,end));
- assertTrue(content.substring(start, end).equals(selectedText.getLexicalForm()));
- } else {
- //if no start position is present, there must also be no end position defined
- assertTrue(!endPosIterator.hasNext());
- }
- }
-
public static void main(String[] args) throws Exception{
+ ZemantaEnhancementEngineTest.setUpServices();
ZemantaEnhancementEngineTest test = new ZemantaEnhancementEngineTest();
- test.setUpServices();
test.tesetBioText();
- test.shutdownServices();
+ ZemantaEnhancementEngineTest.shutdownServices();
}
}
Modified: incubator/stanbol/trunk/enhancer/generic/test/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/generic/test/pom.xml?rev=1338363&r1=1338362&r2=1338363&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/generic/test/pom.xml (original)
+++ incubator/stanbol/trunk/enhancer/generic/test/pom.xml Mon May 14 19:43:22 2012
@@ -82,6 +82,10 @@
<artifactId>rdf.core</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.clerezza</groupId>
+ <artifactId>rdf.ontologies</artifactId>
+ </dependency>
+ <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>