You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2013/05/16 15:08:31 UTC
svn commit: r1483337 - in /stanbol/trunk/enhancement-engines: ./
entitycomention/ entitycomention/src/ entitycomention/src/license/
entitycomention/src/main/ entitycomention/src/main/java/
entitycomention/src/main/java/org/ entitycomention/src/main/jav...
Author: rwesten
Date: Thu May 16 13:08:30 2013
New Revision: 1483337
URL: http://svn.apache.org/r1483337
Log:
STANBOL-1070: initial commit of the Entity Co-Mention Engine: Currentyl it does support co-mentions based on fise:TextAnnotation fise:selected-text value.
Added:
stanbol/trunk/enhancement-engines/entitycomention/
stanbol/trunk/enhancement-engines/entitycomention/pom.xml
stanbol/trunk/enhancement-engines/entitycomention/src/
stanbol/trunk/enhancement-engines/entitycomention/src/license/
stanbol/trunk/enhancement-engines/entitycomention/src/license/THIRD-PARTY.properties
stanbol/trunk/enhancement-engines/entitycomention/src/main/
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/CoMentionConstants.java
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/EntityCoMentionEngine.java
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/ContentItemMentionBuilder.java
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/EntityMention.java
stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/InMemoryEntityIndex.java
stanbol/trunk/enhancement-engines/entitycomention/src/main/resources/
stanbol/trunk/enhancement-engines/entitycomention/src/main/resources/OSGI-INF/
stanbol/trunk/enhancement-engines/entitycomention/src/main/resources/OSGI-INF/metatype/
stanbol/trunk/enhancement-engines/entitycomention/src/main/resources/OSGI-INF/metatype/metatype.properties
stanbol/trunk/enhancement-engines/entitycomention/src/test/
stanbol/trunk/enhancement-engines/entitycomention/src/test/java/
stanbol/trunk/enhancement-engines/entitycomention/src/test/java/org/
stanbol/trunk/enhancement-engines/entitycomention/src/test/java/org/apache/
stanbol/trunk/enhancement-engines/entitycomention/src/test/java/org/apache/stanbol/
stanbol/trunk/enhancement-engines/entitycomention/src/test/java/org/apache/stanbol/enhancer/
stanbol/trunk/enhancement-engines/entitycomention/src/test/java/org/apache/stanbol/enhancer/engines/
stanbol/trunk/enhancement-engines/entitycomention/src/test/resources/
stanbol/trunk/enhancement-engines/entitycomention/src/test/resources/log4j.properties
Modified:
stanbol/trunk/enhancement-engines/pom.xml
Added: stanbol/trunk/enhancement-engines/entitycomention/pom.xml
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitycomention/pom.xml?rev=1483337&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitycomention/pom.xml (added)
+++ stanbol/trunk/enhancement-engines/entitycomention/pom.xml Thu May 16 13:08:30 2013
@@ -0,0 +1,115 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>apache-stanbol-enhancement-engines</artifactId>
+ <version>0.10.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.entitycomention</artifactId>
+ <version>0.10.1-SNAPSHOT</version>
+ <packaging>bundle</packaging>
+
+ <name>Apache Stanbol Enhancement Engine : Entity Co-Mention</name>
+ <description>
+ An engine that find co-mentions of Entities earlier mentioned on the Text.
+ NOTE: This Engine does NOT use NLP level co-reference detection. Therefore
+ it will not link Pronouns with the correct Entity.
+ </description>
+
+ <inceptionYear>2013</inceptionYear>
+
+ <scm>
+ <connection>
+ scm:svn:http://svn.apache.org/repos/asf/stanbol/trunk/enhancement-engines/entitycomention/
+ </connection>
+ <developerConnection>
+ scm:svn:https://svn.apache.org/repos/asf/stanbol/trunk/enhancement-engines/entitycomention/
+ </developerConnection>
+ <url>http://stanbol.apache.org/</url>
+ </scm>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <Import-Package>
+ org.apache.stanbol.enhancer.servicesapi; provide:=true; version="[0.10,0.12)",
+ org.apache.stanbol.enhancer.engines.entitylinking;version=${project.version}; provide:=true,
+ *
+ </Import-Package>
+ <Private-Package>
+ org.apache.stanbol.enhancer.engines.entitycomention.*;version=${project.version}
+ </Private-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-scr-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.commons.namespaceprefix.service</artifactId>
+ <version>0.11.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.entitylinking.engine</artifactId>
+ <version>0.10.1-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>org.apache.felix.scr.annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency> <!-- used for debug level logging during tests -->
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+</project>
Added: stanbol/trunk/enhancement-engines/entitycomention/src/license/THIRD-PARTY.properties
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitycomention/src/license/THIRD-PARTY.properties?rev=1483337&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitycomention/src/license/THIRD-PARTY.properties (added)
+++ stanbol/trunk/enhancement-engines/entitycomention/src/license/THIRD-PARTY.properties Thu May 16 13:08:30 2013
@@ -0,0 +1,25 @@
+# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
+#-------------------------------------------------------------------------------
+# Already used licenses in project :
+# - Apache Software License
+# - Apache Software License, Version 2.0
+# - BSD License
+# - Common Development And Distribution License (CDDL), Version 1.0
+# - Common Development And Distribution License (CDDL), Version 1.1
+# - Common Public License, Version 1.0
+# - Eclipse Public License, Version 1.0
+# - GNU General Public License (GPL), Version 2 with classpath exception
+# - GNU Lesser General Public License (LGPL)
+# - GNU Lesser General Public License (LGPL), Version 2.1
+# - ICU License
+# - MIT License
+# - New BSD License
+# - Public Domain License
+#-------------------------------------------------------------------------------
+# Please fill the missing licenses for dependencies :
+#
+#
+#Thu Feb 07 14:27:21 CET 2013
+jwnl--jwnl--1.3.3=BSD License
+org.osgi--org.osgi.compendium--4.1.0=The Apache Software License, Version 2.0
+org.osgi--org.osgi.core--4.1.0=The Apache Software License, Version 2.0
Added: stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/CoMentionConstants.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/CoMentionConstants.java?rev=1483337&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/CoMentionConstants.java (added)
+++ stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/CoMentionConstants.java Thu May 16 13:08:30 2013
@@ -0,0 +1,19 @@
+package org.apache.stanbol.enhancer.engines.entitycomention;
+
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig;
+
+public interface CoMentionConstants {
+
+ /**
+ * The {@link EntityLinkerConfig#NAME_FIELD} uri internally used by the
+ * {@link EntityCoMentionEngine}.
+ */
+ UriRef CO_MENTION_LABEL_FIELD = new UriRef("urn:org.stanbol:enhander.engine.entitycomention:co-mention-label");
+
+ /**
+ * The {@link EntityLinkerConfig#TYPE_FIELD} uri internally used by the
+ * {@link EntityCoMentionEngine}.
+ */
+ UriRef CO_MENTION_TYPE_FIELD = new UriRef("urn:org.stanbol:enhander.engine.entitycomention:co-mention-type");
+}
Added: stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/EntityCoMentionEngine.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/EntityCoMentionEngine.java?rev=1483337&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/EntityCoMentionEngine.java (added)
+++ stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/EntityCoMentionEngine.java Thu May 16 13:08:30 2013
@@ -0,0 +1,399 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.stanbol.enhancer.engines.entitycomention;
+
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.CASE_SENSITIVE;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.DEFAULT_CASE_SENSITIVE_MATCHING_STATE;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.DEFAULT_DEREFERENCE_ENTITIES_STATE;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.DEFAULT_MATCHING_LANGUAGE;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.DEFAULT_MIN_TOKEN_SCORE;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.DEFAULT_SUGGESTIONS;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.DEREFERENCE_ENTITIES;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.DEREFERENCE_ENTITIES_FIELDS;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.MIN_TOKEN_SCORE;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.NAME_FIELD;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.REDIRECT_FIELD;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.REDIRECT_MODE;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.SUGGESTIONS;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.TYPE_FIELD;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.TYPE_MAPPINGS;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.TextProcessingConfig.DEFAULT_MIN_SEARCH_TOKEN_LENGTH;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.TextProcessingConfig.DEFAULT_PROCESS_ONLY_PROPER_NOUNS_STATE;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.TextProcessingConfig.MIN_SEARCH_TOKEN_LENGTH;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.TextProcessingConfig.PROCESSED_LANGUAGES;
+import static org.apache.stanbol.enhancer.engines.entitylinking.config.TextProcessingConfig.PROCESS_ONLY_PROPER_NOUNS_STATE;
+import static org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper.getAnalysedText;
+import static org.apache.stanbol.enhancer.nlp.utils.NlpEngineHelper.getLanguage;
+import static org.apache.stanbol.enhancer.servicesapi.EnhancementEngine.PROPERTY_NAME;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_CONTRIBUTOR;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_RELATION;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_TYPE;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_CONFIDENCE;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_END;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_REFERENCE;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_START;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.RDF_TYPE;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_TEXTANNOTATION;
+import static org.osgi.framework.Constants.SERVICE_RANKING;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Dictionary;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import org.apache.clerezza.rdf.core.Language;
+import org.apache.clerezza.rdf.core.Literal;
+import org.apache.clerezza.rdf.core.LiteralFactory;
+import org.apache.clerezza.rdf.core.MGraph;
+import org.apache.clerezza.rdf.core.NonLiteral;
+import org.apache.clerezza.rdf.core.PlainLiteral;
+import org.apache.clerezza.rdf.core.Resource;
+import org.apache.clerezza.rdf.core.Triple;
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl;
+import org.apache.clerezza.rdf.core.impl.TripleImpl;
+import org.apache.commons.lang.StringUtils;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.ConfigurationPolicy;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.PropertyOption;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.Service;
+import org.apache.stanbol.commons.namespaceprefix.NamespacePrefixService;
+import org.apache.stanbol.enhancer.engines.entitycomention.impl.ContentItemMentionBuilder;
+import org.apache.stanbol.enhancer.engines.entitycomention.impl.InMemoryEntityIndex;
+import org.apache.stanbol.enhancer.engines.entitylinking.Entity;
+import org.apache.stanbol.enhancer.engines.entitylinking.EntitySearcher;
+import org.apache.stanbol.enhancer.engines.entitylinking.EntitySearcherException;
+import org.apache.stanbol.enhancer.engines.entitylinking.LabelTokenizer;
+import org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig;
+import org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.RedirectProcessingMode;
+import org.apache.stanbol.enhancer.engines.entitylinking.config.LanguageProcessingConfig;
+import org.apache.stanbol.enhancer.engines.entitylinking.config.TextProcessingConfig;
+import org.apache.stanbol.enhancer.engines.entitylinking.impl.EntityLinker;
+import org.apache.stanbol.enhancer.engines.entitylinking.impl.LinkedEntity;
+import org.apache.stanbol.enhancer.engines.entitylinking.impl.Suggestion;
+import org.apache.stanbol.enhancer.engines.entitylinking.impl.LinkedEntity.Occurrence;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedText;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.enhancer.servicesapi.EngineException;
+import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.ServiceProperties;
+import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
+import org.apache.stanbol.enhancer.servicesapi.impl.AbstractEnhancementEngine;
+import org.apache.stanbol.enhancer.servicesapi.rdf.Properties;
+import org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses;
+import org.osgi.framework.BundleContext;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+/**
+ * The Entity Co-Mentiaon Engine builds a local knowledge base already extracted
+ * <code>fise:TextAnnotation</code>s and suggested
+ * <code>fise:EntityAnnotation</code>s. This information are then used to perform
+ * an entity linking process. By doing so this engine will be able to detect
+ * Co-Mentions of Entities within the processed document. <p>
+ *
+ *
+ *
+ * @author Rupert Westenthaler
+ *
+ */
+@Component(
+ configurationFactory = true,
+ policy = ConfigurationPolicy.REQUIRE, // the baseUri is required!
+ specVersion = "1.1",
+ metatype = true,
+ immediate = true,
+ inherit = true)
+@org.apache.felix.scr.annotations.Properties(value={
+ @Property(name=PROPERTY_NAME),
+ @Property(name=CASE_SENSITIVE,boolValue=DEFAULT_CASE_SENSITIVE_MATCHING_STATE),
+ @Property(name=MIN_SEARCH_TOKEN_LENGTH, intValue=DEFAULT_MIN_SEARCH_TOKEN_LENGTH),
+ @Property(name=PROCESS_ONLY_PROPER_NOUNS_STATE, boolValue=DEFAULT_PROCESS_ONLY_PROPER_NOUNS_STATE),
+ @Property(name=PROCESSED_LANGUAGES,
+ cardinality=Integer.MAX_VALUE,
+ value={"*;lmmtip;uc=LINK;prop=0.75;pprob=0.75", // link multiple matchable tokens in chunks; link upper case words
+ "de;uc=MATCH", //in German all Nouns are upper case
+ "es;lc=Noun", //the OpenNLP POS tagger for Spanish does not support ProperNouns
+ "nl;lc=Noun"}), //same for Dutch
+ @Property(name=DEFAULT_MATCHING_LANGUAGE,value=""),
+ @Property(name=TYPE_MAPPINGS,cardinality=Integer.MAX_VALUE, value={
+ "dbp-ont:Organisation; dbp-ont:Newspaper; schema:Organization > dbp-ont:Organisation",
+ "dbp-ont:Person; foaf:Person; schema:Person > dbp-ont:Person",
+ "dbp-ont:Place; schema:Place > dbp-ont:Place",
+ "dbp-ont:Work; schema:CreativeWork > dbp-ont:Work",
+ "dbp-ont:Event; schema:Event > dbp-ont:Event",
+ "schema:Product > schema:Product",
+ "skos:Concept > skos:Concept"}),
+ @Property(name=SERVICE_RANKING,intValue=0)
+})
+@Service(value=EnhancementEngine.class)
+public class EntityCoMentionEngine extends AbstractEnhancementEngine<RuntimeException,RuntimeException> implements ServiceProperties {
+
+ private static final Integer ENGINE_ORDERING = ServiceProperties.ORDERING_POST_PROCESSING - 90;
+ private static final Map<String,Object> SERVICE_PROPERTIES =
+ Collections.unmodifiableMap(Collections.singletonMap(
+ ServiceProperties.ENHANCEMENT_ENGINE_ORDERING,
+ (Object)ENGINE_ORDERING));
+
+
+ private final Logger log = LoggerFactory.getLogger(EntityCoMentionEngine.class);
+
+ private final LiteralFactory literalFactory = LiteralFactory.getInstance();
+
+ @Reference
+ protected NamespacePrefixService prefixService;
+
+ @Reference
+ protected LabelTokenizer labelTokenizer;
+
+ private BundleContext bundleContext;
+ /**
+ * EntityLinking configuration used for Co-Mention extractions
+ */
+ private EntityLinkerConfig linkerConfig;
+ /**
+ * TextProcessingConfig used for Co-Mention extraction
+ */
+ private TextProcessingConfig textProcessingConfig;
+
+ /**
+ * Default constructor as used by OSGI. This expects that
+ * {@link #activate(ComponentContext)} is called before usage
+ */
+ public EntityCoMentionEngine() {
+ }
+
+ @Activate
+ @SuppressWarnings("unchecked")
+ protected void activate(ComponentContext ctx) throws ConfigurationException {
+ super.activate(ctx);
+ log.info("activate {}[name:{}]",getClass().getSimpleName(),getName());
+ Dictionary<String,Object> properties = ctx.getProperties();
+ bundleContext = ctx.getBundleContext();
+ //extract TextProcessing and EnityLinking config from the provided properties
+ textProcessingConfig = TextProcessingConfig.createInstance(properties);
+ linkerConfig = EntityLinkerConfig.createInstance(properties,prefixService);
+ //some of the confiugration is predefined
+ linkerConfig.setNameField(CoMentionConstants.CO_MENTION_LABEL_FIELD);
+ linkerConfig.setTypeField(CoMentionConstants.CO_MENTION_TYPE_FIELD);
+ linkerConfig.setMaxSuggestions(5); //there should not be more as 5 suggestions
+ linkerConfig.setMinFoundTokens(1); //a single token is enough
+ linkerConfig.setMinLabelScore(0.24); //1/4 of the tokens
+ linkerConfig.setMinMatchScore( //labelScore * token match factor
+ linkerConfig.getMinLabelScore()*linkerConfig.getMinTokenMatchFactor());
+ linkerConfig.setRedirectProcessingMode(RedirectProcessingMode.IGNORE);
+ //get the metadata later set to the enhancement engine
+ }
+ /**
+ * Deactivates this components.
+ */
+ @Deactivate
+ protected void deactivate(ComponentContext ctx) {
+ log.info("deactivate {}[name:{}]",getClass().getSimpleName(),getName());
+ textProcessingConfig = null;
+ linkerConfig = null;
+ super.deactivate(ctx);
+ }
+
+ @Override
+ public int canEnhance(ContentItem ci) throws EngineException {
+ String language = getLanguage(this, ci, false);
+ if(language == null || textProcessingConfig.getConfiguration(language) == null){
+ log.debug("Engine {} ignores ContentItem {} becuase language {} is not condigured.",
+ new Object[]{ getName(), ci.getUri(), language});
+ return CANNOT_ENHANCE;
+ }
+ //we need a detected language, the AnalyzedText contentPart with Tokens.
+ AnalysedText at = getAnalysedText(this, ci, false);
+ return at != null && at.getTokens().hasNext() ?
+ ENHANCE_ASYNC : CANNOT_ENHANCE;
+ }
+
+ @Override
+ public void computeEnhancements(ContentItem ci) throws EngineException {
+ AnalysedText at = getAnalysedText(this, ci, true);
+ String language = getLanguage(this, ci, true);
+ LanguageProcessingConfig languageConfig = textProcessingConfig.getConfiguration(language);
+ if(languageConfig == null){
+ throw new IllegalStateException("The language '"+language+"' is not configured "
+ + "to be processed by this Engine. As this is already checked within the "
+ + "canEnhance(..) method this may indicate an bug in the used "
+ + "EnhanceemntJobManager implementation!");
+ }
+ if(log.isDebugEnabled()){
+ log.debug("compute co-mentions for ContentItem {} language {} text={}",
+ new Object []{ci.getUri().getUnicodeString(), language, StringUtils.abbreviate(at.getSpan(), 100)});
+ }
+ //create the in-memory database for the mentioned Entities
+ ContentItemMentionBuilder entityMentionIndex = new ContentItemMentionBuilder(ci,
+ labelTokenizer, language, linkerConfig.getDefaultLanguage());
+ EntityLinker entityLinker = new EntityLinker(at,language,
+ languageConfig, entityMentionIndex, linkerConfig, labelTokenizer,entityMentionIndex);
+ //process
+ try {
+ entityLinker.process();
+ } catch (EntitySearcherException e) {
+ log.error("Unable to link Entities with "+entityLinker,e);
+ throw new EngineException(this, ci, "Unable to link Entities with "+entityLinker, e);
+ }
+ //TODO: write results
+ ci.getLock().writeLock().lock();
+ try {
+ writeComentions(ci,entityLinker.getLinkedEntities().values(), language);
+ } finally {
+ ci.getLock().writeLock().unlock();
+ }
+ log.info("Found co-mentions:");
+ for(LinkedEntity linkedEntity : entityLinker.getLinkedEntities().values()){
+ log.info(" > {}",linkedEntity);
+ }
+ }
+
+ private void writeComentions(ContentItem ci,Collection<LinkedEntity> comentions, String language) {
+ Language languageObject = null;
+ if(language != null && !language.isEmpty()){
+ languageObject = new Language(language);
+ }
+
+ MGraph metadata = ci.getMetadata();
+
+ for(LinkedEntity comention : comentions){
+ //URIs of TextAnnotations for the initial mention of this co-mention
+ Collection<UriRef> initialMentions = new ArrayList<UriRef>(comention.getOccurrences().size());
+ for(Suggestion suggestion : comention.getSuggestions()){
+ Entity entity = suggestion.getEntity();
+ if(entity.getData().filter(entity.getUri(),RDF_TYPE,ENHANCER_TEXTANNOTATION).hasNext()){
+ //this is a textAnnotation
+ initialMentions.add(entity.getUri());
+ } //else TODO support also Entities!!
+ }
+ //first create the TextAnnotations for the co-mention
+ for(Occurrence occurrence : comention.getOccurrences()){
+ Literal startLiteral = literalFactory.createTypedLiteral(occurrence.getStart());
+ Literal endLiteral = literalFactory.createTypedLiteral(occurrence.getEnd());
+ //search for existing text annotation
+ boolean ignore = false;
+ //search for textAnnotations with the same end
+ UriRef textAnnotation = null;
+ Iterator<Triple> it = metadata.filter(null, ENHANCER_START, startLiteral);
+ while(it.hasNext()){
+ Triple t = it.next();
+ Integer end = EnhancementEngineHelper.get(metadata, t.getSubject(), ENHANCER_END, Integer.class, literalFactory);
+ if(end != null &&
+ metadata.filter(t.getSubject(), RDF_TYPE, ENHANCER_TEXTANNOTATION).hasNext()){
+ textAnnotation = (UriRef)t.getSubject();
+ if(end > occurrence.getEnd()){
+ // there is an other TextAnnotation selecting a bigger Span
+ //so we should ignore this Occurrence
+ ignore = true;
+ }
+ }
+ }
+ it = metadata.filter(null, ENHANCER_END, endLiteral);
+ while(it.hasNext()){
+ Triple t = it.next();
+ Integer start = EnhancementEngineHelper.get(metadata, t.getSubject(), ENHANCER_START, Integer.class, literalFactory);
+ if(start != null &&
+ metadata.filter(t.getSubject(), RDF_TYPE, ENHANCER_TEXTANNOTATION).hasNext()){
+ textAnnotation = (UriRef)t.getSubject();
+ if(start < occurrence.getStart()){
+ // there is an other TextAnnotation selecting a bigger Span
+ //so we should ignore this Occurrence
+ ignore = true;
+ }
+ }
+ }
+ if(!ignore){
+ //collect confidence values of co-mentions
+ Double maxConfidence = null;
+ if(textAnnotation == null){ //not found ... create a new TextAnnotation for the co-mention
+ textAnnotation = EnhancementEngineHelper.createTextEnhancement(ci, this);
+ metadata.add(new TripleImpl(textAnnotation,
+ Properties.ENHANCER_START,
+ startLiteral));
+ metadata.add(new TripleImpl(textAnnotation,
+ Properties.ENHANCER_END,
+ endLiteral));
+ metadata.add(new TripleImpl(textAnnotation,
+ Properties.ENHANCER_SELECTION_CONTEXT,
+ new PlainLiteralImpl(occurrence.getContext(),languageObject)));
+ metadata.add(new TripleImpl(textAnnotation,
+ Properties.ENHANCER_SELECTED_TEXT,
+ new PlainLiteralImpl(occurrence.getSelectedText(),languageObject)));
+ } else { //if existing add this engine as contributor
+ metadata.add(new TripleImpl(textAnnotation, DC_CONTRIBUTOR,
+ new PlainLiteralImpl(this.getClass().getName())));
+ //consider the confidence value of the existing TextAnnotation
+ maxConfidence = EnhancementEngineHelper.get(metadata, textAnnotation,
+ ENHANCER_CONFIDENCE, Double.class, literalFactory);
+ }
+ //now process initial mention(s) for the co-mention
+ for(UriRef initialMention : initialMentions){
+ //link the co-mentation with the initial one
+ metadata.add(new TripleImpl(textAnnotation, DC_RELATION, initialMention));
+ //check confidence of the initial one
+ Double confidnece = EnhancementEngineHelper.get(metadata, initialMention,
+ ENHANCER_CONFIDENCE, Double.class, literalFactory);
+ if(confidnece != null){
+ if(maxConfidence == null){
+ maxConfidence = confidnece;
+ } else if(maxConfidence.compareTo(confidnece) <= 0){
+ maxConfidence = confidnece;
+ }
+ }
+ //add suggestions of the initial mention
+ Set<Resource> values = new HashSet<Resource>();
+ for(Iterator<Triple> suggestions = metadata.filter(initialMention, DC_TYPE, null); suggestions.hasNext();){
+ values.add(suggestions.next().getObject());
+ }
+ for(Resource dcType : values){
+ metadata.add(new TripleImpl(textAnnotation, DC_TYPE, dcType));
+ }
+ values.clear();
+ //add the suggestions of the initial mention to this one
+ for(Iterator<Triple> suggestions = metadata.filter(null, DC_RELATION, initialMention); suggestions.hasNext();){
+ values.add(suggestions.next().getSubject());
+ }
+ for(Resource suggestion : values){
+ metadata.add(new TripleImpl((NonLiteral)suggestion, DC_RELATION, textAnnotation));
+
+ }
+ }
+ //TODO: support also Entities
+ if(maxConfidence != null){ //set the confidence value (if known)
+ EnhancementEngineHelper.set(metadata, textAnnotation, ENHANCER_CONFIDENCE, maxConfidence, literalFactory);
+ }
+ } //else ignore this occurence
+ }
+ }
+ }
+ @Override
+ public Map<String,Object> getServiceProperties() {
+ return SERVICE_PROPERTIES;
+ }
+
+}
Added: stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/ContentItemMentionBuilder.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/ContentItemMentionBuilder.java?rev=1483337&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/ContentItemMentionBuilder.java (added)
+++ stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/ContentItemMentionBuilder.java Thu May 16 13:08:30 2013
@@ -0,0 +1,124 @@
+package org.apache.stanbol.enhancer.engines.entitycomention.impl;
+
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_TYPE;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_CONFIDENCE;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_END;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTED_TEXT;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_START;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.RDF_TYPE;
+import static org.apache.stanbol.enhancer.servicesapi.rdf.TechnicalClasses.ENHANCER_TEXTANNOTATION;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.clerezza.rdf.core.LiteralFactory;
+import org.apache.clerezza.rdf.core.MGraph;
+import org.apache.clerezza.rdf.core.Triple;
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.stanbol.enhancer.engines.entitycomention.CoMentionConstants;
+import org.apache.stanbol.enhancer.engines.entitylinking.LabelTokenizer;
+import org.apache.stanbol.enhancer.engines.entitylinking.impl.LinkingStateAware;
+import org.apache.stanbol.enhancer.nlp.model.Section;
+import org.apache.stanbol.enhancer.nlp.model.Token;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ContentItemMentionBuilder extends InMemoryEntityIndex implements LinkingStateAware{
+
+ private static final Logger log = LoggerFactory.getLogger(ContentItemMentionBuilder.class);
+ private static final LiteralFactory lf = LiteralFactory.getInstance();
+
+ private ContentItem ci;
+ /**
+ * The last index notified via {@link #startToken(Token)}
+ */
+ private Integer lastIndex = 0;
+
+ private SortedMap<Integer,Collection<EntityMention>> mentionIndex = new TreeMap<Integer,Collection<EntityMention>>();
+
+
+ public ContentItemMentionBuilder(ContentItem ci, LabelTokenizer labelTokenizer,
+ String...languages){
+ super(labelTokenizer,CoMentionConstants.CO_MENTION_LABEL_FIELD, languages);
+ this.ci = ci;
+ ci.getLock().readLock().lock();
+ try {
+ initContext();
+ } finally {
+ ci.getLock().readLock().unlock();
+ }
+ }
+
+
+ private void initContext() {
+ MGraph m = ci.getMetadata();
+ for(Iterator<Triple> it = m.filter(null, RDF_TYPE, ENHANCER_TEXTANNOTATION); it.hasNext();){
+ UriRef ta = (UriRef)it.next().getSubject();
+ String selectedText = EnhancementEngineHelper.getString(m, ta, ENHANCER_SELECTED_TEXT);
+ if(selectedText != null){
+ //NOTE: Typically it is not possible to find co-mentions for Entities with a
+ // single Token, so can ignore those.
+ // The only exception would be to use proper-nouns for initial linking and
+ // Nouns for the co-mention resolution. In such cases this might result
+ // in additional extractions.
+ String[] tokens = tokenizer.tokenize(selectedText, language);
+ if(tokens.length > 1){ //TODO make configurable
+ Double confidence = EnhancementEngineHelper.get(m,ta,ENHANCER_CONFIDENCE,Double.class,lf);
+ if(confidence == null || confidence > 0.85){ //TODO make configurable
+ Integer start = EnhancementEngineHelper.get(m,ta,ENHANCER_START,Integer.class,lf);
+ Integer end = EnhancementEngineHelper.get(m,ta,ENHANCER_END,Integer.class,lf);
+ registerMention(new EntityMention(ta,m, ENHANCER_SELECTED_TEXT, DC_TYPE,
+ start != null && end != null ? new Integer[]{start,end} : null));
+ } // else confidence to low
+ } //else ignore Tokens with a single token
+ } // else no selected text
+ }
+ }
+
+ private void registerMention(EntityMention entityMention){
+ if(entityMention.getStart() == null || entityMention.getStart() < 0){
+ log.debug(" > add global Mention[entity: {}]",entityMention.getId());
+ addEntity(entityMention);
+ } else {
+ Collection<EntityMention> mentions = mentionIndex.get(entityMention.getEnd());
+ if(mentions == null){
+ mentions = new ArrayList<EntityMention>();
+ mentionIndex.put(entityMention.getEnd(), mentions);
+ }
+ mentions.add(entityMention);
+ }
+ }
+
+ /**
+ * Everytime the entityLinker starts to process a token we need to check
+ * if we need to add additional contextual information from the {@link ContentItem}
+ * to the {@link InMemoryEntityIndex}
+ */
+ @Override
+ public void startToken(Token token) {
+ Integer actIndex = token.getStart();
+ for(Collection<EntityMention> mentions : mentionIndex.subMap(lastIndex, actIndex).values()){
+ for(EntityMention mention : mentions){
+ log.debug(" > add Mention[index: [{},{}], entity: {}]",new Object[]{
+ mention.getStart(),mention.getEnd(), mention.getId()});
+ addEntity(mention);
+ }
+ }
+ lastIndex = actIndex;
+ }
+
+ @Override
+ public void startSection(Section sentence) {/* not used */}
+ @Override
+ public void endSection(Section sentence) {/* not used */}
+ @Override
+ public void endToken(Token token) {/* not used */}
+
+
+}
Added: stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/EntityMention.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/EntityMention.java?rev=1483337&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/EntityMention.java (added)
+++ stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/EntityMention.java Thu May 16 13:08:30 2013
@@ -0,0 +1,116 @@
+package org.apache.stanbol.enhancer.engines.entitycomention.impl;
+
+import java.util.Iterator;
+
+import org.apache.clerezza.rdf.core.MGraph;
+import org.apache.clerezza.rdf.core.PlainLiteral;
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.stanbol.enhancer.engines.entitycomention.CoMentionConstants;
+import org.apache.stanbol.enhancer.engines.entitycomention.EntityCoMentionEngine;
+import org.apache.stanbol.enhancer.engines.entitylinking.Entity;
+import org.apache.stanbol.enhancer.engines.entitylinking.impl.EntityLinker;
+
+/**
+ * {@link Entity} implementation used by the {@link EntityCoMentionEngine}. It
+ * overrides the {@link #getText(UriRef)} and {@link #getReferences(UriRef)}
+ * methods to use the a different labelField if
+ * {@link CoMentionConstants#CO_MENTION_LABEL_FIELD} is parsed as parameter.
+ * This allows the {@link EntityLinker} to use different properties for different
+ * Entities when linking against the {@link InMemoryEntityIndex}.
+ * @author Rupert Westenthaler
+ *
+ */
+public class EntityMention extends Entity {
+ /**
+ * The label field of this Entity
+ */
+ public final UriRef labelField;
+ /**
+ * The type field of this Entity
+ */
+ private final UriRef typeField;
+ /**
+ * The start/end char indexes char index of the first mention
+ */
+ private final Integer[] span;
+
+ private static int CO_MENTION_FIELD_HASH = CoMentionConstants.CO_MENTION_LABEL_FIELD.hashCode();
+ private static int CO_MENTION_TYPE_HASH = CoMentionConstants.CO_MENTION_TYPE_FIELD.hashCode();
+
+ /**
+ * Creates a new MentionEntity for the parsed parameters
+ * @param uri the {@link UriRef} of the Entity
+ * @param data the {@link MGraph} with the data for the Entity
+ * @param labelField the {@link UriRef} of the property holding the
+ * labels of this Entity. This property will be used for all calls to
+ * {@link #getText(UriRef)} and {@link #getReferences(UriRef)} if
+ * {@link CoMentionConstants#CO_MENTION_LABEL_FIELD} is parsed as parameter
+ * @param span the start/end char indexes of the mention
+ */
+ public EntityMention(UriRef uri, MGraph data, UriRef labelField, UriRef typeField, Integer[] span) {
+ super(uri, data);
+ if(labelField == null){
+ throw new IllegalArgumentException("The LabelField MUST NOT be NULL!");
+ }
+ this.labelField = labelField;
+ if(typeField == null){
+ throw new IllegalArgumentException("The TypeFeild MUST NOT be NULL!");
+ }
+ this.typeField = typeField;
+ if(span != null && (span.length != 2 || span[0] == null || span[1] == null)){
+ throw new IllegalArgumentException("If a span is parsed the length of the Array MUST BE 2 " +
+ "AND start, end MUST NOT be NULL (parsed: "+span+")!");
+ }
+ this.span = span;
+
+ }
+ /**
+ * Wrapps the parsed Entity and redirects calls to
+ * {@link CoMentionConstants#CO_MENTION_LABEL_FIELD} to the parsed labelField
+ * @param entity the Entity to wrap
+ * @param labelField the {@link UriRef} of the property holding the
+ * labels of this Entity. This property will be used for all calls to
+ * {@link #getText(UriRef)} and {@link #getReferences(UriRef)} if
+ * {@link CoMentionConstants#CO_MENTION_LABEL_FIELD} is parsed as parameter
+ * @param index the char index of the initial mention in the document
+ */
+ public EntityMention(Entity entity, UriRef labelField, UriRef typeField, Integer[] span) {
+ this(entity.getUri(), entity.getData(),labelField,typeField,span);
+ }
+
+ @Override
+ public Iterator<PlainLiteral> getText(UriRef field) {
+ if(CO_MENTION_FIELD_HASH == field.hashCode() && //avoid calling equals
+ CoMentionConstants.CO_MENTION_LABEL_FIELD.equals(field)){
+ return super.getText(labelField);
+ } else if(CO_MENTION_TYPE_HASH == field.hashCode() && //avoid calling equals
+ CoMentionConstants.CO_MENTION_TYPE_FIELD.equals(field)){
+ return super.getText(typeField);
+ } else {
+ return super.getText(field);
+ }
+ }
+
+ @Override
+ public Iterator<UriRef> getReferences(UriRef field) {
+ if(CO_MENTION_FIELD_HASH == field.hashCode() && //avoid calling equals
+ CoMentionConstants.CO_MENTION_LABEL_FIELD.equals(field)){
+ return super.getReferences(labelField);
+ } else if(CO_MENTION_TYPE_HASH == field.hashCode() && //avoid calling equals
+ CoMentionConstants.CO_MENTION_TYPE_FIELD.equals(field)){
+ return super.getReferences(typeField);
+ } else {
+ return super.getReferences(field);
+ }
+ }
+
+ public boolean hasSpan(){
+ return span != null;
+ }
+ public Integer getStart(){
+ return span != null ? span[0] : null;
+ }
+ public Integer getEnd(){
+ return span != null ? span[1] : null;
+ }
+}
Added: stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/InMemoryEntityIndex.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/InMemoryEntityIndex.java?rev=1483337&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/InMemoryEntityIndex.java (added)
+++ stanbol/trunk/enhancement-engines/entitycomention/src/main/java/org/apache/stanbol/enhancer/engines/entitycomention/impl/InMemoryEntityIndex.java Thu May 16 13:08:30 2013
@@ -0,0 +1,222 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.stanbol.enhancer.engines.entitycomention.impl;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.clerezza.rdf.core.Language;
+import org.apache.clerezza.rdf.core.MGraph;
+import org.apache.clerezza.rdf.core.PlainLiteral;
+import org.apache.clerezza.rdf.core.Resource;
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl;
+import org.apache.stanbol.commons.indexedgraph.IndexedMGraph;
+import org.apache.stanbol.enhancer.engines.entitylinking.Entity;
+import org.apache.stanbol.enhancer.engines.entitylinking.EntitySearcher;
+import org.apache.stanbol.enhancer.engines.entitylinking.LabelTokenizer;
+import org.apache.stanbol.enhancer.servicesapi.rdf.NamespaceEnum;
+/**
+ * EntitySearch implementation that does hold Entity information of mentioned
+ * Entities in memory.
+ * @author Rupert Westenthaler
+ *
+ */
+public class InMemoryEntityIndex implements EntitySearcher {
+
+ protected final LabelTokenizer tokenizer;
+ //Holds Entity data
+ private SortedMap<String,Collection<Entity>> index = new TreeMap<String,Collection<Entity>>(String.CASE_INSENSITIVE_ORDER);
+ private Map<UriRef,Entity> entities = new HashMap<UriRef,Entity>();
+ private Set<String> indexLanguages;
+ protected String language;
+ protected UriRef nameField;
+
+
+ public InMemoryEntityIndex(LabelTokenizer tokenizer, UriRef nameField, String...languages) {
+ this.indexLanguages = languages == null || languages.length < 1 ?
+ Collections.singleton((String)null) :
+ new HashSet<String>(Arrays.asList(languages));
+ this.language = languages == null || languages.length < 1 ? null :
+ languages[0];
+ this.tokenizer = tokenizer;
+ this.nameField = nameField;
+ }
+
+
+ public void addEntity(Entity entity){
+ entities.put(entity.getUri(), entity);
+ Iterator<PlainLiteral> labels = entity.getText(nameField);
+ while(labels.hasNext()){
+ PlainLiteral label = labels.next();
+ String lang = label.getLanguage() == null ? null : label.getLanguage().toString();
+ if(indexLanguages.contains(lang)){
+ for(String token : tokenizer.tokenize(label.getLexicalForm(),null)){
+ token = token.toLowerCase(Locale.ROOT);
+ Collection<Entity> values = index.get(token);
+ if(values == null){
+ values = new ArrayList<Entity>();
+ index.put(token, values);
+ }
+ values.add(entity);
+ }
+ } //else ignore labels in other languages
+ }
+
+ }
+
+ @Override
+ public Entity get(UriRef id, Set<UriRef> includeFields) throws IllegalStateException {
+ return entities.get(id);
+ }
+
+ @Override
+ public Collection<? extends Entity> lookup(UriRef field,
+ Set<UriRef> includeFields,
+ List<String> search,
+ String[] languages,Integer numResults) throws IllegalStateException {
+ //this assumes that
+ assert nameField.equals(field); //the nameField is the field
+ assert Arrays.asList(languages).contains(language); //the parsed languages include the language
+ //NOTES:
+ // We can ignore the following parameters
+ // * includeFields: as we will return the Entities as added to the index
+
+ //The Syntax requires to
+ // * AND over the tokenized elements of the search List
+ // * OR over the elements in the search
+ // * Elements that do match more search elements need to be ranked first
+ Map<Entity, int[]> results = new HashMap<Entity,int[]>();
+ for(String qe : search){
+ Set<Entity> qeResult = join(tokenizer.tokenize(qe, language));
+ for(Entity e : qeResult){
+ int[] count = results.get(e);
+ if(count != null){
+ count[0] = count[0]+qe.length();
+ } else {
+ results.put(e, new int[]{qe.length()});
+ }
+ }
+ }
+ @SuppressWarnings("unchecked") //TODO how to create generic arrays
+ Entry<Entity,int[]>[] resultArray = results.entrySet().toArray(new Entry[results.size()]);
+ Arrays.sort(resultArray, RESULT_SCORE_COMPARATOR);
+ //final ranking
+ List<Entity> resultList = new ArrayList<Entity>(Math.min(numResults+3, results.size()));
+ int lastScore = -1;
+ boolean done = false;
+ for(int i = 0; i < resultArray.length && !done;i++){
+ if(i < numResults){
+ resultList.add(resultArray[i].getKey());
+ if(i == (numResults - 1)){ //memorize the score of the last included
+ lastScore = resultArray[i].getValue()[0];
+ }
+ } else if (lastScore == resultArray[i].getValue()[0]){
+ //include additional results with the same score
+ resultList.add(resultArray[i].getKey());
+ } else { //cut of
+ done = true;
+ }
+ }
+ return resultList;
+ }
+
+ private static final Comparator<Collection<?>> COLLECTION_SIZE_COMPARATOR = new Comparator<Collection<?>>() {
+
+ @Override
+ public int compare(Collection<?> c1, Collection<?> c2) {
+ return c1 == null && c2 == null ? 0 :
+ c1 == null ? -1 : c2 == null ? 1 : //null values last
+ c2.size() - c1.size(); //lowest size first
+ }
+
+ };
+ private static final Comparator<Entry<Entity,int[]>> RESULT_SCORE_COMPARATOR = new Comparator<Entry<Entity,int[]>>() {
+
+ @Override
+ public int compare(Entry<Entity,int[]> e1, Entry<Entity,int[]> e2) {
+ return e1 == null && e2 == null ? 0 :
+ e1 == null ? -1 : e2 == null ? 1 : //null values last
+ e1.getValue()[0] - e2.getValue()[0]; //highest score first!
+ }};
+
+ /**
+ * Searches for Elements that do contain all the parsed Query Tokens
+ * @param queryTokens the query tokens. MUST NOT be NULL, empty or contain
+ * any NULL or empty string as element
+ * @return matching entities or an empty Set if none.
+ */
+ private Set<Entity> join(String...queryTokens) {
+ @SuppressWarnings("unchecked") //TODO: how to create a generic typed array
+ Collection<Entity>[] tokenResults = new Collection[queryTokens.length];
+ for(int i=0;i<queryTokens.length;i++){
+ Collection<Entity> tokenResult = index.get(queryTokens[i].toLowerCase(Locale.ROOT));
+ if(tokenResult == null || tokenResult.isEmpty()){
+ return Collections.emptySet();
+ }
+ tokenResults[i] = tokenResult;
+ }
+ Set<Entity> join = new HashSet<Entity>(tokenResults[0]);
+ if(tokenResults.length == 1){
+ return join;
+ } //else we need to join the single results
+
+ //we want to join the shortest results first
+ Arrays.sort(tokenResults,COLLECTION_SIZE_COMPARATOR);
+ for(int i = 1; i < tokenResults.length && !join.isEmpty(); i++){
+ Set<Entity> old = join;
+ join = new HashSet<Entity>(); //new set to add all elements
+ for(Iterator<Entity> it = tokenResults[i].iterator(); it.hasNext() && !old.isEmpty();){
+ Entity e = it.next();
+ if(old.remove(e)){
+ join.add(e);
+ }
+ }
+ }
+ return join;
+ }
+
+
+ @Override
+ public boolean supportsOfflineMode() {
+ return true;
+ }
+
+
+ @Override
+ public Integer getLimit() {
+ return null;
+ }
+
+ @Override
+ public Map<UriRef,Collection<Resource>> getOriginInformation() {
+ return Collections.emptyMap();
+ }
+}
Added: stanbol/trunk/enhancement-engines/entitycomention/src/main/resources/OSGI-INF/metatype/metatype.properties
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitycomention/src/main/resources/OSGI-INF/metatype/metatype.properties?rev=1483337&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitycomention/src/main/resources/OSGI-INF/metatype/metatype.properties (added)
+++ stanbol/trunk/enhancement-engines/entitycomention/src/main/resources/OSGI-INF/metatype/metatype.properties Thu May 16 13:08:30 2013
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+stanbol.enhancer.engine.name.name=Name
+stanbol.enhancer.engine.name.description=The name of the enhancement engine as \
+used in the RESTful interface '/engine/<name>'
+
+service.ranking.name=Ranking
+service.ranking.description=If two enhancement engines with the same name are active the \
+one with the higher ranking will be used to process parsed content items.
+
+
+#===============================================================================
+#Properties and Options used to configure
+#===============================================================================
+org.apache.stanbol.enhancer.engines.entitycomention.EntityCoMentionEngine.name=Apache \
+Stanbol Enhancer Engine: Entity Co-Mention
+org.apache.stanbol.enhancer.engines.entitycomention.EntityCoMentionEngine.description=An engine \
+that find co-mentions of Entities earlier mentioned on the Text.
+
+enhancer.engines.linking.minSearchTokenLength.name=Min Token Length
+enhancer.engines.linking.minSearchTokenLength.description=The minimum \
+length of Tokens used to lookup Entities within the Controlled Vocabulary. This parameter is ignored \
+in case a POS (Part of Speech) tagger is available for the language of the parsed content.
+
+enhancer.engines.linking.caseSensitive.name=Case Sensitivity
+enhancer.engines.linking.caseSensitive.description=Allows to enable/disable \
+case sensitive matching
+
+enhancer.engines.linking.properNounsState.name=Link ProperNouns only
+enhancer.engines.linking.properNounsState.description=If activated \
+only ProperNouns will be matched against the Vocabulary. If deactivated any Noun will be matched. \
+NOTE that this parameter requires a tag of the POS TagSet to be mapped against 'olia:PorperNoun'. \
+Otherwise mapping will not work as expected.
+
+enhancer.engines.linking.processedLanguages.name=Processed Languages
+enhancer.engines.linking.processedLanguages.description=Languages to \
+process and optionally language specific configurations. Syntax "{lang};{param-name}={param-value};\
+{param-name}={param-value};...". Supported {param-name}s: "lc" - processed Lexical Categories (see \
+LexicalCategory enumeration for possible values); "pos" - processed Pos types (see Pos enumeration \
+for possible values); "tag" - processed string pos tags; "prob" - minumum probability of pos annotations.
+
+enhancer.engines.linking.defaultMatchingLanguage.name=Default Matching Language
+enhancer.engines.linking.defaultMatchingLanguage.description=The language \
+used in addition to the language detected for the analysed text to search for Entities. Typically this \
+configuration is an empty string to search for labels without any language defined, but for some data \
+sets (such as DBpedia.org) that add languages to any labels it might improve resuls to change this \
+configuration (e.g. to 'en' in the case of DBpedia.org).
+
+enhancer.engines.linking.typeMappings.name=Type Mappings
+enhancer.engines.linking.typeMappings.description=This allows to add \
+additional entity-type > text-annotation-type mappings. Such mappings are used to determine the \
+'dc:type' value of the 'fise:TextAnnotation' created for extracted entities. Usage: \
+variant (a) '{uri}' short for {uri} > {uri} or (b) '{source1};{source2};..;{sourceN} > {target}'. \
+Note that a {source} may be only mapped to a single {target}. Multiple {source} types \
+can be mapped to the same {target}.
Added: stanbol/trunk/enhancement-engines/entitycomention/src/test/resources/log4j.properties
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitycomention/src/test/resources/log4j.properties?rev=1483337&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitycomention/src/test/resources/log4j.properties (added)
+++ stanbol/trunk/enhancement-engines/entitycomention/src/test/resources/log4j.properties Thu May 16 13:08:30 2013
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Root logger option
+log4j.rootLogger=INFO, stdout
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n
+log4j.logger.org.apache.stanbol.enhancer.engines.keywordextraction=DEBUG
\ No newline at end of file
Modified: stanbol/trunk/enhancement-engines/pom.xml
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/pom.xml?rev=1483337&r1=1483336&r2=1483337&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/pom.xml (original)
+++ stanbol/trunk/enhancement-engines/pom.xml Thu May 16 13:08:30 2013
@@ -94,7 +94,10 @@
<module>uima</module>
<!-- Post-Processing-->
- <module>textannotationnewmodel</module> <!-- converts TextAnnotations to the STANBOL-987 model -->
+ <!-- converts TextAnnotations to the STANBOL-987 model -->
+ <module>textannotationnewmodel</module>
+ <!-- finds co-mentions of Entities earlier mentioned on the Text (STANBOL-1070) -->
+ <module>entitycomention</module>
<!-- Enhancement Engines using external services -->
<module>celi</module> <!-- http://linguagrid.org -->