You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2012/12/21 11:04:06 UTC
svn commit: r1424833 - in /stanbol/trunk/enhancement-engines:
entityhublinking/ entitylinking/ entitylinking/engine/
entitylinking/engine/src/
entitylinking/engine/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/config/
entitylinking/en...
Author: rwesten
Date: Fri Dec 21 10:04:04 2012
New Revision: 1424833
URL: http://svn.apache.org/viewvc?rev=1424833&view=rev
Log:
implementation of STANBOL-849, STANBOL-850, STANBOL-851
Added:
stanbol/trunk/enhancement-engines/entitylinking/engine/ (with props)
stanbol/trunk/enhancement-engines/entitylinking/engine/pom.xml
- copied, changed from r1424422, stanbol/trunk/enhancement-engines/entitylinking/pom.xml
stanbol/trunk/enhancement-engines/entitylinking/engine/src/
- copied from r1424422, stanbol/trunk/enhancement-engines/entitylinking/src/
stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/
stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/TestSimpleLabelTokenizer.java
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/ (with props)
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/pom.xml
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/license/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/license/THIRD-PARTY.properties
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/apache/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/apache/stanbol/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/apache/stanbol/enhancer/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/apache/stanbol/enhancer/engines/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/apache/stanbol/enhancer/engines/entityhublinking/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/LuceneLabelTokenizer.java
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/resources/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/resources/OSGI-INF/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/resources/OSGI-INF/metatype/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/resources/OSGI-INF/metatype/metatype.properties
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/LuceneLabelTokenizerTest.java
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/MockComponentContext.java
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/resources/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/ (with props)
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/pom.xml
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/license/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/license/THIRD-PARTY.properties
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/apache/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/apache/stanbol/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/apache/stanbol/enhancer/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/apache/stanbol/enhancer/engines/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/OpenNlpLabelTokenizer.java
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/resources/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/resources/OSGI-INF/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/resources/OSGI-INF/metatype/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/resources/OSGI-INF/metatype/metatype.properties
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/ClasspathDataFileProvider.java
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/MockComponentContext.java
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/OpenNlpLabelTokenizerTest.java
stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/resources/
Removed:
stanbol/trunk/enhancement-engines/entitylinking/engine/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/OpenNlpLabelTokenizer.java
stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/impl/ClasspathDataFileProvider.java
stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/impl/MockComponentContext.java
stanbol/trunk/enhancement-engines/entitylinking/pom.xml
stanbol/trunk/enhancement-engines/entitylinking/src/
Modified:
stanbol/trunk/enhancement-engines/entityhublinking/pom.xml
stanbol/trunk/enhancement-engines/entitylinking/engine/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/config/TextProcessingConfig.java
stanbol/trunk/enhancement-engines/entitylinking/engine/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/SimpleLabelTokenizer.java
stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngineTest.java
stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/impl/TestSearcherImpl.java
Modified: stanbol/trunk/enhancement-engines/entityhublinking/pom.xml
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entityhublinking/pom.xml?rev=1424833&r1=1424832&r2=1424833&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/entityhublinking/pom.xml (original)
+++ stanbol/trunk/enhancement-engines/entityhublinking/pom.xml Fri Dec 21 10:04:04 2012
@@ -86,7 +86,7 @@
<dependency>
<groupId>org.apache.stanbol</groupId>
- <artifactId>org.apache.stanbol.enhancer.engines.entitylinking</artifactId>
+ <artifactId>org.apache.stanbol.enhancer.engines.entitylinking.engine</artifactId>
<version>0.10.0-SNAPSHOT</version>
</dependency>
<dependency>
Propchange: stanbol/trunk/enhancement-engines/entitylinking/engine/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Fri Dec 21 10:04:04 2012
@@ -0,0 +1,7 @@
+.settings
+
+.classpath
+
+.project
+
+target
Copied: stanbol/trunk/enhancement-engines/entitylinking/engine/pom.xml (from r1424422, stanbol/trunk/enhancement-engines/entitylinking/pom.xml)
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/engine/pom.xml?p2=stanbol/trunk/enhancement-engines/entitylinking/engine/pom.xml&p1=stanbol/trunk/enhancement-engines/entitylinking/pom.xml&r1=1424422&r2=1424833&rev=1424833&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/pom.xml (original)
+++ stanbol/trunk/enhancement-engines/entitylinking/engine/pom.xml Fri Dec 21 10:04:04 2012
@@ -24,11 +24,11 @@
<groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.enhancer.parent</artifactId>
<version>0.10.0-SNAPSHOT</version>
- <relativePath>../../enhancer/parent</relativePath>
+ <relativePath>../../../enhancer/parent</relativePath>
</parent>
<groupId>org.apache.stanbol</groupId>
- <artifactId>org.apache.stanbol.enhancer.engines.entitylinking</artifactId>
+ <artifactId>org.apache.stanbol.enhancer.engines.entitylinking.engine</artifactId>
<version>0.10.0-SNAPSHOT</version>
<packaging>bundle</packaging>
@@ -61,8 +61,6 @@
<configuration>
<instructions>
<Import-Package>
- opennlp.tools.tokenize; resolution:=optional,
- org.apache.stanbol.commons.opennlp; resolution:=optional,
org.apache.stanbol.enhancer.servicesapi; provide:=true,
*
</Import-Package>
@@ -108,13 +106,6 @@
<version>0.9.0-incubating</version>
</dependency>
- <dependency> <!-- used for the optional OpenNlpLabelTokenizer -->
- <groupId>org.apache.stanbol</groupId>
- <artifactId>org.apache.stanbol.commons.opennlp</artifactId>
- <version>0.10.0-SNAPSHOT</version>
- <optional>true</optional>
- </dependency>
-
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
Modified: stanbol/trunk/enhancement-engines/entitylinking/engine/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/config/TextProcessingConfig.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/engine/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/config/TextProcessingConfig.java?rev=1424833&r1=1424422&r2=1424833&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/engine/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/config/TextProcessingConfig.java (original)
+++ stanbol/trunk/enhancement-engines/entitylinking/engine/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/config/TextProcessingConfig.java Fri Dec 21 10:04:04 2012
@@ -9,7 +9,6 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import org.apache.stanbol.commons.opennlp.TextAnalyzer;
import org.apache.stanbol.enhancer.engines.entitylinking.engine.EntityLinkingEngine;
import org.apache.stanbol.enhancer.nlp.pos.LexicalCategory;
import org.apache.stanbol.enhancer.nlp.pos.Pos;
Modified: stanbol/trunk/enhancement-engines/entitylinking/engine/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/SimpleLabelTokenizer.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/engine/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/SimpleLabelTokenizer.java?rev=1424833&r1=1424422&r2=1424833&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/engine/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/SimpleLabelTokenizer.java (original)
+++ stanbol/trunk/enhancement-engines/entitylinking/engine/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/SimpleLabelTokenizer.java Fri Dec 21 10:04:04 2012
@@ -17,8 +17,7 @@ import org.osgi.framework.Constants;
@Component(immediate=true)
@Service
@Properties(value={
- @Property(name=Constants.SERVICE_RANKING,intValue=-1000),
- @Property(name=LabelTokenizer.SUPPORTED_LANUAGES,value="*")
+ @Property(name=Constants.SERVICE_RANKING,intValue=-1000)
})
public class SimpleLabelTokenizer implements LabelTokenizer {
@@ -26,6 +25,9 @@ public class SimpleLabelTokenizer implem
@Override
public String[] tokenize(String label, String language) {
+ if(label == null){
+ throw new IllegalArgumentException("The parsed Label MUST NOT be NULL!");
+ }
ArrayList<String> tokens = new ArrayList<String>();
int start = -1;
int pc = 0;
Modified: stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngineTest.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngineTest.java?rev=1424833&r1=1424422&r2=1424833&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngineTest.java (original)
+++ stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/engine/EntityLinkingEngineTest.java Fri Dec 21 10:04:04 2012
@@ -40,8 +40,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import opennlp.tools.tokenize.SimpleTokenizer;
-
import org.apache.clerezza.rdf.core.Literal;
import org.apache.clerezza.rdf.core.LiteralFactory;
import org.apache.clerezza.rdf.core.MGraph;
@@ -51,7 +49,6 @@ import org.apache.clerezza.rdf.core.UriR
import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl;
import org.apache.clerezza.rdf.core.impl.TripleImpl;
import org.apache.stanbol.commons.indexedgraph.IndexedMGraph;
-import org.apache.stanbol.commons.opennlp.OpenNLP;
import org.apache.stanbol.enhancer.contentitem.inmemory.InMemoryContentItemFactory;
import org.apache.stanbol.enhancer.engines.entitylinking.Entity;
import org.apache.stanbol.enhancer.engines.entitylinking.LabelTokenizer;
@@ -59,7 +56,6 @@ import org.apache.stanbol.enhancer.engin
import org.apache.stanbol.enhancer.engines.entitylinking.config.LanguageProcessingConfig;
import org.apache.stanbol.enhancer.engines.entitylinking.config.TextProcessingConfig;
import org.apache.stanbol.enhancer.engines.entitylinking.config.EntityLinkerConfig.RedirectProcessingMode;
-import org.apache.stanbol.enhancer.engines.entitylinking.impl.ClasspathDataFileProvider;
import org.apache.stanbol.enhancer.engines.entitylinking.impl.EntityLinker;
import org.apache.stanbol.enhancer.engines.entitylinking.impl.LinkedEntity;
import org.apache.stanbol.enhancer.engines.entitylinking.impl.Suggestion;
@@ -120,7 +116,7 @@ public class EntityLinkingEngineTest {
@BeforeClass
public static void setUpServices() throws IOException {
- searcher = new TestSearcherImpl(TEST_REFERENCED_SITE_NAME,NAME,SimpleTokenizer.INSTANCE);
+ searcher = new TestSearcherImpl(TEST_REFERENCED_SITE_NAME,NAME,new SimpleLabelTokenizer());
//add some terms to the searcher
MGraph graph = new IndexedMGraph();
UriRef uri = new UriRef("urn:test:PatrickMarshall");
@@ -210,7 +206,6 @@ public class EntityLinkingEngineTest {
TEST_ANALYSED_TEXT.addToken(start+23,start+24).addAnnotation(POS_ANNOTATION, Value.value(new PosTag(".",Pos.Point),1d));
}
- private OpenNLP openNLP = new OpenNLP(new ClasspathDataFileProvider(null));
private LabelTokenizer labelTokenizer = new SimpleLabelTokenizer();
Modified: stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/impl/TestSearcherImpl.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/impl/TestSearcherImpl.java?rev=1424833&r1=1424422&r2=1424833&view=diff
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/impl/TestSearcherImpl.java (original)
+++ stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/impl/TestSearcherImpl.java Fri Dec 21 10:04:04 2012
@@ -28,27 +28,26 @@ import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
-import opennlp.tools.tokenize.Tokenizer;
-
import org.apache.clerezza.rdf.core.PlainLiteral;
import org.apache.clerezza.rdf.core.Resource;
import org.apache.clerezza.rdf.core.UriRef;
import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl;
import org.apache.stanbol.enhancer.engines.entitylinking.Entity;
import org.apache.stanbol.enhancer.engines.entitylinking.EntitySearcher;
+import org.apache.stanbol.enhancer.engines.entitylinking.LabelTokenizer;
import org.apache.stanbol.enhancer.servicesapi.rdf.NamespaceEnum;
public class TestSearcherImpl implements EntitySearcher {
private final UriRef nameField;
- private final Tokenizer tokenizer;
+ private final LabelTokenizer tokenizer;
private SortedMap<String,Collection<Entity>> data = new TreeMap<String,Collection<Entity>>(String.CASE_INSENSITIVE_ORDER);
private Map<UriRef,Entity> entities = new HashMap<UriRef,Entity>();
private Map<UriRef,Collection<Resource>> originInfo;
- public TestSearcherImpl(String siteId,UriRef nameField, Tokenizer tokenizer) {
+ public TestSearcherImpl(String siteId,UriRef nameField, LabelTokenizer tokenizer) {
this.nameField = nameField;
this.tokenizer = tokenizer;
this.originInfo = Collections.singletonMap(
@@ -63,7 +62,7 @@ public class TestSearcherImpl implements
Iterator<PlainLiteral> labels = rep.getText(nameField);
while(labels.hasNext()){
PlainLiteral label = labels.next();
- for(String token : tokenizer.tokenize(label.getLexicalForm())){
+ for(String token : tokenizer.tokenize(label.getLexicalForm(),null)){
Collection<Entity> values = data.get(token);
if(values == null){
values = new ArrayList<Entity>();
Added: stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/TestSimpleLabelTokenizer.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/TestSimpleLabelTokenizer.java?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/TestSimpleLabelTokenizer.java (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/engine/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/TestSimpleLabelTokenizer.java Fri Dec 21 10:04:04 2012
@@ -0,0 +1,41 @@
+package org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestSimpleLabelTokenizer {
+
+
+ private static SimpleLabelTokenizer tokenizer;
+ private String label = "This is only a Test";
+ private String[] expected = label.split(" ");
+
+ @BeforeClass
+ public static final void init(){
+ tokenizer = new SimpleLabelTokenizer();
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testNullLabel(){
+ tokenizer.tokenize(null, "en");
+ }
+ @Test
+ public void testNullLanguate(){
+ String[] tokens = tokenizer.tokenize(label, null);
+ Assert.assertNotNull(tokens);
+ Assert.assertArrayEquals(expected, tokens);
+ }
+ @Test
+ public void testTokenizer(){
+ String[] tokens = tokenizer.tokenize(label, "en");
+ Assert.assertNotNull(tokens);
+ Assert.assertArrayEquals(expected, tokens);
+ }
+ @Test
+ public void testEmptyLabel(){
+ String[] tokens = tokenizer.tokenize("", "en");
+ Assert.assertNotNull(tokens);
+ Assert.assertTrue(tokens.length == 0);
+ }
+}
Propchange: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Fri Dec 21 10:04:04 2012
@@ -0,0 +1,7 @@
+target
+
+.classpath
+
+.settings
+
+.project
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/pom.xml
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/pom.xml?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/pom.xml (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/pom.xml Fri Dec 21 10:04:04 2012
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.parent</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ <relativePath>../../../enhancer/parent</relativePath>
+ </parent>
+
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer.lucene</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ <packaging>bundle</packaging>
+
+ <name>Apache Stanbol Enhancer EntityLinking LabelTokenizer Lucene </name>
+ <description>
+ LabelTokenizer implementation based on the Apache Lucene Tokenizer interface.
+ </description>
+
+ <inceptionYear>2012</inceptionYear>
+
+ <scm>
+ <connection>
+ scm:svn:http://svn.apache.org/repos/asf/stanbol/trunk/enhancement-engines/entityhublinking/lucene-labeltokenizer
+ </connection>
+ <developerConnection>
+ scm:svn:https://svn.apache.org/repos/asf/stanbol/trunk/enhancement-engines/entityhublinking/lucene-labeltokenizer
+ </developerConnection>
+ <url>http://stanbol.apache.org/</url>
+ </scm>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <Import-Package>
+ org.apache.stanbol.enhancer.engines.entitylinking;version=${project.version}; provide:=true,
+ *
+ </Import-Package>
+ <Private-Package>
+ org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer.lucene;version=${project.version}
+ </Private-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-scr-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.entitylinking.engine</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>org.apache.felix.scr.annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/license/THIRD-PARTY.properties
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/license/THIRD-PARTY.properties?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/license/THIRD-PARTY.properties (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/license/THIRD-PARTY.properties Fri Dec 21 10:04:04 2012
@@ -0,0 +1,19 @@
+# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
+#-------------------------------------------------------------------------------
+# Already used licenses in project :
+# - Apache 2
+# - Apache License
+# - Common Development and Distribution License (CDDL) v1.0
+# - Common Public License Version 1.0
+# - ICU License
+# - MIT License
+# - The Apache Software License, Version 2.0
+#-------------------------------------------------------------------------------
+# Please fill the missing licenses for dependencies :
+#
+#
+#Wed Feb 15 19:07:08 CET 2012
+javax.servlet--servlet-api--2.4=The Apache Software License, Version 2.0
+jwnl--jwnl--1.3.3=BSD License
+org.osgi--org.osgi.compendium--4.1.0=The Apache Software License, Version 2.0
+org.osgi--org.osgi.core--4.1.0=The Apache Software License, Version 2.0
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/LuceneLabelTokenizer.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/LuceneLabelTokenizer.java?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/LuceneLabelTokenizer.java (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/LuceneLabelTokenizer.java Fri Dec 21 10:04:04 2012
@@ -0,0 +1,126 @@
+package org.apache.stanbol.enhancer.engines.entityhublinking.labeltokenizer.lucene;
+
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.ConfigurationPolicy;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Properties;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Service;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.Version;
+import org.apache.solr.analysis.TokenizerFactory;
+import org.apache.stanbol.enhancer.engines.entitylinking.LabelTokenizer;
+import org.apache.stanbol.enhancer.nlp.utils.LanguageConfiguration;
+import org.osgi.framework.Constants;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Service
+@Component(
+ configurationFactory=true,
+ policy=ConfigurationPolicy.REQUIRE,
+ metatype=true)
+@Properties(value={
+ @Property(name=LuceneLabelTokenizer.PROPERTY_TOKENIZER_FACTORY,value="{full-qualified-class-name}"),
+ @Property(name=LabelTokenizer.SUPPORTED_LANUAGES,value="{lang1},{lang2},!{lang3},{*}"),
+ @Property(name=Constants.SERVICE_RANKING,intValue=0)
+})
+public class LuceneLabelTokenizer implements LabelTokenizer {
+
+ Logger log = LoggerFactory.getLogger(LuceneLabelTokenizer.class);
+
+ private static final String[] EMPTY = new String[]{};
+
+ public static final String PROPERTY_TOKENIZER_FACTORY = "enhancer.engine.linking.labeltokenizer.lucene.tokenizerFactory";
+ private TokenizerFactory tokenizerFactory;
+ private LanguageConfiguration langConf = new LanguageConfiguration(SUPPORTED_LANUAGES, new String[]{});
+
+ @Activate
+ protected void activate(ComponentContext ctx) throws ConfigurationException {
+ //init the Solr TokenizerFactory
+ Object value = ctx.getProperties().get(PROPERTY_TOKENIZER_FACTORY);
+ if(value == null || value.toString().isEmpty()){
+ throw new ConfigurationException(PROPERTY_TOKENIZER_FACTORY,"The class name of the Lucene Tokemizer MUST BE configured");
+ }
+ Class<?> tokenizerFactoryClass;
+ try {
+ tokenizerFactoryClass = getClass().getClassLoader().loadClass(value.toString());
+ } catch (ClassNotFoundException e) {
+ throw new ConfigurationException(PROPERTY_TOKENIZER_FACTORY, "Unable to load the "
+ + "class for the parsed name '"+value+"'!");
+ }
+ Object factoryObject;
+ try {
+ factoryObject = tokenizerFactoryClass.newInstance();
+ } catch (InstantiationException e) {
+ throw new ConfigurationException(PROPERTY_TOKENIZER_FACTORY, "Unable to instantiate the "
+ + "class '"+tokenizerFactoryClass+"'!", e);
+ } catch (IllegalAccessException e) {
+ throw new ConfigurationException(PROPERTY_TOKENIZER_FACTORY, "Unable to instantiate the "
+ + "class '"+tokenizerFactoryClass+"'!", e);
+ }
+
+ if(factoryObject instanceof TokenizerFactory){
+ tokenizerFactory = (TokenizerFactory)factoryObject;
+ tokenizerFactory.init(Collections.singletonMap("luceneMatchVersion", Version.LUCENE_36.toString()));
+ } else {
+ throw new ConfigurationException(PROPERTY_TOKENIZER_FACTORY, "The parsed class '"
+ + tokenizerFactoryClass +"' is not assignable to "+TokenizerFactory.class);
+ }
+ //init the language configuration
+ value = ctx.getProperties().get(LabelTokenizer.SUPPORTED_LANUAGES);
+ if(value == null){
+ throw new ConfigurationException(LabelTokenizer.SUPPORTED_LANUAGES, "The language "
+ + "configuration MUST BE present!");
+ }
+ langConf.setConfiguration(ctx.getProperties());
+ }
+
+ @Deactivate
+ protected void deactivate(ComponentContext ctx){
+ tokenizerFactory = null;
+ langConf.setDefault();
+ }
+
+ @Override
+ public String[] tokenize(String label, String language) {
+ if(label == null){
+ throw new IllegalArgumentException("The parsed label MUST NOT be NULL!");
+ }
+ if((language == null && langConf.useWildcard()) ||
+ langConf.isLanguage(language)){
+ if(label.isEmpty()){
+ return EMPTY;
+ }
+ Tokenizer tokenizer = tokenizerFactory.create(new StringReader(label));
+ List<String> tokens = new ArrayList<String>(8);
+ try {
+ while(tokenizer.incrementToken()){
+ OffsetAttribute offset = tokenizer.addAttribute(OffsetAttribute.class);
+ tokens.add(label.substring(offset.startOffset(), offset.endOffset()));
+ }
+ } catch (IOException e) {
+ log.error("IOException while reading from a StringReader :(",e);
+ return null;
+ }
+ return tokens.toArray(new String[tokens.size()]);
+ } else {
+ log.trace("Language {} not configured to be supported",language);
+ return null;
+ }
+
+ }
+
+}
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/resources/OSGI-INF/metatype/metatype.properties
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/resources/OSGI-INF/metatype/metatype.properties?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/resources/OSGI-INF/metatype/metatype.properties (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/main/resources/OSGI-INF/metatype/metatype.properties Fri Dec 21 10:04:04 2012
@@ -0,0 +1,128 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+stanbol.enhancer.engine.name.name=Name
+stanbol.enhancer.engine.name.description=The name of the enhancement engine as \
+used in the RESTful interface '/engine/<name>'
+
+service.ranking.name=Ranking
+service.ranking.description=If two enhancement engines with the same name are active the \
+one with the higher ranking will be used to process parsed content items.
+
+
+#===============================================================================
+#Properties and Options used to configure
+#===============================================================================
+org.apache.stanbol.enhancer.engines.entityhublinking.EntityhubLinkingEngine.name=Apache \
+Stanbol Enhancer Engine: Entityhub Linking
+org.apache.stanbol.enhancer.engines.entityhublinking.EntityhubLinkingEngine.description=An engine \
+that links NLP processed text with Entities of the Entityhub Sites.
+
+enhancer.engines.linking.entityhub.siteId.name=Referenced Site
+enhancer.engines.linking.entityhub.siteId.description=The ID of the \
+Entityhub Referenced Site holding the Controlled Vocabulary (e.g. a taxonomy or just a set of \
+named entities). To match against the whole Entityhub use "entityhub" as value.
+
+enhancer.engines.linking.minSearchTokenLength.name=Min Token Length
+enhancer.engines.linking.minSearchTokenLength.description=The minimum \
+length of Tokens used to lookup Entities within the Controlled Vocabulary. This parameter is ignored \
+in case a POS (Part of Speech) tagger is available for the language of the parsed content.
+
+enhancer.engines.linking.labelField.name=Label Field
+enhancer.engines.linking.labelField.description=The field used to match \
+Entities with a mentions within the parsed text.
+
+enhancer.engines.linking.typeField.name=Type Field
+enhancer.engines.linking.typeField.description=The field used to \
+retrieve the types of matched Entities. Values of that field are expected to be URIs
+
+enhancer.engines.linking.caseSensitive.name=Case Sensitivity
+enhancer.engines.linking.caseSensitive.description=Allows to enable/disable \
+case sensitive matching
+
+enhancer.engines.linking.redirectField.name=Redirect Field
+enhancer.engines.linking.redirectField.description=Entities may \
+define redirects to other Entities (e.g. "USA"(http://dbpedia.org/resource/USA) -> \
+"United States"(http://dbpedia.org/resource/United_States). Values of this field are \
+expected to link to other entities part of the controlled vocabulary
+
+enhancer.engines.linking.suggestions.name=Suggestions
+enhancer.engines.linking.suggestions.description=The maximal \
+number of suggestions returned for a single mention.
+
+enhancer.engines.linking.minFoundTokens.name=Number of Required Tokens
+enhancer.engines.linking.minFoundTokens.description=For lookups with \
+several words (e.g. Dr Patrick Marshall) this is the minimum number of Tokens the label of an \
+entity must match to be suggested. This is only used of the label does not exactly match a part \
+of the text.
+
+enhancer.engines.linking.redirectMode.name=Redirect Mode
+enhancer.engines.linking.redirectMode.description=Defines how to \
+process redirects of Entities mentioned in the parsed content.. Three modes to deal with such \
+links are supported: Ignore redirects; Add values from redirected Entities to extracted; Follow \
+Redirects and suggest the redirected Entity instead of the extracted.
+enhancer.engines.linking.redirectMode.option.follow=Follow Redirects
+enhancer.engines.linking.redirectMode.option.addValues=Keep extracted \
+Entity, but add information of the redirected
+enhancer.engines.linking.redirectMode.option.ignore=Ignore Redirects
+
+enhancer.engines.linking.properNounsState.name=Link ProperNouns only
+enhancer.engines.linking.properNounsState.description=If activated \
+only ProperNouns will be matched against the Vocabulary. If deactivated any Noun will be matched. \
+NOTE that this parameter requires a tag of the POS TagSet to be mapped against 'olia:PorperNoun'. \
+Otherwise mapping will not work as expected.
+
+enhancer.engines.linking.processedLanguages.name=Processed Languages
+enhancer.engines.linking.processedLanguages.description=Languages to \
+process and optionally language specific configurations. Syntax "{lang};{param-name}={param-value};\
+{param-name}={param-value};...". Supported {param-name}s: "lc" - processed Lexical Categories (see \
+LexicalCategory enumeration for possible values); "pos" - processed Pos types (see Pos enumeration \
+for possible values); "tag" - processed string pos tags; "prob" - minumum probability of pos annotations.
+
+enhancer.engines.linking.defaultMatchingLanguage.name=Default Matching Language
+enhancer.engines.linking.defaultMatchingLanguage.description=The language \
+used in addition to the language detected for the analysed text to search for Entities. Typically this \
+configuration is an empty string to search for labels without any language defined, but for some data \
+sets (such as DBpedia.org) that add languages to any labels it might improve resuls to change this \
+configuration (e.g. to 'en' in the case of DBpedia.org).
+
+enhancer.engines.linking.dereference.name=Dereference Entities
+enhancer.engines.linking.dereference.description=If enabled additional \
+data for suggested Entities are included.
+
+enhancer.engines.linking.dereferenceFields.name=Fields used for dereferencing
+enhancer.engines.linking.dereferenceFields.description=If 'Dereference Entities' is enabled \
+data from this fields will be included for suggested Entities.
+
+enhancer.engines.linking.typeMappings.name=Type Mappings
+enhancer.engines.linking.typeMappings.description=This allows to add \
+additional entity-type > text-annotation-type mappings. Such mappings are used to determine the \
+'dc:type' value of the 'fise:TextAnnotation' created for extracted entities. Usage: \
+variant (a) '{uri}' short for {uri} > {uri} or (b) '{source1};{source2};..;{sourceN} > {target}'. \
+Note that a {source} may be only mapped to a single {target}. Multiple {source} types \
+can be mapped to the same {target}.
+
+enhancer.engines.linking.keywordTokenizer.name=Keyword Tokenizer
+enhancer.engines.linking.keywordTokenizer.description=This allows \
+to use a special Tokenizer for matching keywords and alpha numeric IDs. Typical language \
+specific Tokenizers tned to split such IDs in several tokens and therefore might prevent \
+a correct matching.
+
+enhancer.engines.linking.minTokenScore.name=Minimum Token Score
+enhancer.engines.linking.minTokenScore.description=The minimum score a single Token \
+of the text must match a Token of the Label so that it is considered to match. [0..1].\
+The score is calculated by comparing the matching characters from the beginning of the two \
+tokens compared to the overal size of the token. So it allows derivations at the end of the \
+of the tokens (e.g because of inflected forms of words).
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/LuceneLabelTokenizerTest.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/LuceneLabelTokenizerTest.java?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/LuceneLabelTokenizerTest.java (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/LuceneLabelTokenizerTest.java Fri Dec 21 10:04:04 2012
@@ -0,0 +1,61 @@
+package org.apache.stanbol.enhancer.engines.entityhublinking.labeltokenizer.lucene;
+
+import java.util.Dictionary;
+import java.util.Hashtable;
+
+import org.apache.stanbol.enhancer.engines.entitylinking.LabelTokenizer;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+
+public class LuceneLabelTokenizerTest {
+
+
+ private static final Object TOKENIZER_FACTORY_CLASS = "org.apache.solr.analysis.WhitespaceTokenizerFactory";
+ private static LuceneLabelTokenizer luceneLabelTokenizer;
+
+ @BeforeClass
+ public static void init() throws ConfigurationException {
+ Dictionary<String,Object> config = new Hashtable<String,Object>();
+ config.put(LuceneLabelTokenizer.PROPERTY_TOKENIZER_FACTORY, TOKENIZER_FACTORY_CLASS);
+ config.put(LabelTokenizer.SUPPORTED_LANUAGES, "en");
+ ComponentContext cc = new MockComponentContext(config);
+ luceneLabelTokenizer = new LuceneLabelTokenizer();
+ luceneLabelTokenizer.activate(cc);
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testNullLabel(){
+ luceneLabelTokenizer.tokenize(null, "en");
+ }
+ @Test
+ public void testNullLanguate(){
+ Assert.assertNull(luceneLabelTokenizer.tokenize("test", null));
+ }
+ @Test
+ public void testUnsupportedLanguage(){
+ Assert.assertNull(luceneLabelTokenizer.tokenize("test", "de"));
+ }
+ @Test
+ public void testLuceneLabelTokenizer(){
+ String label = "This is only a Test";
+ String[] expected = label.split(" ");
+ String[] tokens = luceneLabelTokenizer.tokenize(label, "en");
+ Assert.assertNotNull(tokens);
+ Assert.assertArrayEquals(expected, tokens);
+ }
+ @Test
+ public void testEmptyLabel(){
+ String[] tokens = luceneLabelTokenizer.tokenize("", "en");
+ Assert.assertNotNull(tokens);
+ Assert.assertTrue(tokens.length == 0);
+ }
+
+ @AfterClass
+ public static void close(){
+ luceneLabelTokenizer.deactivate(null);
+ }
+}
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/MockComponentContext.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/MockComponentContext.java?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/MockComponentContext.java (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-lucene/src/test/java/org/apache/stanbol/enhancer/engines/entityhublinking/labeltokenizer/lucene/MockComponentContext.java Fri Dec 21 10:04:04 2012
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entityhublinking.labeltokenizer.lucene;
+
+import java.io.File;
+import java.io.InputStream;
+import java.util.Dictionary;
+import java.util.Hashtable;
+
+import org.osgi.framework.Bundle;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.BundleException;
+import org.osgi.framework.BundleListener;
+import org.osgi.framework.Filter;
+import org.osgi.framework.FrameworkListener;
+import org.osgi.framework.InvalidSyntaxException;
+import org.osgi.framework.ServiceListener;
+import org.osgi.framework.ServiceReference;
+import org.osgi.framework.ServiceRegistration;
+import org.osgi.service.component.ComponentContext;
+import org.osgi.service.component.ComponentInstance;
+
+public class MockComponentContext implements ComponentContext {
+
+ protected final Dictionary<String, Object> properties;
+
+ public MockComponentContext() {
+ properties = new Hashtable<String, Object>();
+ }
+
+ public MockComponentContext(Dictionary<String, Object> properties) {
+ this.properties = properties;
+ }
+
+ public void disableComponent(String name) {
+ }
+
+ public void enableComponent(String name) {
+ }
+
+ public BundleContext getBundleContext() {
+ return new BundleContext() {
+
+ @Override
+ public boolean ungetService(ServiceReference reference) {
+ return false;
+ }
+
+ @Override
+ public void removeServiceListener(ServiceListener listener) {
+ }
+
+ @Override
+ public void removeFrameworkListener(FrameworkListener listener) {
+ }
+
+ @Override
+ public void removeBundleListener(BundleListener listener) {
+ }
+
+ @Override
+ public ServiceRegistration registerService(String clazz,
+ Object service, Dictionary properties) {
+ return null;
+ }
+
+ @Override
+ public ServiceRegistration registerService(String[] clazzes,
+ Object service, Dictionary properties) {
+ return null;
+ }
+
+ @Override
+ public Bundle installBundle(String location, InputStream input)
+ throws BundleException {
+ return null;
+ }
+
+ @Override
+ public Bundle installBundle(String location) throws BundleException {
+ return null;
+ }
+
+ @Override
+ public ServiceReference[] getServiceReferences(String clazz,
+ String filter) throws InvalidSyntaxException {
+ return null;
+ }
+
+ @Override
+ public ServiceReference getServiceReference(String clazz) {
+ return null;
+ }
+
+ @Override
+ public Object getService(ServiceReference reference) {
+ return null;
+ }
+
+ @Override
+ public String getProperty(String key) {
+ return null;
+ }
+
+ @Override
+ public File getDataFile(String filename) {
+ return new File(System.getProperty("java.io.tmpdir"));
+ }
+
+ @Override
+ public Bundle[] getBundles() {
+ return null;
+ }
+
+ @Override
+ public Bundle getBundle(long id) {
+ return null;
+ }
+
+ @Override
+ public Bundle getBundle() {
+ return null;
+ }
+
+ @Override
+ public ServiceReference[] getAllServiceReferences(String clazz,
+ String filter) throws InvalidSyntaxException {
+ return null;
+ }
+
+ @Override
+ public Filter createFilter(String filter)
+ throws InvalidSyntaxException {
+ return null;
+ }
+
+ @Override
+ public void addServiceListener(ServiceListener listener,
+ String filter) throws InvalidSyntaxException {
+
+ }
+
+ @Override
+ public void addServiceListener(ServiceListener listener) {
+ }
+
+ @Override
+ public void addFrameworkListener(FrameworkListener listener) {
+ }
+
+ @Override
+ public void addBundleListener(BundleListener listener) {
+ }
+ };
+ }
+
+ public ComponentInstance getComponentInstance() {
+ return null;
+ }
+
+ public Dictionary<String, Object> getProperties() {
+ return properties;
+ }
+
+ public ServiceReference getServiceReference() {
+ return null;
+ }
+
+ public Bundle getUsingBundle() {
+ return null;
+ }
+
+ public Object locateService(String name) {
+ return null;
+ }
+
+ public Object locateService(String name, ServiceReference reference) {
+ return null;
+ }
+
+ public Object[] locateServices(String name) {
+ return null;
+ }
+
+}
Propchange: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Fri Dec 21 10:04:04 2012
@@ -0,0 +1,7 @@
+.classpath
+
+.project
+
+.settings
+
+target
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/pom.xml
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/pom.xml?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/pom.xml (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/pom.xml Fri Dec 21 10:04:04 2012
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.parent</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ <relativePath>../../../enhancer/parent</relativePath>
+ </parent>
+
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer.opennlp</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ <packaging>bundle</packaging>
+
+ <name>Apache Stanbol Enhancer EntityLinking LabelTokenizer OpenNLP </name>
+ <description>
+ LabelTokenizer implementation based on OpenNLP.
+ </description>
+
+ <inceptionYear>2012</inceptionYear>
+
+ <scm>
+ <connection>
+ scm:svn:http://svn.apache.org/repos/asf/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp
+ </connection>
+ <developerConnection>
+ scm:svn:https://svn.apache.org/repos/asf/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp
+ </developerConnection>
+ <url>http://stanbol.apache.org/</url>
+ </scm>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <Import-Package>
+ org.apache.stanbol.enhancer.engines.entitylinking;version=${project.version}; provide:=true,
+ *
+ </Import-Package>
+ <Export-Package>
+ </Export-Package>
+ <Private-Package>
+ org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer.opennlp;version=${project.version}
+ </Private-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-scr-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.enhancer.engines.entitylinking.engine</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.commons.opennlp</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>org.apache.felix.scr.annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <!-- Testing -->
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.commons.stanboltools.datafileprovider</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+</project>
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/license/THIRD-PARTY.properties
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/license/THIRD-PARTY.properties?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/license/THIRD-PARTY.properties (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/license/THIRD-PARTY.properties Fri Dec 21 10:04:04 2012
@@ -0,0 +1,19 @@
+# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
+#-------------------------------------------------------------------------------
+# Already used licenses in project :
+# - Apache 2
+# - Apache License
+# - Common Development and Distribution License (CDDL) v1.0
+# - Common Public License Version 1.0
+# - ICU License
+# - MIT License
+# - The Apache Software License, Version 2.0
+#-------------------------------------------------------------------------------
+# Please fill the missing licenses for dependencies :
+#
+#
+#Wed Feb 15 19:07:08 CET 2012
+javax.servlet--servlet-api--2.4=The Apache Software License, Version 2.0
+jwnl--jwnl--1.3.3=BSD License
+org.osgi--org.osgi.compendium--4.1.0=The Apache Software License, Version 2.0
+org.osgi--org.osgi.core--4.1.0=The Apache Software License, Version 2.0
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/OpenNlpLabelTokenizer.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/OpenNlpLabelTokenizer.java?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/OpenNlpLabelTokenizer.java (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/OpenNlpLabelTokenizer.java Fri Dec 21 10:04:04 2012
@@ -0,0 +1,92 @@
+package org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer.opennlp;
+
+
+import opennlp.tools.tokenize.Tokenizer;
+import opennlp.tools.tokenize.TokenizerME;
+import opennlp.tools.tokenize.TokenizerModel;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Properties;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.Service;
+import org.apache.stanbol.commons.opennlp.OpenNLP;
+import org.apache.stanbol.enhancer.engines.entitylinking.LabelTokenizer;
+import org.apache.stanbol.enhancer.nlp.utils.LanguageConfiguration;
+import org.osgi.framework.Constants;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implementation of a LabelTokenizer based on OpenNLP that
+ * allows to configure custom Tokenizer models for specific
+ * languages.<p>
+ * <b>NOTE:</b> This component requires the optional dependency
+ * to <code>o.a.stanbol.commons.opennlp</code> as it dependes
+ * on the the {@link OpenNLP} service to retrieve {@link Tokenizer}
+ * and load {@link TokenizerModel}s.<p>
+ * This component registers itself with a service ranking of <code>-100</code>
+ * @author Rupert Westenthaler
+ *
+ */
+@Component(immediate=true)
+@Service
+@Properties(value={
+ @Property(name=Constants.SERVICE_RANKING,intValue=-100),
+ @Property(name=LabelTokenizer.SUPPORTED_LANUAGES,value="*")})
+public class OpenNlpLabelTokenizer implements LabelTokenizer {
+
+ private final Logger log = LoggerFactory.getLogger(OpenNlpLabelTokenizer.class);
+
+ public static final String PARAM_MODEL = "model";
+
+ @Reference
+ protected OpenNLP openNlp;
+
+ public OpenNlpLabelTokenizer(){}
+
+ public OpenNlpLabelTokenizer(OpenNLP openNLP){
+ this.openNlp = openNLP;
+ }
+
+ private LanguageConfiguration languageConfig = new LanguageConfiguration(
+ LabelTokenizer.SUPPORTED_LANUAGES, new String[]{"*"});
+
+ @Activate
+ protected void activate(ComponentContext ctx) throws ConfigurationException {
+ languageConfig.setConfiguration(ctx.getProperties());
+ }
+ @Deactivate
+ protected void deactivate(ComponentContext ctx){
+ languageConfig.setDefault();
+ }
+
+ @Override
+ public String[] tokenize(String label, String language) {
+ if(label == null){
+ throw new IllegalArgumentException("The parsed Label MUST NOT be NULL!");
+ }
+ if(languageConfig.isLanguage(language)){
+ String modelName = languageConfig.getParameter(language, PARAM_MODEL);
+ if(modelName != null){
+ try {
+ TokenizerModel model = openNlp.getModel(TokenizerModel.class, modelName, null);
+ return new TokenizerME(model).tokenize(label);
+ } catch (Exception e) {
+ log.warn("Unable to load configured TokenizerModel '"+modelName
+ + "' for language '"+language
+ + "! Fallback to default Tokenizers",e);
+ }
+ }
+ //fallback to the defaults
+ return openNlp.getTokenizer(language).tokenize(label);
+ } else { //language not configured
+ return null;
+ }
+ }
+
+}
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/resources/OSGI-INF/metatype/metatype.properties
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/resources/OSGI-INF/metatype/metatype.properties?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/resources/OSGI-INF/metatype/metatype.properties (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/main/resources/OSGI-INF/metatype/metatype.properties Fri Dec 21 10:04:04 2012
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer.SimpleLabelTokenizer.name=Stanbol Enhancer \
+EntityLinking SimpleLabelTokenizer
+org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer.SimpleLabelTokenizer.description=This \
+is the default LabelTokenizer implementation. It behaves like the OpenNLP SimpleTokenizer but does not \
+have any external dependencies.
+
+org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer.OpenNlpLabelTokenizer.name=Stanbol Enhancer \
+EntityLinking LabelTokenizer using OpenNLP
+org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer.OpenNlpLabelTokenizer.description=This \
+LabelTokenizer implementation uses the OpenNLP Tokenizers API for tokenizing Entity labels processed \
+by the EntityLinkingEngine. It can be configured to load custom tokenizer models for specific \
+languages.
+
+service.ranking.name=Ranking
+service.ranking.description=If two LabelTokenizer support the same language, that the one with the \
+higher ranking is called first. Lower ranking LabelTokenizer are only used if others return NULL \
+on tokenize requests.
+
+enhancer.engines.entitylinking.labeltokenizer.languages.name=Supported Languages
+enhancer.engines.entitylinking.labeltokenizer.languages.description=The supported languages \
+of this Tokenizer. Values can use '!{lang}' to exclude languages; '{lang}' to explicitly \
+include languages and '*' as wildcard. In addition parameters are supported by using the \
+'{lang};{param}=value' syntax. Example: "!zh,de;model=my-de-tokenizer-model.zip,*" This will disable \
+processing of Chinese use the model with the name "my-de-tokenizer-model.zip" and the defaults \
+for all other languages.
\ No newline at end of file
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/ClasspathDataFileProvider.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/ClasspathDataFileProvider.java?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/ClasspathDataFileProvider.java (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/ClasspathDataFileProvider.java Fri Dec 21 10:04:04 2012
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer.opennlp;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Map;
+
+import org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** DataFileProvider that looks in our class resources */
+public class ClasspathDataFileProvider implements DataFileProvider {
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
+ public static final String RESOURCE_BASE_PATH = "org/apache/stanbol/data/opennlp/";
+
+ private final String symbolicName;
+
+ public ClasspathDataFileProvider(String bundleSymbolicName) {
+ symbolicName = bundleSymbolicName;
+ }
+
+ @Override
+ public InputStream getInputStream(String bundleSymbolicName,
+ String filename, Map<String, String> comments)
+ throws IOException {
+ final URL dataFile = getDataFile(bundleSymbolicName, filename);
+
+ // Returning null is fine - if we don't have the data file, another
+ // provider might supply it
+ return dataFile != null ? dataFile.openStream() : null;
+ }
+ @Override
+ public boolean isAvailable(String bundleSymbolicName, String filename, Map<String,String> comments) {
+ return getDataFile(bundleSymbolicName, filename) != null;
+ }
+ /**
+ * @param bundleSymbolicName
+ * @param filename
+ * @return
+ */
+ private URL getDataFile(String bundleSymbolicName, String filename) {
+ //If the symbolic name is not null check that is equals to the symbolic
+ //name used to create this classpath data file provider
+ if(bundleSymbolicName != null && symbolicName != null &&
+ !bundleSymbolicName.equals(symbolicName)) {
+ log.debug("Requested bundleSymbolicName {} does not match mine ({}), request ignored",
+ bundleSymbolicName, symbolicName);
+ return null;
+ }
+
+ // load default OpenNLP models from classpath (embedded in the defaultdata bundle)
+ final String resourcePath = RESOURCE_BASE_PATH + filename;
+ final URL dataFile = getClass().getClassLoader().getResource(resourcePath);
+ //log.debug("Resource {} found: {}", (in == null ? "NOT" : ""), resourcePath);
+ return dataFile;
+ }
+}
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/MockComponentContext.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/MockComponentContext.java?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/MockComponentContext.java (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/MockComponentContext.java Fri Dec 21 10:04:04 2012
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer.opennlp;
+
+import java.io.File;
+import java.io.InputStream;
+import java.util.Dictionary;
+import java.util.Hashtable;
+
+import org.osgi.framework.Bundle;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.BundleException;
+import org.osgi.framework.BundleListener;
+import org.osgi.framework.Filter;
+import org.osgi.framework.FrameworkListener;
+import org.osgi.framework.InvalidSyntaxException;
+import org.osgi.framework.ServiceListener;
+import org.osgi.framework.ServiceReference;
+import org.osgi.framework.ServiceRegistration;
+import org.osgi.service.component.ComponentContext;
+import org.osgi.service.component.ComponentInstance;
+
+public class MockComponentContext implements ComponentContext {
+
+ protected final Dictionary<String, Object> properties;
+
+ public MockComponentContext() {
+ properties = new Hashtable<String, Object>();
+ }
+
+ public MockComponentContext(Dictionary<String, Object> properties) {
+ this.properties = properties;
+ }
+
+ public void disableComponent(String name) {
+ }
+
+ public void enableComponent(String name) {
+ }
+
+ public BundleContext getBundleContext() {
+ return new BundleContext() {
+
+ @Override
+ public boolean ungetService(ServiceReference reference) {
+ return false;
+ }
+
+ @Override
+ public void removeServiceListener(ServiceListener listener) {
+ }
+
+ @Override
+ public void removeFrameworkListener(FrameworkListener listener) {
+ }
+
+ @Override
+ public void removeBundleListener(BundleListener listener) {
+ }
+
+ @Override
+ public ServiceRegistration registerService(String clazz,
+ Object service, Dictionary properties) {
+ return null;
+ }
+
+ @Override
+ public ServiceRegistration registerService(String[] clazzes,
+ Object service, Dictionary properties) {
+ return null;
+ }
+
+ @Override
+ public Bundle installBundle(String location, InputStream input)
+ throws BundleException {
+ return null;
+ }
+
+ @Override
+ public Bundle installBundle(String location) throws BundleException {
+ return null;
+ }
+
+ @Override
+ public ServiceReference[] getServiceReferences(String clazz,
+ String filter) throws InvalidSyntaxException {
+ return null;
+ }
+
+ @Override
+ public ServiceReference getServiceReference(String clazz) {
+ return null;
+ }
+
+ @Override
+ public Object getService(ServiceReference reference) {
+ return null;
+ }
+
+ @Override
+ public String getProperty(String key) {
+ return null;
+ }
+
+ @Override
+ public File getDataFile(String filename) {
+ return new File(System.getProperty("java.io.tmpdir"));
+ }
+
+ @Override
+ public Bundle[] getBundles() {
+ return null;
+ }
+
+ @Override
+ public Bundle getBundle(long id) {
+ return null;
+ }
+
+ @Override
+ public Bundle getBundle() {
+ return null;
+ }
+
+ @Override
+ public ServiceReference[] getAllServiceReferences(String clazz,
+ String filter) throws InvalidSyntaxException {
+ return null;
+ }
+
+ @Override
+ public Filter createFilter(String filter)
+ throws InvalidSyntaxException {
+ return null;
+ }
+
+ @Override
+ public void addServiceListener(ServiceListener listener,
+ String filter) throws InvalidSyntaxException {
+
+ }
+
+ @Override
+ public void addServiceListener(ServiceListener listener) {
+ }
+
+ @Override
+ public void addFrameworkListener(FrameworkListener listener) {
+ }
+
+ @Override
+ public void addBundleListener(BundleListener listener) {
+ }
+ };
+ }
+
+ public ComponentInstance getComponentInstance() {
+ return null;
+ }
+
+ public Dictionary<String, Object> getProperties() {
+ return properties;
+ }
+
+ public ServiceReference getServiceReference() {
+ return null;
+ }
+
+ public Bundle getUsingBundle() {
+ return null;
+ }
+
+ public Object locateService(String name) {
+ return null;
+ }
+
+ public Object locateService(String name, ServiceReference reference) {
+ return null;
+ }
+
+ public Object[] locateServices(String name) {
+ return null;
+ }
+
+}
Added: stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/OpenNlpLabelTokenizerTest.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/OpenNlpLabelTokenizerTest.java?rev=1424833&view=auto
==============================================================================
--- stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/OpenNlpLabelTokenizerTest.java (added)
+++ stanbol/trunk/enhancement-engines/entitylinking/labeltokenizer-opennlp/src/test/java/org/apache/stanbol/enhancer/engines/entitylinking/labeltokenizer/opennlp/OpenNlpLabelTokenizerTest.java Fri Dec 21 10:04:04 2012
@@ -0,0 +1,57 @@
+package org.apache.stanbol.enhancer.engines.entitylinking.labeltokenizer.opennlp;
+
+import java.util.Dictionary;
+import java.util.Hashtable;
+
+import org.apache.stanbol.commons.opennlp.OpenNLP;
+import org.apache.stanbol.enhancer.engines.entitylinking.LabelTokenizer;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+
+public class OpenNlpLabelTokenizerTest {
+
+ private static OpenNlpLabelTokenizer tokenizer;
+ private String label = "This is only a Test";
+ private String[] expected = label.split(" ");
+
+ @BeforeClass
+ public static final void init() throws ConfigurationException {
+ Dictionary<String,Object> config = new Hashtable<String,Object>();
+ config.put(LabelTokenizer.SUPPORTED_LANUAGES, "*");
+ ComponentContext cc = new MockComponentContext(config);
+ tokenizer = new OpenNlpLabelTokenizer();
+ tokenizer.openNlp = new OpenNLP(new ClasspathDataFileProvider(null));
+ tokenizer.activate(cc);
+ }
+ @Test(expected=IllegalArgumentException.class)
+ public void testNullLabel(){
+ tokenizer.tokenize(null, "en");
+ }
+ @Test
+ public void testNullLanguate(){
+ String[] tokens = tokenizer.tokenize(label, null);
+ Assert.assertNotNull(tokens);
+ Assert.assertArrayEquals(expected, tokens);
+ }
+ @Test
+ public void testTokenizer(){
+ String[] tokens = tokenizer.tokenize(label, "en");
+ Assert.assertNotNull(tokens);
+ Assert.assertArrayEquals(expected, tokens);
+ }
+ @Test
+ public void testEmptyLabel(){
+ String[] tokens = tokenizer.tokenize("", "en");
+ Assert.assertNotNull(tokens);
+ Assert.assertTrue(tokens.length == 0);
+ }
+
+ @AfterClass
+ public static void close(){
+ tokenizer.deactivate(null);
+ }
+}