You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2012/10/29 11:24:39 UTC
svn commit: r1403238 [3/3] - in
/stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction:
./
src/main/java/org/apache/stanbol/enhancer/engines/keywordextraction/engine/
src/main/java/org/apache/stanbol/enhancer/engines/keywordextract...
Modified: stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/engine/KeywordLinkingEngineTest.java
URL: http://svn.apache.org/viewvc/stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/engine/KeywordLinkingEngineTest.java?rev=1403238&r1=1403237&r2=1403238&view=diff
==============================================================================
--- stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/engine/KeywordLinkingEngineTest.java (original)
+++ stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/engine/KeywordLinkingEngineTest.java Mon Oct 29 10:24:38 2012
@@ -16,6 +16,8 @@
*/
package org.apache.stanbol.enhancer.engines.keywordextraction.engine;
+import static org.apache.stanbol.enhancer.nlp.NlpAnnotations.PHRASE_ANNOTATION;
+import static org.apache.stanbol.enhancer.nlp.NlpAnnotations.POS_ANNOTATION;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_CREATOR;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_LANGUAGE;
import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_CONFIDENCE;
@@ -32,11 +34,14 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import junit.framework.Assert;
+
import opennlp.tools.tokenize.SimpleTokenizer;
import org.apache.clerezza.rdf.core.Literal;
@@ -48,17 +53,24 @@ import org.apache.clerezza.rdf.core.UriR
import org.apache.clerezza.rdf.core.impl.PlainLiteralImpl;
import org.apache.clerezza.rdf.core.impl.TripleImpl;
import org.apache.stanbol.commons.opennlp.OpenNLP;
-import org.apache.stanbol.commons.opennlp.TextAnalyzer.TextAnalyzerConfig;
import org.apache.stanbol.enhancer.contentitem.inmemory.InMemoryContentItemFactory;
import org.apache.stanbol.enhancer.engines.keywordextraction.engine.KeywordLinkingEngine;
-import org.apache.stanbol.enhancer.engines.keywordextraction.impl.ClasspathDataFileProvider;
+import org.apache.stanbol.enhancer.engines.keywordextraction.impl.EntityLinker;
+import org.apache.stanbol.enhancer.engines.keywordextraction.impl.LinkedEntity;
+import org.apache.stanbol.enhancer.engines.keywordextraction.impl.Suggestion;
import org.apache.stanbol.enhancer.engines.keywordextraction.impl.TestSearcherImpl;
-import org.apache.stanbol.enhancer.engines.keywordextraction.linking.EntityLinker;
import org.apache.stanbol.enhancer.engines.keywordextraction.linking.EntityLinkerConfig;
import org.apache.stanbol.enhancer.engines.keywordextraction.linking.EntityLinkerConfig.RedirectProcessingMode;
-import org.apache.stanbol.enhancer.engines.keywordextraction.linking.LinkedEntity;
-import org.apache.stanbol.enhancer.engines.keywordextraction.linking.Suggestion;
-import org.apache.stanbol.enhancer.engines.keywordextraction.linking.impl.OpenNlpAnalysedContentFactory;
+import org.apache.stanbol.enhancer.engines.keywordextraction.linking.TextProcessingConfig;
+import org.apache.stanbol.enhancer.engines.keywordextraction.linking.impl.OpenNlpLabelTokenizer;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedText;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedTextFactory;
+import org.apache.stanbol.enhancer.nlp.model.AnalysedTextUtils;
+import org.apache.stanbol.enhancer.nlp.model.annotation.Value;
+import org.apache.stanbol.enhancer.nlp.phrase.PhraseTag;
+import org.apache.stanbol.enhancer.nlp.pos.LexicalCategory;
+import org.apache.stanbol.enhancer.nlp.pos.Pos;
+import org.apache.stanbol.enhancer.nlp.pos.PosTag;
import org.apache.stanbol.enhancer.servicesapi.ContentItem;
import org.apache.stanbol.enhancer.servicesapi.ContentItemFactory;
import org.apache.stanbol.enhancer.servicesapi.EngineException;
@@ -92,9 +104,12 @@ public class KeywordLinkingEngineTest {
*/
public static final String TEST_TEXT = "Dr. Patrick Marshall (1869 - November 1950) was a"
+ " geologist who lived in New Zealand and worked at the University of Otago.";
- public static final String TEST_TEXT2 = "A CBS televised debate between Australia's " +
- "candidates for Prime Minister in the upcoming US election has been rescheduled " +
- "and shortend, to avoid a clash with popular cookery sow MasterChef.";
+
+ private static AnalysedText TEST_ANALYSED_TEXT;
+
+// public static final String TEST_TEXT2 = "A CBS televised debate between Australia's " +
+// "candidates for Prime Minister in the upcoming US election has been rescheduled " +
+// "and shortend, to avoid a clash with popular cookery sow MasterChef.";
private static final ContentItemFactory ciFactory = InMemoryContentItemFactory.getInstance();
@@ -102,7 +117,6 @@ public class KeywordLinkingEngineTest {
static TestSearcherImpl searcher;
static ValueFactory factory = InMemoryValueFactory.getInstance();
- private static OpenNLP openNLP;
public static final String NAME = NamespaceEnum.rdfs+"label";
public static final String TYPE = NamespaceEnum.rdf+"type";
@@ -110,7 +124,6 @@ public class KeywordLinkingEngineTest {
@BeforeClass
public static void setUpServices() throws IOException {
- openNLP = new OpenNLP(new ClasspathDataFileProvider("DUMMY_SYMBOLIC_NAME"));
searcher = new TestSearcherImpl(NAME,SimpleTokenizer.INSTANCE);
//add some terms to the searcher
Representation rep = factory.createRepresentation("urn:test:PatrickMarshall");
@@ -153,6 +166,46 @@ public class KeywordLinkingEngineTest {
rep.addNaturalText(NAME, "University of Otago (Texas)");
rep.addReference(TYPE, OntologicalClasses.DBPEDIA_ORGANISATION.getUnicodeString());
searcher.addEntity(rep);
+
+ Value<PhraseTag> nounPhrase = Value.value(new PhraseTag("NP",LexicalCategory.Noun),1d);
+ TEST_ANALYSED_TEXT = AnalysedTextFactory.getDefaultInstance().createAnalysedText(
+ ciFactory.createBlob(new StringSource(TEST_TEXT)));
+ TEST_ANALYSED_TEXT.addSentence(0, TEST_ANALYSED_TEXT.getEnd());
+ //add some noun phrases
+ TEST_ANALYSED_TEXT.addChunk(0, "Dr. Patrick Marshall".length()).addAnnotation(PHRASE_ANNOTATION, nounPhrase);
+ TEST_ANALYSED_TEXT.addChunk(TEST_TEXT.indexOf("New Zealand"), TEST_TEXT.indexOf("New Zealand")+"New Zealand".length())
+ .addAnnotation(PHRASE_ANNOTATION, nounPhrase);
+ TEST_ANALYSED_TEXT.addChunk(TEST_TEXT.indexOf("geologist"), TEST_TEXT.indexOf("geologist")+"geologist".length())
+ .addAnnotation(PHRASE_ANNOTATION, nounPhrase);
+ TEST_ANALYSED_TEXT.addChunk(TEST_TEXT.indexOf("the University of Otago"),
+ TEST_TEXT.length()-1).addAnnotation(PHRASE_ANNOTATION, nounPhrase);
+ //add some tokens
+ TEST_ANALYSED_TEXT.addToken(0, 2).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NE",Pos.Abbreviation),1d));
+ TEST_ANALYSED_TEXT.addToken(2, 3).addAnnotation(POS_ANNOTATION, Value.value(new PosTag(".",Pos.Point),1d));
+ TEST_ANALYSED_TEXT.addToken(4, 11).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NP",Pos.ProperNoun),1d));
+ TEST_ANALYSED_TEXT.addToken(12, 20).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NP",Pos.ProperNoun),1d));
+ int start = TEST_TEXT.indexOf("(1869 - November 1950)");
+ TEST_ANALYSED_TEXT.addToken(start,start+1).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("(",Pos.OpenBracket),1d));
+ TEST_ANALYSED_TEXT.addToken(start+1,start+5).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NUM",Pos.Numeral),1d));
+ TEST_ANALYSED_TEXT.addToken(start+6,start+7).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("-",Pos.Hyphen),1d));
+ TEST_ANALYSED_TEXT.addToken(start+8,start+16).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NE",Pos.CommonNoun),1d));
+ TEST_ANALYSED_TEXT.addToken(start+17,start+21).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NUM",Pos.Numeral),1d));
+ TEST_ANALYSED_TEXT.addToken(start+21,start+22).addAnnotation(POS_ANNOTATION, Value.value(new PosTag(")",Pos.CloseBracket),1d));
+
+ start = TEST_TEXT.indexOf("geologist");
+ TEST_ANALYSED_TEXT.addToken(start,start+9).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NE",Pos.CommonNoun),1d));
+
+ start = TEST_TEXT.indexOf("New Zealand");
+ TEST_ANALYSED_TEXT.addToken(start,start+3).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NE",Pos.CommonNoun),1d));
+ TEST_ANALYSED_TEXT.addToken(start+4,start+11).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NP",Pos.ProperNoun),1d));
+
+ start = TEST_TEXT.indexOf("the University of Otago");
+ TEST_ANALYSED_TEXT.addToken(start,start+3).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("ART",Pos.Article),1d));
+ TEST_ANALYSED_TEXT.addToken(start+4,start+14).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NE",Pos.CommonNoun),1d));
+ TEST_ANALYSED_TEXT.addToken(start+15,start+17).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("OF",LexicalCategory.PronounOrDeterminer),1d));
+ TEST_ANALYSED_TEXT.addToken(start+18,start+23).addAnnotation(POS_ANNOTATION, Value.value(new PosTag("NP",Pos.ProperNoun),1d));
+ TEST_ANALYSED_TEXT.addToken(start+23,start+24).addAnnotation(POS_ANNOTATION, Value.value(new PosTag(".",Pos.Point),1d));
+
}
@Before
@@ -172,17 +225,19 @@ public class KeywordLinkingEngineTest {
}
/**
* This tests the EntityLinker functionality (if the expected Entities
- * are linked)
+ * are linked). In this case with the default configurations for
+ * {@link LexicalCategory#Noun}.
* @throws Exception
*/
@Test
- public void testTaxonomyLinker() throws Exception {
- OpenNlpAnalysedContentFactory acf = OpenNlpAnalysedContentFactory.getInstance(openNLP,
- new TextAnalyzerConfig());
+ public void testEntityLinkerWithNouns() throws Exception {
+ TextProcessingConfig tpc = new TextProcessingConfig();
+ tpc.setProcessedLexicalCategories(KeywordLinkingEngine.DEFAULT_PROCESSED_LEXICAL_CATEGORIES);
+ tpc.setProcessedPos(Collections.EMPTY_SET);
EntityLinkerConfig config = new EntityLinkerConfig();
config.setRedirectProcessingMode(RedirectProcessingMode.FOLLOW);
- EntityLinker linker = new EntityLinker(
- acf.create(TEST_TEXT,"en"), searcher, config);
+ EntityLinker linker = new EntityLinker(TEST_ANALYSED_TEXT,"en",
+ tpc, searcher, config, new OpenNlpLabelTokenizer(null));
linker.process();
Map<String,List<String>> expectedResults = new HashMap<String,List<String>>();
expectedResults.put("Patrick Marshall", new ArrayList<String>(
@@ -193,10 +248,45 @@ public class KeywordLinkingEngineTest {
Arrays.asList("urn:test:NewZealand")));
expectedResults.put("University of Otago", new ArrayList<String>(
Arrays.asList("urn:test:UniversityOfOtago","urn:test:UniversityOfOtago_Texas")));
+ validateEntityLinkerResults(linker, expectedResults);
+ }
+ /**
+ * This tests the EntityLinker functionality (if the expected Entities
+ * are linked). In this case with the default configurations for
+ * {@link Pos#ProperNoun}.
+ * @throws Exception
+ */
+ @Test
+ public void testEntityLinkerWithProperNouns() throws Exception {
+ TextProcessingConfig tpc = new TextProcessingConfig();
+ tpc.setProcessedLexicalCategories(Collections.EMPTY_SET);
+ tpc.setProcessedPos(KeywordLinkingEngine.DEFAULT_PROCESSED_POS_TYPES);
+ EntityLinkerConfig config = new EntityLinkerConfig();
+ config.setRedirectProcessingMode(RedirectProcessingMode.FOLLOW);
+ EntityLinker linker = new EntityLinker(TEST_ANALYSED_TEXT,"en",
+ tpc, searcher, config, new OpenNlpLabelTokenizer(null));
+ linker.process();
+ Map<String,List<String>> expectedResults = new HashMap<String,List<String>>();
+ expectedResults.put("Patrick Marshall", new ArrayList<String>(
+ Arrays.asList("urn:test:PatrickMarshall")));
+ //Geologist is a common noun and MUST NOT be found
+ //expectedResults.put("geologist", new ArrayList<String>(
+ // Arrays.asList("urn:test:redirect:Geologist"))); //the redirected entity
+ expectedResults.put("New Zealand", new ArrayList<String>(
+ Arrays.asList("urn:test:NewZealand")));
+ expectedResults.put("University of Otago", new ArrayList<String>(
+ Arrays.asList("urn:test:UniversityOfOtago","urn:test:UniversityOfOtago_Texas")));
+ validateEntityLinkerResults(linker, expectedResults);
+ }
+ private void validateEntityLinkerResults(EntityLinker linker, Map<String,List<String>> expectedResults) {
+ log.info("---------------------");
+ log.info("- Validating Results-");
+ log.info("---------------------");
for(LinkedEntity linkedEntity : linker.getLinkedEntities().values()){
+ log.info("> LinkedEntity {}",linkedEntity);
List<String> expectedSuggestions = expectedResults.remove(linkedEntity.getSelectedText());
- assertNotNull("LinkedEntity "+linkedEntity.getSelectedText()+
- "is not an expected Result (or was found twice)", expectedSuggestions);
+ assertNotNull("LinkedEntity '"+linkedEntity.getSelectedText()+
+ "' is not an expected Result (or was found twice)", expectedSuggestions);
linkedEntity.getSuggestions().iterator();
assertEquals("Number of suggestions "+linkedEntity.getSuggestions().size()+
" != number of expected suggestions "+expectedSuggestions.size()+
@@ -217,6 +307,8 @@ public class KeywordLinkingEngineTest {
score = suggestion.getScore();
}
}
+ Assert.assertTrue("The expected Result(s) "+expectedResults+" wehre not found",
+ expectedResults.isEmpty());
}
/**
* This tests if the Enhancements created by the Engine confirm to the
@@ -228,12 +320,14 @@ public class KeywordLinkingEngineTest {
public void testEngine() throws IOException, EngineException {
EntityLinkerConfig linkerConfig = new EntityLinkerConfig();
linkerConfig.setRedirectProcessingMode(RedirectProcessingMode.FOLLOW);
- KeywordLinkingEngine engine = KeywordLinkingEngine.createInstance(openNLP, searcher, new TextAnalyzerConfig(),
- linkerConfig);
+ KeywordLinkingEngine engine = KeywordLinkingEngine.createInstance(searcher, new TextProcessingConfig(),
+ linkerConfig, new OpenNlpLabelTokenizer());
engine.referencedSiteName = TEST_REFERENCED_SITE_NAME;
ContentItem ci = ciFactory.createContentItem(new StringSource(TEST_TEXT));
//tells the engine that this is an English text
ci.getMetadata().add(new TripleImpl(ci.getUri(), DC_LANGUAGE, new PlainLiteralImpl("en")));
+ //and add the AnalysedText instance used for this test
+ ci.addPart(AnalysedText.ANALYSED_TEXT_URI, TEST_ANALYSED_TEXT);
//compute the enhancements
engine.computeEnhancements(ci);
//validate the enhancement results
Modified: stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/impl/TestSearcherImpl.java
URL: http://svn.apache.org/viewvc/stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/impl/TestSearcherImpl.java?rev=1403238&r1=1403237&r2=1403238&view=diff
==============================================================================
--- stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/impl/TestSearcherImpl.java (original)
+++ stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/java/org/apache/stanbol/enhancer/engines/keywordextraction/impl/TestSearcherImpl.java Mon Oct 29 10:24:38 2012
@@ -74,7 +74,7 @@ public class TestSearcherImpl implements
public Collection<? extends Representation> lookup(String field,
Set<String> includeFields,
List<String> search,
- String... languages) throws IllegalStateException {
+ String[] languages,Integer numResults) throws IllegalStateException {
if(field.equals(nameField)){
//we do not need sorting
//Representation needs to implement equals, therefore results filters multiple matches
Added: stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/resources/log4j.properties
URL: http://svn.apache.org/viewvc/stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/resources/log4j.properties?rev=1403238&view=auto
==============================================================================
--- stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/resources/log4j.properties (added)
+++ stanbol/branches/stanbol-nlp-processing/enhancer/engines/keywordextraction/src/test/resources/log4j.properties Mon Oct 29 10:24:38 2012
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Root logger option
+log4j.rootLogger=INFO, stdout
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n
+log4j.logger.org.apache.stanbol.enhancer.engines.keywordextraction=DEBUG
\ No newline at end of file