You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2013/10/29 11:53:42 UTC
svn commit: r1536650 -
/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/
Author: markg
Date: Tue Oct 29 10:53:42 2013
New Revision: 1536650
URL: http://svn.apache.org/r1536650
Log:
OPENNLP-611
GeoHashBinScorer has java 1.7 dependant objects.
Added:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryProximityScorer.java
- copied, changed from r1534840, opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityScorer.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatchScorer.java
- copied, changed from r1534840, opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatcher.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeohashBinScorer.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/LinkedEntityScorer.java
Removed:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatcher.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityScorer.java
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/BaseEntityLinker.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryContext.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinker.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinkerFactory.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityLinker.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazLinkable.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazLinkable.java
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/BaseEntityLinker.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/BaseEntityLinker.java?rev=1536650&r1=1536649&r2=1536650&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/BaseEntityLinker.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/BaseEntityLinker.java Tue Oct 29 10:53:42 2013
@@ -17,7 +17,6 @@ package opennlp.tools.entitylinker;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import opennlp.tools.entitylinker.domain.BaseLink;
import opennlp.tools.entitylinker.domain.LinkedSpan;
@@ -33,189 +32,40 @@ public abstract class BaseEntityLinker {
/**
* Cache of linkers
*/
- protected Map<String, List<EntityLinker>> linkerMap = new HashMap<String, List<EntityLinker>>();
+
+ protected Map<String, EntityLinker> singleLinkerMap = new HashMap<String, EntityLinker>();
/**
* Sets the LinkerMap to empty
*/
protected void clearLinkerMap() {
- linkerMap = new HashMap<String, List<EntityLinker>>();
+ singleLinkerMap = new HashMap<>();
}
- /**
- *
- * @param entitytypes the list of types (to corresponding properties keys) to
- * get linkers for
- * @param docText the document text
- * @param sentences the sentence spans that correspond to the doc text
- * @param tokens the token spans that correspond to one of the sentences
- * @param nameSpans the name spans that correspond to the tokens
- * @param properties the EntityLinkerProperties file with the proper
- * configuration
- * @return
- */
- protected ArrayList<LinkedSpan<BaseLink>> getAggregatedLinkedSpans(String[] entitytypes, String docText, Span[] sentences, Span[] tokens,
- Span[] nameSpans, EntityLinkerProperties properties) {
-
- ArrayList<LinkedSpan<BaseLink>> outLinkedSpans = new ArrayList<LinkedSpan<BaseLink>>();
- for (String type : entitytypes) {
- List<EntityLinker> linkers = getInstances(type, properties);
- for (EntityLinker linker : linkers) {
- outLinkedSpans.addAll(linker.find(docText, sentences, tokens, nameSpans));
- }
- }
- return outLinkedSpans;
- }
-
- /**
- *
- * @param docText the document text
- * @param sentences the sentence spans that correspond to the doc text
- * @param tokens the token spans that correspond to one of the
- * sentences
- * @param nameSpans the name spans that correspond to the tokens
- * @param sentenceIndex the index to the sentence span that the tokens[]
- * Span[] corresponds to
- * @param properties the EntityLinkerProperties file with the proper
- * configuration
- * @return
- */
- public ArrayList<LinkedSpan<BaseLink>> getLinkedSpans(String docText, Span[] sentences, Span[] tokens,
+ public ArrayList<LinkedSpan<BaseLink>> link(String docText, Span[] sentences, Span[] tokens,
Span[] nameSpans, int sentenceIndex, EntityLinkerProperties properties) {
ArrayList<LinkedSpan<BaseLink>> outLinkedSpans = new ArrayList<LinkedSpan<BaseLink>>();
if (nameSpans.length == 0 || nameSpans == null) {
return outLinkedSpans;
}
- List<EntityLinker> linkers;
- boolean multiType = isMultitype(nameSpans);
- if (multiType) {
- for (Span s : nameSpans) {
- linkers = getInstances(s.getType(), properties);
- for (EntityLinker linker : linkers) {
- outLinkedSpans.addAll(linker.find(docText, sentences, tokens, nameSpans, sentenceIndex));
- }
- }
- } else {
- linkers = getInstances(nameSpans[0].getType(), properties);
- for (Span s : nameSpans) {
- for (EntityLinker linker : linkers) {
- outLinkedSpans.addAll(linker.find(docText, sentences, tokens, nameSpans, sentenceIndex));
- }
- }
- }
- return outLinkedSpans;
- }
-
- /**
- *
- * @param docText the document text
- * @param sentences the sentence spans that correspond to the doc text
- * @param tokens the token spans that correspond to one of the sentences
- * @param nameSpans the name spans that correspond to the tokens
- *
- * @param properties the EntityLinkerProperties file with the proper
- * configuration
- * @return
- */
- public ArrayList<LinkedSpan<BaseLink>> getLinkedSpans(String docText, Span[] sentences, Span[] tokens,
- Span[] nameSpans, EntityLinkerProperties properties) {
- ArrayList<LinkedSpan<BaseLink>> outLinkedSpans = new ArrayList<LinkedSpan<BaseLink>>();
- if (nameSpans.length == 0 || nameSpans == null) {
- return outLinkedSpans;
- }
- List<EntityLinker> linkers;
- boolean multiType = isMultitype(nameSpans);
- if (multiType) {
- for (Span s : nameSpans) {
- linkers = getInstances(s.getType(), properties);
- for (EntityLinker linker : linkers) {
- outLinkedSpans.addAll(linker.find(docText, sentences, tokens, nameSpans));
- }
- }
- } else {
- linkers = getInstances(nameSpans[0].getType(), properties);
- for (Span s : nameSpans) {
- for (EntityLinker linker : linkers) {
- outLinkedSpans.addAll(linker.find(docText, sentences, tokens, nameSpans));
- }
- }
- }
- return outLinkedSpans;
- }
- /**
- *
- * @param docText the document text
- * @param sentences the sentence spans that correspond to the doc text
- * @param tokens the token strings that correspond to one of the
- * sentences
- * @param nameSpans the name spans that correspond to the tokens
- * @param sentenceIndex the index to the sentence span that the tokens[]
- * Span[] corresponds to
- * @param properties the EntityLinkerProperties file with the proper
- * configuration
- * @return
- */
- public ArrayList<LinkedSpan<BaseLink>> getLinkedSpans(String docText, Span[] sentences, String[] tokens,
- Span[] nameSpans, EntityLinkerProperties properties) {
- ArrayList<LinkedSpan<BaseLink>> outLinkedSpans = new ArrayList<LinkedSpan<BaseLink>>();
- if (nameSpans.length == 0 || nameSpans == null) {
- return outLinkedSpans;
- }
- List<EntityLinker> linkers;
- boolean multiType = isMultitype(nameSpans);
- if (multiType) {
- for (Span s : nameSpans) {
- linkers = getInstances(s.getType(), properties);
- for (EntityLinker linker : linkers) {
- outLinkedSpans.addAll(linker.find(docText, sentences, tokens, nameSpans));
- }
- }
- } else {
- linkers = getInstances(nameSpans[0].getType(), properties);
- for (Span s : nameSpans) {
- for (EntityLinker linker : linkers) {
- outLinkedSpans.addAll(linker.find(docText, sentences, tokens, nameSpans));
- }
- }
+ for (Span s : nameSpans) {
+ EntityLinker linker = getInstance(s.getType(), properties);
+ outLinkedSpans.addAll(linker.find(docText, sentences, tokens, nameSpans, sentenceIndex));
}
return outLinkedSpans;
}
- /**
- * checks to see if a list of spans contains more than one type
- *
- * @param spans
- * @return
- */
- private boolean isMultitype(Span[] spans) {
- boolean multitype = false;
- String type = spans[0].getType();
- for (int i = 1; i < spans.length; i++) {
- if (!type.equals(spans[i].getType())) {
- multitype = true;
- break;
- }
- }
- return multitype;
- }
+
- /**
- * returns instances of entitylinkers, and caches them in a map so they are
- * lazily instantiated
- *
- * @param type the entitytype
- * @param properties the entity liker properties
- * @return
- */
- private List<EntityLinker> getInstances(String type, EntityLinkerProperties properties) {
- List<EntityLinker> linkers = new ArrayList<EntityLinker>();
- if (linkerMap.containsKey(type)) {
- linkers = linkerMap.get(type);
+ private EntityLinker getInstance(String type, EntityLinkerProperties properties) {
+ EntityLinker linker = null;
+ if (singleLinkerMap.containsKey(type)) {
+ linker = singleLinkerMap.get(type);
} else {
- linkers = EntityLinkerFactory.getLinkers(type, properties);
- linkerMap.put(type, linkers);
+ linker = EntityLinkerFactory.getLinker(type, properties);
+ singleLinkerMap.put(type, linker);
}
- return linkers;
+ return linker;
}
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryContext.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryContext.java?rev=1536650&r1=1536649&r2=1536650&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryContext.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryContext.java Tue Oct 29 10:53:42 2013
@@ -41,6 +41,7 @@ public class CountryContext {
private Connection con;
private List<CountryContextEntry> countrydata;
private Map<String, Set<String>> nameCodesMap = new HashMap<String, Set<String>>();
+ private Map<String, Set<Integer>> countryMentions = new HashMap<String, Set<Integer>>();
public Map<String, Set<String>> getNameCodesMap() {
return nameCodesMap;
@@ -81,15 +82,20 @@ public class CountryContext {
return hits;
}
-/**
- * Finds mentions of countries based on a list from MySQL stored procedure called getCountryList. This method finds country mentions in documents,
- * which is an essential element of the scoring that is done for geo linkedspans. Lazily loads the list from the database.
- * @param docText the full text of the document
- * @param properties EntityLinkerProperties for getting database connection
- * @return
- */
+
+ /**
+ * Finds mentions of countries based on a list from MySQL stored procedure
+ * called getCountryList. This method finds country mentions in documents,
+ * which is an essential element of the scoring that is done for geo
+ * linkedspans. Lazily loads the list from the database.
+ *
+ * @param docText the full text of the document
+ * @param properties EntityLinkerProperties for getting database connection
+ * @return
+ */
public Map<String, Set<Integer>> regexfind(String docText, EntityLinkerProperties properties) {
- Map<String, Set<Integer>> hits = new HashMap<String, Set<Integer>>();
+ countryMentions = new HashMap<String, Set<Integer>>();
+ nameCodesMap.clear();
try {
if (con == null) {
con = getMySqlConnection(properties);
@@ -104,12 +110,12 @@ public class CountryContext {
while (rs.find()) {
Integer start = rs.start();
String hit = rs.group().toLowerCase();
- if (hits.containsKey(code)) {
- hits.get(code).add(start);
+ if (countryMentions.containsKey(code)) {
+ countryMentions.get(code).add(start);
} else {
Set<Integer> newset = new HashSet<Integer>();
newset.add(start);
- hits.put(code, newset);
+ countryMentions.put(code, newset);
}
if (!hit.equals("")) {
if (this.nameCodesMap.containsKey(hit)) {
@@ -128,14 +134,16 @@ public class CountryContext {
Logger.getLogger(CountryContext.class.getName()).log(Level.SEVERE, null, ex);
}
- //System.out.println(hits);
- return hits;
+
+ return countryMentions;
}
-/**
- * returns a unique list of country codes
- * @param hits the hits discovered
- * @return
- */
+
+ /**
+ * returns a unique list of country codes
+ *
+ * @param countryMentions the countryMentions discovered
+ * @return
+ */
public static Set<String> getCountryCodes(List<CountryContextHit> hits) {
Set<String> ccs = new HashSet<String>();
for (CountryContextHit hit : hits) {
@@ -167,12 +175,15 @@ public class CountryContext {
Connection conn = DriverManager.getConnection(url, username, password);
return conn;
}
-/**
- * reads the list from the database by calling a stored procedure getCountryList
- * @param properties
- * @return
- * @throws SQLException
- */
+
+ /**
+ * reads the list from the database by calling a stored procedure
+ * getCountryList
+ *
+ * @param properties
+ * @return
+ * @throws SQLException
+ */
private List<CountryContextEntry> getCountryData(EntityLinkerProperties properties) throws SQLException {
List<CountryContextEntry> entries = new ArrayList<CountryContextEntry>();
try {
@@ -207,4 +218,9 @@ public class CountryContext {
}
return entries;
}
+
+ public Map<String, Set<Integer>> getCountryMentions() {
+ return countryMentions;
+ }
+
}
Copied: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryProximityScorer.java (from r1534840, opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityScorer.java)
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryProximityScorer.java?p2=opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryProximityScorer.java&p1=opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityScorer.java&r1=1534840&r2=1536650&rev=1536650&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityScorer.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/CountryProximityScorer.java Tue Oct 29 10:53:42 2013
@@ -29,11 +29,18 @@ import opennlp.tools.util.Span;
/**
* Scores toponyms based on country context as well as fuzzy string matching
*/
-public class GeoEntityScorer {
+public class CountryProximityScorer implements LinkedEntityScorer<CountryContext> {
private Map<String, Set<String>> nameCodesMap;
String dominantCode = "";
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, CountryContext additionalContext) {
+
+ score(linkedSpans, additionalContext.getCountryMentions(), additionalContext.getNameCodesMap(), docText, sentenceSpans, 1000);
+
+ }
+
/**
* Assigns a score to each BaseLink in each linkedSpan's set of N best
* matches. Currently the scoring indicates the probability that the toponym
@@ -61,15 +68,6 @@ public class GeoEntityScorer {
setDominantCode(countryHits);
for (LinkedSpan<BaseLink> linkedspan : linkedData) {
- for (BaseLink link : linkedspan.getLinkedEntries()) {
- Double dice = FuzzyStringMatcher.getDiceCoefficient(linkedspan.getSearchTerm().toLowerCase().replace(" ", ""), link.getItemName().toLowerCase().replace(" ", ""), 2);
- /**
- * Since MySQL is using "boolean mode" this score will always be very
- * high. To allow more recall, change mysql to "natural language mode",
- * and this score will become more significant
- */
- link.getScoreMap().put("dice", dice);
- }
linkedspan = simpleProximityAnalysis(sentences, countryHits, linkedspan, maxAllowedDist);
}
return linkedData;
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinker.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinker.java?rev=1536650&r1=1536649&r2=1536650&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinker.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinker.java Tue Oct 29 10:53:42 2013
@@ -20,11 +20,11 @@ import opennlp.tools.util.Span;
/**
* EntityLinkers establish connections to external data to enrich extracted
- * entities. For instance, for Location entities a linker can be
- * developed to lookup each found location in a geonames gazateer. Another
- * example may be to find peoples' names and look them up in a database or active
- * directory. Intended to return n best matches for any give search, but can
- * also be implemented as deterministic
+ * entities. For instance, for Location entities a linker can be developed to
+ * lookup each found location in a geonames gazateer. Another example may be to
+ * find peoples' names and look them up in a database or active directory.
+ * Intended to return n best matches for any give search, but can also be
+ * implemented as deterministic
*
* @param <T> A type that extends Span
*
@@ -32,11 +32,31 @@ import opennlp.tools.util.Span;
public interface EntityLinker<T extends Span> {
/**
- * allows for passing properties through the EntityLinkerFactory into all impls dynamically
- * @param properties the EntityLinkerProperties object that contains properties needed by the impl
+ * allows for passing properties through the EntityLinkerFactory into all
+ * impls dynamically
+ *
+ * @param properties the EntityLinkerProperties object that contains
+ * properties needed by the impl
*/
void setEntityLinkerProperties(EntityLinkerProperties properties);
-
+
+ /**
+ * Links an entire document of named entities to an external source
+ *
+ * @param doctext the full text of the document
+ * @param sentences the list of sentences spans that correspond to the
+ * text.
+ * @param tokensBySentence a list of tokens that correspond to each sentence.
+ * The outer array refers to the sentence, the inner
+ * array is the tokens for the outer sentence. Similar in nature to Map<SentenceIndex,List<Tokens>>
+ * @param namesBySentence a list of name spans that correspond to each
+ * sentence. The outer array refers to the sentence,
+ * the inner array refers to the tokens that for the
+ * same sentence.Similar in nature to Map<SentenceIndex,List<Name Spans For This Sentence's Tokens>>
+ * @return
+ */
+ List<T> find(String doctext, Span[] sentences, String[][] tokensBySentence, Span[][] namesBySentence);
+
/**
*
* @param text the document text to be used as additional context, and to
@@ -50,8 +70,8 @@ public interface EntityLinker<T extends
/**
* Links the names that correspond to the tokens[] spans. The sentenceindex
- * can be used to get the sentence text and tokens from the text based on the sentence and token spans.
- * The text is available for additional context.
+ * can be used to get the sentence text and tokens from the text based on the
+ * sentence and token spans. The text is available for additional context.
*
* @param text the document text to be used as additional context,
* and to derive sentences and tokens String[]
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinkerFactory.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinkerFactory.java?rev=1536650&r1=1536649&r2=1536650&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinkerFactory.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinkerFactory.java Tue Oct 29 10:53:42 2013
@@ -29,6 +29,36 @@ import java.util.logging.Logger;
public class EntityLinkerFactory {
/**
+ * instantiates a single linker based on properties file configuration. The properties file supports multiple types.
+ * @param entityType the type of entity, i.e. person, organization, location
+ * @param properties the properties file that holds the configuration for entitylinkers.
+ * @return
+ */
+ public static synchronized EntityLinker getLinker(String entityType, EntityLinkerProperties properties) {
+ if (entityType == null || properties == null) {
+ throw new IllegalArgumentException("Null argument in entityLinkerFactory");
+ }
+ EntityLinker linker = null;
+ try {
+ String linkerImplFullName = properties.getProperty("linker." + entityType, GeoEntityLinker.class.getName());
+ Class theClass = Class.forName(linkerImplFullName);
+ linker = (EntityLinker) theClass.newInstance();
+ System.out.println("EntityLinker factory instantiated: " + linker.getClass().getName());
+ linker.setEntityLinkerProperties(properties);
+
+ } catch (InstantiationException ex) {
+ System.out.println("Check the entity linker properties file. The entry must be formatted as linker.<type>=<fullclassname>, i.e linker.person=org.my.company.MyPersonLinker" + ex);
+ } catch (IllegalAccessException ex) {
+ System.out.println("Check the entity linker properties file. The entry must be formatted as linker.<type>=<fullclassname>, i.e linker.person=org.my.company.MyPersonLinker" + ex);
+ } catch (ClassNotFoundException ex) {
+ System.out.println("Check the entity linker properties file. The entry must be formatted as linker.<type>=<fullclassname>, i.e linker.person=org.my.company.MyPersonLinker" + ex);
+ } catch (IOException ex) {
+ System.out.println("Check the entity linker properties file. The entry must be formatted as linker.<type>=<fullclassname>, i.e linker.person=org.my.company.MyPersonLinker" + ex);
+ }
+ return linker;
+ }
+
+ /**
* Instantiates a list of EntityLinkers based on a properties file entry that
* consists of a comma separated list of full class names. The entityType is
* used to build the key to the properties entry. the entityType will be
@@ -44,6 +74,7 @@ public class EntityLinkerFactory {
* entitylinkers
* @return *
*/
+ @Deprecated
public static synchronized List<EntityLinker> getLinkers(String entityType, EntityLinkerProperties properties) {
List<EntityLinker> linkers = new ArrayList<EntityLinker>();
try {
@@ -75,6 +106,7 @@ public class EntityLinkerFactory {
* entitylinkers
* @return
*/
+ @Deprecated
public static synchronized List<EntityLinker> getLinkers(String[] entityTypes, EntityLinkerProperties properties) {
List<EntityLinker> linkers = new ArrayList<EntityLinker>();
Copied: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatchScorer.java (from r1534840, opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatcher.java)
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatchScorer.java?p2=opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatchScorer.java&p1=opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatcher.java&r1=1534840&r2=1536650&rev=1536650&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatcher.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/FuzzyStringMatchScorer.java Tue Oct 29 10:53:42 2013
@@ -18,22 +18,41 @@ package opennlp.tools.entitylinker;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import opennlp.tools.entitylinker.domain.BaseLink;
+import opennlp.tools.entitylinker.domain.LinkedSpan;
import opennlp.tools.ngram.NGramGenerator;
+import opennlp.tools.util.Span;
/**
*
- *Generates scores for string comparisons.
+ * Generates scores for string comparisons.
*/
-public class FuzzyStringMatcher {
-/**
- * Generates a score based on an overlap of nGrams between two strings using the DiceCoefficient technique.
- *
- * @param s1 first string
- * @param s2 second string
- * @param nGrams number of chars in each gram
- * @return
- */
- public static double getDiceCoefficient(String s1, String s2, int nGrams) {
+public class FuzzyStringMatchScorer implements LinkedEntityScorer<CountryContext> {
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, CountryContext additionalContext) {
+ for (LinkedSpan<BaseLink> linkedSpan : linkedSpans) {
+ for (BaseLink link : linkedSpan.getLinkedEntries()) {
+ Double dice = getDiceCoefficient(linkedSpan.getSearchTerm().toLowerCase().replace(" ", ""), link.getItemName().toLowerCase().replace(" ", ""), 2);
+ link.getScoreMap().put("dice", dice);
+ Double ld = (double) getLevenshteinDistance(linkedSpan.getSearchTerm().toLowerCase().replace(" ", ""), link.getItemName().toLowerCase().replace(" ", ""));
+ link.getScoreMap().put("levenshtein", ld);
+ }
+ }
+
+
+ }
+
+ /**
+ * Generates a score based on an overlap of nGrams between two strings using
+ * the DiceCoefficient technique.
+ *
+ * @param s1 first string
+ * @param s2 second string
+ * @param nGrams number of chars in each gram
+ * @return
+ */
+ public double getDiceCoefficient(String s1, String s2, int nGrams) {
if (s1.equals("") || s1.equals("")) {
return 0d;
}
@@ -46,4 +65,31 @@ public class FuzzyStringMatcher {
return (2 * totcombigrams) / (s1Grams.size() + s2Grams.size());
}
+
+ private int minimum(int a, int b, int c) {
+ return Math.min(Math.min(a, b), c);
+ }
+
+ public int getLevenshteinDistance(CharSequence str1,
+ CharSequence str2) {
+ int[][] distance = new int[str1.length() + 1][str2.length() + 1];
+
+ for (int i = 0; i <= str1.length(); i++) {
+ distance[i][0] = i;
+ }
+ for (int j = 1; j <= str2.length(); j++) {
+ distance[0][j] = j;
+ }
+
+ for (int i = 1; i <= str1.length(); i++) {
+ for (int j = 1; j <= str2.length(); j++) {
+ distance[i][j] = minimum(
+ distance[i - 1][j] + 1,
+ distance[i][j - 1] + 1,
+ distance[i - 1][j - 1] + ((str1.charAt(i - 1) == str2.charAt(j - 1)) ? 0 : 1));
+ }
+ }
+
+ return distance[str1.length()][str2.length()];
+ }
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityLinker.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityLinker.java?rev=1536650&r1=1536649&r2=1536650&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityLinker.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeoEntityLinker.java Tue Oct 29 10:53:42 2013
@@ -35,7 +35,7 @@ import opennlp.tools.util.Span;
*/
public class GeoEntityLinker implements EntityLinker<LinkedSpan> {
- GeoEntityScorer scorer = new GeoEntityScorer();
+ // CountryProximityScorer scorer = new CountryProximityScorer();
private MySQLGeoNamesGazLinkable geoNamesGaz;// = new MySQLGeoNamesGazLinkable();
private MySQLUSGSGazLinkable usgsGaz;//= new MySQLUSGSGazLinkable();
private CountryContext countryContext;
@@ -45,7 +45,7 @@ public class GeoEntityLinker implements
* Flag for deciding whether to search gaz only for toponyms within countries
* that are mentioned in the document
*/
- private Boolean filterCountryContext=true;
+ private Boolean filterCountryContext = true;
public GeoEntityLinker() {
if (geoNamesGaz == null || usgsGaz == null) {
@@ -56,149 +56,77 @@ public class GeoEntityLinker implements
}
}
- public List<LinkedSpan> find(String text, Span[] sentences, String[] tokens, Span[] names) {
+ @Override
+ public List<LinkedSpan> find(String doctext, Span[] sentences, String[][] tokensBySentence, Span[][] namesBySentence) {
ArrayList<LinkedSpan> spans = new ArrayList<LinkedSpan>();
try {
if (linkerProperties == null) {
linkerProperties = new EntityLinkerProperties(new File("C:\\temp\\opennlpmodels\\entitylinker.properties"));
}
-
- countryMentions = countryContext.regexfind(text, linkerProperties);
-
+ countryMentions = countryContext.regexfind(doctext, linkerProperties);
//prioritize query
filterCountryContext = Boolean.valueOf(linkerProperties.getProperty("geoentitylinker.filter_by_country_context", "true"));
- String[] matches = Span.spansToStrings(names, tokens);
- for (int i = 0; i < matches.length; i++) {
-//nga gazateer is for other than US placenames, don't use it unless US is a mention in the document
- ArrayList<BaseLink> geoNamesEntries = new ArrayList<BaseLink>();
- if (!(countryMentions.keySet().contains("us") && countryMentions.keySet().size() == 1) || countryMentions.keySet().size() > 1) {
- geoNamesEntries = geoNamesGaz.find(matches[i], names[i], countryMentions, linkerProperties);
- }
- ArrayList<BaseLink> usgsEntries = new ArrayList<BaseLink>();
- if (countryMentions.keySet().contains("us")) {
- usgsEntries = usgsGaz.find(matches[i], names[i], countryMentions, linkerProperties);
- }
- LinkedSpan<BaseLink> geoSpan = new LinkedSpan<BaseLink>(geoNamesEntries, names[i].getStart(), names[i].getEnd());
+ for (int s = 0; s < sentences.length; s++) {
+ Span[] names = namesBySentence[s];
+ String[] tokens = tokensBySentence[s];
+ String[] matches = Span.spansToStrings(names, tokens);
- if (!usgsEntries.isEmpty()) {
- geoSpan.getLinkedEntries().addAll(usgsEntries);
- geoSpan.setSearchTerm(matches[i]);
- }
+ for (int i = 0; i < matches.length; i++) {
- if (!geoSpan.getLinkedEntries().isEmpty()) {
- geoSpan.setSearchTerm(matches[i]);
- spans.add(geoSpan);
+//nga gazateer is for other than US placenames, don't use it unless US is a mention in the document
+ ArrayList<BaseLink> geoNamesEntries = new ArrayList<BaseLink>();
+ if (!(countryMentions.keySet().contains("us") && countryMentions.keySet().size() == 1) || countryMentions.keySet().size() > 1 || countryMentions.keySet().isEmpty()) {
+ geoNamesEntries = geoNamesGaz.find(matches[i], names[i], countryMentions, linkerProperties);
+ }
+ ArrayList<BaseLink> usgsEntries = new ArrayList<BaseLink>();
+ if (countryMentions.keySet().contains("us") || countryMentions.keySet().isEmpty()) {
+ usgsEntries = usgsGaz.find(matches[i], names[i], linkerProperties);
+ }
+ LinkedSpan<BaseLink> geoSpan = new LinkedSpan<BaseLink>(geoNamesEntries, names[i].getStart(), names[i].getEnd());
+
+ if (!usgsEntries.isEmpty()) {
+ geoSpan.getLinkedEntries().addAll(usgsEntries);
+ geoSpan.setSearchTerm(matches[i]);
+ }
+
+ if (!geoSpan.getLinkedEntries().isEmpty()) {
+ geoSpan.setSearchTerm(matches[i]);
+ geoSpan.setSentenceid(s);
+ spans.add(geoSpan);
+ }
}
-
}
- //score the spans
-
- scorer.score(spans, countryMentions, countryContext.getNameCodesMap(), text, sentences, 1000);
-
- // return spans;
} catch (IOException ex) {
Logger.getLogger(GeoEntityLinker.class.getName()).log(Level.SEVERE, null, ex);
}
- return spans;
- }
-
- public List<LinkedSpan> find(String text, Span[] sentences, Span[] tokens, Span[] names) {
- ArrayList<LinkedSpan> spans = new ArrayList<LinkedSpan>();
- try {
- if (linkerProperties == null) {
- linkerProperties = new EntityLinkerProperties(new File("C:\\temp\\opennlpmodels\\entitylinker.properties"));
- }
-
- // System.out.println("getting country context");
- //hits = countryContext.find(text, linkerProperties);
- countryMentions = countryContext.regexfind(text, linkerProperties);
-
- //get the sentence text....must assume some index
- Span s = sentences[0];
- String sentence = text.substring(s.getStart(), s.getEnd());
-
- String[] stringtokens = Span.spansToStrings(tokens, sentence);
- //get the names based on the tokens
- String[] matches = Span.spansToStrings(names, stringtokens);
- for (int i = 0; i < matches.length; i++) {
- //nga gazateer is for other than US placenames, don't use it unless US is a mention in the document
- ArrayList<BaseLink> geoNamesEntries = new ArrayList<BaseLink>();
- if (!(countryMentions.keySet().contains("us") && countryMentions.keySet().size() == 1) || countryMentions.keySet().size() > 1) {
- geoNamesEntries = geoNamesGaz.find(matches[i], names[i], countryMentions, linkerProperties);
- }
- ArrayList<BaseLink> usgsEntries = new ArrayList<BaseLink>();
- if (countryMentions.keySet().contains("us")) {
- usgsEntries = usgsGaz.find(matches[i], names[i], countryMentions, linkerProperties);
- }
- LinkedSpan<BaseLink> geoSpan = new LinkedSpan<BaseLink>(geoNamesEntries, names[i].getStart(), names[i].getEnd());
-
- if (!usgsEntries.isEmpty()) {
- geoSpan.getLinkedEntries().addAll(usgsEntries);
- geoSpan.setSearchTerm(matches[i]);
- }
-
- if (!geoSpan.getLinkedEntries().isEmpty()) {
- geoSpan.setSearchTerm(matches[i]);
- spans.add(geoSpan);
- }
- }
-
- } catch (IOException ex) {
- Logger.getLogger(GeoEntityLinker.class.getName()).log(Level.SEVERE, null, ex);
+ List<LinkedEntityScorer<CountryContext>> scorers = new ArrayList<>();
+ scorers.add(new GeoHashBinScorer());
+ scorers.add(new CountryProximityScorer());
+ scorers.add(new FuzzyStringMatchScorer());
+ for (LinkedEntityScorer scorer : scorers) {
+ scorer.score(spans, doctext, sentences, countryContext);
}
- scorer.score(spans, countryMentions, countryContext.getNameCodesMap(), text, sentences, 1000);
return spans;
}
- public List<LinkedSpan> find(String text, Span[] sentences, Span[] tokens, Span[] names, int sentenceIndex) {
- ArrayList<LinkedSpan> spans = new ArrayList<LinkedSpan>();
- try {
-
- if (linkerProperties == null) {
- linkerProperties = new EntityLinkerProperties(new File("C:\\temp\\opennlpmodels\\entitylinker.properties"));
- }
-
- countryMentions = countryContext.regexfind(text, linkerProperties);
-
- Span s = sentences[sentenceIndex];
- String sentence = text.substring(s.getStart(), s.getEnd());
-
- String[] stringtokens = Span.spansToStrings(tokens, sentence);
- //get the names based on the tokens
- String[] matches = Span.spansToStrings(names, stringtokens);
-
- for (int i = 0; i < matches.length; i++) {
-//nga gazateer is for other than US placenames, don't use it unless US is a mention in the document
- ArrayList<BaseLink> geoNamesEntries = new ArrayList<BaseLink>();
- if (!(countryMentions.keySet().contains("us") && countryMentions.keySet().size() == 1) || countryMentions.keySet().size() > 1) {
- geoNamesEntries = geoNamesGaz.find(matches[i], names[i], countryMentions, linkerProperties);
- }
- ArrayList<BaseLink> usgsEntries = new ArrayList<BaseLink>();
- if (countryMentions.keySet().contains("us")) {
- usgsEntries = usgsGaz.find(matches[i], names[i], countryMentions, linkerProperties);
- }
- LinkedSpan<BaseLink> geoSpan = new LinkedSpan<BaseLink>(geoNamesEntries, names[i].getStart(), names[i].getEnd());
+ @Override
+ public void setEntityLinkerProperties(EntityLinkerProperties properties) {
+ this.linkerProperties = properties;
+ }
- if (!usgsEntries.isEmpty()) {
- geoSpan.getLinkedEntries().addAll(usgsEntries);
- geoSpan.setSearchTerm(matches[i]);
- }
+ @Override
+ public List<LinkedSpan> find(String text, Span[] sentences, Span[] tokens, Span[] nameSpans) {
+ throw new UnsupportedOperationException("The GeoEntityLinker requires the entire document for proper scoring. This method is unsupported"); //To change body of generated methods, choose Tools | Templates.
+ }
- if (!geoSpan.getLinkedEntries().isEmpty()) {
- geoSpan.setSearchTerm(matches[i]);
- geoSpan.setSentenceid(sentenceIndex);
- spans.add(geoSpan);
- }
- }
- scorer.score(spans, countryMentions, countryContext.getNameCodesMap(), text, sentences, 2000);
- } catch (IOException ex) {
- Logger.getLogger(GeoEntityLinker.class.getName()).log(Level.SEVERE, null, ex);
- }
- return spans;
+ @Override
+ public List<LinkedSpan> find(String text, Span[] sentences, Span[] tokens, Span[] nameSpans, int sentenceIndex) {
+ throw new UnsupportedOperationException("The GeoEntityLinker requires the entire document for proper scoring. This method is unsupported"); //To change body of generated methods, choose Tools | Templates.
}
- public void setEntityLinkerProperties(EntityLinkerProperties properties) {
- this.linkerProperties = properties;
+ @Override
+ public List<LinkedSpan> find(String text, Span[] sentences, String[] tokens, Span[] nameSpans) {
+ throw new UnsupportedOperationException("The GeoEntityLinker requires the entire document for proper scoring."); //To change body of generated methods, choose Tools | Templates.
}
}
Added: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeohashBinScorer.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeohashBinScorer.java?rev=1536650&view=auto
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeohashBinScorer.java (added)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/GeohashBinScorer.java Tue Oct 29 10:53:42 2013
@@ -0,0 +1,276 @@
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.entitylinker;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import opennlp.tools.entitylinker.domain.BaseLink;
+import opennlp.tools.entitylinker.domain.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ *
+ * @author Owner
+ */
+public class GeoHashBinScorer implements LinkedEntityScorer<CountryContext> {
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, CountryContext additionalContext) {
+ score( linkedSpans);
+ }
+
+ private void score(List<LinkedSpan> geospans) {
+ Map<Double, Double> latLongs = new HashMap<Double, Double>();
+
+ /**
+ * collect all the lat longs
+ */
+ for (LinkedSpan<BaseLink> ls : geospans) {
+ for (BaseLink bl : ls.getLinkedEntries()) {
+ if (bl instanceof MySQLGeoNamesGazEntry) {
+ MySQLGeoNamesGazEntry entry = (MySQLGeoNamesGazEntry) bl;
+ latLongs.put(entry.getLATITUDE(), entry.getLONGITUDE());
+ }
+ if (bl instanceof MySQLUSGSGazEntry) {
+ MySQLUSGSGazEntry entry = (MySQLUSGSGazEntry) bl;
+ latLongs.put(entry.getPrimarylatitudeDEC(), entry.getPrimarylongitudeDEC());
+ }
+ }
+ }
+
+ /**
+ * convert to geohash and add to sortedset
+ */
+ TreeSet<Long> geoHashes = new TreeSet<Long>();
+ for (Map.Entry<Double, Double> entry : latLongs.entrySet()) {
+ geoHashes.add(geoHash(entry.getKey(), entry.getValue()));
+ }
+ /**
+ * bin the points and generate a scoremap
+ */
+ Map<Long, Set<Long>> bins = bin(geoHashes);
+ Map<Long, Double> scores = getScore((TreeMap<Long, Set<Long>>) bins);
+ /**
+ * iterate over the data again and assign the score based on the bins
+ */
+ for (LinkedSpan<BaseLink> ls : geospans) {
+ for (BaseLink bl : ls.getLinkedEntries()) {
+ Long geohash = -1L;
+ Double score = 0d;
+ if (bl instanceof MySQLGeoNamesGazEntry) {
+ MySQLGeoNamesGazEntry entry = (MySQLGeoNamesGazEntry) bl;
+ geohash = geoHash(entry.getLATITUDE(), entry.getLONGITUDE());
+ }
+ if (bl instanceof MySQLUSGSGazEntry) {
+ MySQLUSGSGazEntry entry = (MySQLUSGSGazEntry) bl;
+ geohash = geoHash(entry.getPrimarylatitudeDEC(), entry.getPrimarylongitudeDEC());
+ }
+ if (scores.containsKey(geohash)) {
+ score = scores.get(geohash);
+
+ } else {
+ for (Long bin : bins.keySet()) {
+ if (bin == geohash || bins.get(bin).contains(geohash)) {
+ score = scores.get(bin);
+ break;
+ }
+ }
+ }
+ bl.getScoreMap().put("geohashbin", score);
+ }
+ }
+
+
+ }
+
+ private Long normalize(Double coordpart, Boolean isLat) {
+ Integer add = isLat ? 90 : 180;
+ coordpart = Math.abs(coordpart + add);
+ coordpart = coordpart * 1000000;
+
+ Long l = Math.round(coordpart);
+ String coord = String.valueOf(l);
+ if (coord.length() < 8) {
+ while (coord.length() < 8) {
+ coord += "0";
+ }
+ }
+ coord = coord.substring(0, 8);
+ l = Long.valueOf(coord);
+ return l;
+ }
+
+ /**
+ * interleaves a lat and a long to place the coordinate in linear sortable
+ * space for binning simplicity
+ *
+ * @param lat
+ * @param lon
+ * @return
+ */
+ private Long geoHash(double lat, double lon) {
+ Long normLat = normalize(lat, Boolean.TRUE);
+ Long normLon = normalize(lon, Boolean.FALSE);
+ String sLat = String.valueOf(normLat);
+ String sLon = String.valueOf(normLon);
+ char[] latInts = sLat.toCharArray();
+ char[] lonInts = sLon.toCharArray();
+ String geoHash = "";
+ int len = latInts.length > lonInts.length ? lonInts.length : latInts.length;
+ for (int i = 0; i < len - 1; i++) {
+ String a = String.valueOf(latInts[i]);
+ String b = String.valueOf(lonInts[i]);
+ geoHash += a + b;
+ }
+
+ return Long.valueOf(geoHash);
+ }
+
+ private Map<Long, Set<Long>> bin(TreeSet<Long> sets) {
+ ArrayList<Long> list = new ArrayList<Long>(sets);
+ ArrayList<Long> diffs = new ArrayList<Long>();
+ /**
+ * create a set of differences between the points
+ */
+ for (int i = 0; i < list.size() - 1; i++) {
+ Long n = list.get(i + 1);
+ Long v = list.get(i);
+ diffs.add(Math.abs(n - v));
+ }
+ /**
+ * generate an average "distance" between the normed points
+ */
+ Long sum = 0L;
+ for (Long l : diffs) {
+ sum += l;
+ }
+ Long avg = sum / diffs.size();
+
+ avg = avg / (long) (diffs.size() * .10);
+ /**
+ * generate break values where the disparity is greater than the average
+ */
+ TreeSet<Long> breaks = new TreeSet<Long>();
+ for (int i = 0; i < list.size() - 1; i++) {
+ Long n = list.get(i + 1);
+ Long v = list.get(i);
+ //Long percent = 100 - (v / n * 100);
+ Long diff = n - v;
+ if (diff > avg) {
+ breaks.add(v);
+ }
+ }
+ /**
+ * based on the break values, place subsets of close points into bins
+ */
+ TreeMap<Long, Set<Long>> binToAmount = new TreeMap<Long, Set<Long>>();
+ Long lastBreak = -1L;
+ for (Long br : breaks) {
+ if (lastBreak == -1L) {
+ binToAmount.put(br, sets.subSet(0L, true, br, true));
+ } else {
+ binToAmount.put(br, sets.subSet(lastBreak, false, br, true));
+ }
+ lastBreak = br;
+ }
+ lastBreak = sets.higher(lastBreak);
+ if (lastBreak != null) {
+ binToAmount.put(lastBreak, sets.subSet(lastBreak, true, sets.last(), true));
+ if (binToAmount.get(lastBreak).isEmpty()) {
+ binToAmount.get(lastBreak).add(lastBreak);
+ }
+ }
+ /**
+ * "binToAmount" is a map of the break value to all the points behind it
+ * (it's sorted), so the key is the max value of its set of values
+ */
+ return binToAmount;
+ }
+
+ /**
+ * returns a map of geohashes and their score
+ *
+ * @param binToAmount
+ * @return Map< Geohash, score>
+ */
+ private Map<Long, Double> getScore(TreeMap<Long, Set<Long>> binToAmount) {
+ TreeMap<Long, Double> ranks = new TreeMap<Long, Double>();
+ TreeMap<Long, Double> normRanks = new TreeMap<Long, Double>();
+ /**
+ * if there is only one bin return 1 as the rank for each item in the value
+ */
+ if (binToAmount.keySet().size() == 1 || binToAmount.keySet().isEmpty()) {
+ for (Long bin : binToAmount.keySet()) {
+ for (Long hash : binToAmount.get(bin)) {
+ ranks.put(bin, 1d);
+ }
+ }
+ return ranks;
+ }
+ int total = 0;
+ /**
+ * generate a total number of points
+ */
+ for (Set<Long> geohashes : binToAmount.values()) {
+ total += geohashes.size();
+ }
+ /**
+ * divide total by bin size, largest bin size gets best score, everything in
+ * that bin gets that score because it is part of that primary cluster
+ * TODO... do an extra iteration of clustering within the predominant
+ * cluster to refine the scoring or make the basis of the binning more
+ * granular than > avg
+ */
+ TreeSet<Double> rankSet = new TreeSet<Double>();
+ for (Long key : binToAmount.keySet()) {
+ int size = binToAmount.get(key).size();
+ Double rank = (double) total / size;
+ rankSet.add(rank);
+ ranks.put(key, rank);
+ }
+ /**
+ * load the final score map with normalized values
+ */
+ for (Map.Entry<Long, Double> rank : ranks.entrySet()) {
+ double norm = normalize(rank.getValue(), rankSet.first() + .1, rankSet.last() + .1);
+ double reverse = Math.abs(norm - 1);
+ double score = reverse > 1d ? 1.0 : reverse;
+ normRanks.put(rank.getKey(), score);
+ }
+
+ return normRanks;
+ }
+
+ /**
+ * transposes a number in a range to a double between 0 and 1
+ *
+ * @param valueToNormalize the value to be normalized (placed within a new
+ * range of 0-1)
+ * @param minimum the min of the current range
+ * @param maximum the max of the current range
+ * @return
+ */
+ private Double normalize(Double valueToNormalize, Double minimum, Double maximum) {
+ Double d = (double) ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
+ d = d == null ? 0d : d;
+ return d;
+ }
+}
Added: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/LinkedEntityScorer.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/LinkedEntityScorer.java?rev=1536650&view=auto
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/LinkedEntityScorer.java (added)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/LinkedEntityScorer.java Tue Oct 29 10:53:42 2013
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.entitylinker;
+
+import java.util.List;
+import opennlp.tools.entitylinker.domain.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ * Structure for scoring linked entities. The Map logically represents a pair :
+ * "Score type" to the "actual Score."
+ */
+public interface LinkedEntityScorer<T> {
+
+/**
+ * Scores a collection of linked entities. Implementations should populate the scoreMap in the list of BaseLink for each linkedSpan
+ * @param linkedSpans the spans that have been linked to some external source and have all the data they need to be scored
+ * @param docText the full text of the document.
+ * @param sentenceSpans the sentence spans the correspond to the document text
+ * @param additionalContext any additional data required to perform the scoring operation
+ * @return void
+ */
+ void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, T additionalContext);
+}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazLinkable.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazLinkable.java?rev=1536650&r1=1536649&r2=1536650&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazLinkable.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLGeoNamesGazLinkable.java Tue Oct 29 10:53:42 2013
@@ -36,33 +36,49 @@ public final class MySQLGeoNamesGazLinka
filterCountryContext = Boolean.valueOf(properties.getProperty("geoentitylinker.filter_by_country_context", "false"));
- String thresh = properties.getProperty("mysqlusgsgazscorethresh", "200");
+ String thresh = properties.getProperty("geonames.gaz.rowsreturned", "200");
int threshhold = -1;
if (!thresh.matches("[azAZ]")) {
threshhold = Integer.valueOf(thresh);
}
- returnlocs.addAll(this.searchGaz(locationText, threshhold, countryHits.keySet(), properties));
-
-
+ /**
+ * Because we need equal amount of candidate toponyms from each country
+ * code, we iterate over the set of codes and do a search with the name
+ * for each code returning n rows based on threah
+ */
+ for (String code : countryHits.keySet()) {
+ returnlocs.addAll(this.searchGaz(locationText, threshhold, code, properties));
+ }
} catch (Exception ex) {
Logger.getLogger(MySQLUSGSGazLinkable.class.getName()).log(Level.SEVERE, null, ex);
}
return returnlocs;
}
- protected Connection getMySqlConnection(EntityLinkerProperties property) throws Exception {
+ private Connection getMySqlConnection(EntityLinkerProperties property) throws Exception {
// EntityLinkerProperties property = new EntityLinkerProperties(new File("c:\\temp\\opennlpmodels\\entitylinker.properties"));
- String driver = property.getProperty("mysql.driver", "org.gjt.mm.mysql.Driver");
- String url = property.getProperty("mysql.url", "jdbc:mysql://localhost:3306/world");
- String username = property.getProperty("mysql.username", "root");
- String password = property.getProperty("mysql.password", "?");
+ String driver = property.getProperty("db.driver", "org.gjt.mm.mysql.Driver");
+ String url = property.getProperty("db.url", "jdbc:mysql://localhost:3306/world");
+ String username = property.getProperty("db.username", "root");
+ String password = property.getProperty("db.password", "?");
Class.forName(driver);
Connection conn = DriverManager.getConnection(url, username, password);
return conn;
}
- public ArrayList<MySQLGeoNamesGazEntry> searchGaz(String searchString, int matchthresh, Set<String> countryCodes, EntityLinkerProperties properties) throws SQLException, Exception {
+ /**
+ *
+ * @param searchString the name to look up in the gazateer
+ * @param rowsReturned number of rows to return
+ * @param code the two digit country code
+ * @param properties EntityLinkerProperties that identifies the database
+ * connection properties
+ * @return
+ * @throws SQLException
+ * @throws Exception
+ */
+ public ArrayList<MySQLGeoNamesGazEntry> searchGaz(String searchString, int rowsReturned, String code, EntityLinkerProperties properties) throws SQLException, Exception {
if (con.isClosed()) {
con = getMySqlConnection(properties);
@@ -70,11 +86,10 @@ public final class MySQLGeoNamesGazLinka
CallableStatement cs;
cs = con.prepareCall("CALL `search_geonames`(?, ?, ?)");
cs.setString(1, this.format(searchString));
- cs.setInt(2, matchthresh);
+ cs.setInt(2, rowsReturned);
if (filterCountryContext) {
- cs.setString(3, CountryContext.getCountryCodeCSV(countryCodes));
+ cs.setString(3, code);
} else {
- //database stored procedure handles empty string
cs.setString(3, "");
}
@@ -90,33 +105,19 @@ public final class MySQLGeoNamesGazLinka
while (rs.next()) {
MySQLGeoNamesGazEntry s = new MySQLGeoNamesGazEntry();
rs.getDouble(2);
- //ufi
s.setUFI(rs.getString(1));
-//latitude,
s.setLATITUDE(rs.getDouble(2));
-//longitude,
s.setLONGITUDE(rs.getDouble(3));
-//cc1,
s.setCC1(rs.getString(4));
-//adm1,
s.setADM1(rs.getString(5));
-//dsg,
s.setDSG(rs.getString(6));
-//SHORT_FORM ,
s.setSHORT_FORM(rs.getString(7));
-// SORT_NAME_RO ,
s.setSORT_NAME_RO(rs.getString(8));
-// FULL_NAME_RO ,
s.setFULL_NAME_RO(rs.getString(9));
-// FULL_NAME_ND_RO ,
s.setFULL_NAME_ND_RO(rs.getString(10));
-// SORT_NAME_RG ,
s.setSORT_NAME_RG(rs.getString(11));
-// FULL_NAME_RG ,
s.setFULL_NAME_RG(rs.getString(12));
-// FULL_NAME_ND_RG ,
s.setFULL_NAME_ND_RG(rs.getString(13));
-
s.setRank(rs.getDouble(14));
//set the base link data
@@ -124,7 +125,7 @@ public final class MySQLGeoNamesGazLinka
s.setItemID(s.getUFI());
s.setItemType(s.getDSG());
s.setItemParentID(s.getCC1().toLowerCase());
- s.getScoreMap().put("mysqlfulltext", s.getRank());
+ s.getScoreMap().put("dbfulltext", s.getRank());
toponyms.add(s);
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazLinkable.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazLinkable.java?rev=1536650&r1=1536649&r2=1536650&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazLinkable.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/entitylinker/MySQLUSGSGazLinkable.java Tue Oct 29 10:53:42 2013
@@ -21,43 +21,35 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import opennlp.tools.entitylinker.domain.BaseLink;
import opennlp.tools.util.Span;
/**
- * Links names to the USGS gazateer
+ * Links names to the USGS gazateer that resides in a database
*/
public class MySQLUSGSGazLinkable {
private Connection con;
- private Boolean filterCountryContext;
public MySQLUSGSGazLinkable() {
}
- public ArrayList<BaseLink> find(String locationText, Span span, Map<String, Set<Integer>> countryHits, EntityLinkerProperties properties) {
+ public ArrayList<BaseLink> find(String locationText, Span span, EntityLinkerProperties properties) {
ArrayList<BaseLink> returnlocs = new ArrayList<BaseLink>();
try {
- filterCountryContext = Boolean.valueOf(properties.getProperty("geoentitylinker.filter_by_country_context", "false"));
- //the usgs gazateer only has us geonames, so only use it if the user doesn't care about country isolation or the hits contain us
- if (countryHits.keySet().contains("us") || !filterCountryContext) {
-
- if (con == null) {
- con = getMySqlConnection(properties);
- }
- String thresh = properties.getProperty("mysqlusgsgazscorethresh", "10");
- int threshhold = -1;
- if (!thresh.matches("[azAZ]")) {
- threshhold = Integer.valueOf(thresh);
- }
- returnlocs.addAll(this.searchGaz(locationText, threshhold, countryHits.keySet(), properties));
+
+ if (con == null) {
+ con = getMySqlConnection(properties);
+ }
+ String thresh = properties.getProperty("usgs.gaz.rowsreturned", "5");
+ int threshhold = -1;
+ if (!thresh.matches("[azAZ]")) {
+ threshhold = Integer.valueOf(thresh);
}
+ returnlocs.addAll(this.searchGaz(locationText, threshhold, properties));
+
} catch (Exception ex) {
Logger.getLogger(MySQLUSGSGazLinkable.class.getName()).log(Level.SEVERE, null, ex);
}
@@ -65,25 +57,36 @@ public class MySQLUSGSGazLinkable {
return returnlocs;
}
- protected Connection getMySqlConnection(EntityLinkerProperties properties) throws Exception {
- String driver = properties.getProperty("mysql.driver", "org.gjt.mm.mysql.Driver");
- String url = properties.getProperty("mysql.url", "jdbc:mysql://127.0.0.1:3306/world");
- String username = properties.getProperty("mysql.username", "root");
- String password = properties.getProperty("mysql.password", "?");
+ private Connection getMySqlConnection(EntityLinkerProperties properties) throws Exception {
+ String driver = properties.getProperty("db.driver", "org.gjt.mm.mysql.Driver");
+ String url = properties.getProperty("db.url", "jdbc:mysql://127.0.0.1:3306/world");
+ String username = properties.getProperty("db.username", "root");
+ String password = properties.getProperty("db.password", "?");
Class.forName(driver);
Connection conn = DriverManager.getConnection(url, username, password);
return conn;
}
- private ArrayList<MySQLUSGSGazEntry> searchGaz(String searchString, int matchthresh, Set<String> countryCodes, EntityLinkerProperties properties) throws SQLException, Exception {
+ /**
+ *
+ * @param searchString the name to look up in the gazateer
+ * @param rowsReturned number of rows to return
+ * @param properties EntityLinkerProperties that identifies the database
+ * connection properties
+ *
+ * @return
+ * @throws SQLException
+ * @throws Exception
+ */
+ public ArrayList<MySQLUSGSGazEntry> searchGaz(String searchString, int rowsReturned, EntityLinkerProperties properties) throws SQLException, Exception {
if (con.isClosed()) {
con = getMySqlConnection(properties);
}
CallableStatement cs;
cs = con.prepareCall("CALL `search_gaz`(?, ?)");
cs.setString(1, this.format(searchString));
- cs.setInt(2, matchthresh);
+ cs.setInt(2, rowsReturned);
ArrayList<MySQLUSGSGazEntry> toponyms = new ArrayList<MySQLUSGSGazEntry>();
ResultSet rs;
try {
@@ -96,22 +99,20 @@ public class MySQLUSGSGazLinkable {
while (rs.next()) {
MySQLUSGSGazEntry s = new MySQLUSGSGazEntry();
s.setRank(rs.getDouble(1));
-
s.setFeatureid(String.valueOf(rs.getLong(2)));
s.setFeaturename(rs.getString(3));
-
s.setFeatureclass(rs.getString(4));
s.setStatealpha(rs.getString(5));
s.setPrimarylatitudeDEC(rs.getDouble(6));
s.setPrimarylongitudeDEC(rs.getDouble(7));
s.setMapname(rs.getString(8));
- //set the base link data
+ //set the baselink data
s.setItemName(s.getFeaturename().toLowerCase().trim());
s.setItemID(s.getFeatureid());
s.setItemType(s.getFeatureclass());
s.setItemParentID("us");
- s.getScoreMap().put("mysqlfulltext", s.getRank());
+ s.getScoreMap().put("dbfulltext", s.getRank());
toponyms.add(s);
}
@@ -126,14 +127,6 @@ public class MySQLUSGSGazLinkable {
return toponyms;
}
- private Set<String> getCountryCodes(List<CountryContextHit> hits) {
- Set<String> ccs = new HashSet<String>();
- for (CountryContextHit hit : hits) {
- ccs.add(hit.getCountryCode().toLowerCase());
- }
- return ccs;
- }
-
public String format(String entity) {
return "\"" + entity + "\"";
}