You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2023/05/01 17:11:50 UTC
[opennlp-addons] 02/02: Normalize line endings (#13)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/opennlp-addons.git
commit 13e985c893eb2b8aab66985482f6eeaac452c7df
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Mon May 1 18:40:10 2023 +0200
Normalize line endings (#13)
---
.gitattributes | 48 ++
geoentitylinker-addon/pom.xml | 186 ++---
.../addons/geoentitylinker/AdminBoundary.java | 276 +++----
.../geoentitylinker/AdminBoundaryContext.java | 306 ++++----
.../AdminBoundaryContextGenerator.java | 824 ++++++++++-----------
.../geoentitylinker/CountryContextEntry.java | 236 +++---
.../addons/geoentitylinker/GazetteerEntry.java | 346 ++++-----
.../geoentitylinker/GazetteerSearchCache.java | 98 +--
.../addons/geoentitylinker/GazetteerSearcher.java | 502 ++++++-------
.../addons/geoentitylinker/GeoEntityLinker.java | 440 +++++------
.../geoentitylinker/indexing/GazetteerIndexer.java | 420 +++++------
.../indexing/GeonamesFileDownloader.java | 240 +++---
.../indexing/GeonamesProcessor.java | 592 +++++++--------
.../geoentitylinker/indexing/RegionProcessor.java | 230 +++---
.../geoentitylinker/indexing/USGSProcessor.java | 500 ++++++-------
.../scoring/CountryProximityScorer.java | 554 +++++++-------
.../scoring/FuzzyStringMatchScorer.java | 248 +++----
.../scoring/GeoHashBinningScorer.java | 122 +--
.../scoring/LinkedEntityScorer.java | 80 +-
.../geoentitylinker/scoring/ModelBasedScorer.java | 314 ++++----
.../geoentitylinker/scoring/PlacetypeScorer.java | 164 ++--
.../geoentitylinker/scoring/PointClustering.java | 242 +++---
.../scoring/ProvinceProximityScorer.java | 588 +++++++--------
jwnl-addon/pom.xml | 156 ++--
modelbuilder-addon/pom.xml | 144 ++--
.../modelbuilder/DefaultModelBuilderUtil.java | 242 +++---
.../addons/modelbuilder/KnownEntityProvider.java | 96 +--
.../modelbuilder/ModelGenerationValidator.java | 70 +-
.../addons/modelbuilder/ModelParameter.java | 56 +-
.../opennlp/addons/modelbuilder/Modelable.java | 90 +--
.../modelbuilder/SemiSupervisedModelGenerator.java | 56 +-
.../addons/modelbuilder/SentenceProvider.java | 54 +-
.../modelbuilder/impls/BaseModelBuilderParams.java | 178 ++---
.../impls/FileKnownEntityProvider.java | 160 ++--
.../modelbuilder/impls/FileModelValidatorImpl.java | 172 ++---
.../modelbuilder/impls/FileSentenceProvider.java | 114 +--
.../modelbuilder/impls/GenericModelGenerator.java | 206 +++---
.../modelbuilder/impls/GenericModelableImpl.java | 242 +++---
morfologik-addon/pom.xml | 304 ++++----
39 files changed, 4972 insertions(+), 4924 deletions(-)
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..39bfc13
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,48 @@
+# Handle line endings automatically for files detected as text
+# and leave all files detected as binary untouched.
+* text=auto
+
+#
+# The above will handle all files NOT found below
+#
+# These files are text and should be normalized (Convert crlf => lf)
+*.adoc text eol=lf
+*.html text eol=lf
+*.java text eol=lf
+*.jspf text eol=lf
+*.md text eol=lf
+*.properties text eol=lf
+*.sh text eol=lf
+*.txt text eol=lf
+*.xml text eol=lf
+*.xsd text eol=lf
+*.xsl text eol=lf
+*.yml text eol=lf
+
+LICENSE text eol=lf
+NOTICE text eol=lf
+
+# These files are binary and should be left untouched
+# (binary is a macro for -text -diff)
+*.class binary
+*.dll binary
+*.ear binary
+*.gif binary
+*.ico binary
+*.jar binary
+*.jpg binary
+*.jpeg binary
+*.png binary
+*.ser binary
+*.so binary
+*.war binary
+*.zip binary
+*.exe binary
+*.gz binary
+
+#Windows
+*.bat text eol=crlf
+*.cmd text eol=crlf
+
+#Unix/Linux
+*.sh text eol=lf
\ No newline at end of file
diff --git a/geoentitylinker-addon/pom.xml b/geoentitylinker-addon/pom.xml
index 75be328..6a08a11 100644
--- a/geoentitylinker-addon/pom.xml
+++ b/geoentitylinker-addon/pom.xml
@@ -1,93 +1,93 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-addons</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>geoentitylinker-addon</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Apache OpenNLP GeoentityLinker Addon</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-tools</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>6.0.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers-common</artifactId>
- <version>6.0.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-queryparser</artifactId>
- <version>6.0.0</version>
- </dependency>
- <dependency>
- <groupId>com.spatial4j</groupId>
- <artifactId>spatial4j</artifactId>
- <version>0.4.1</version>
- <type>jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-engine</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-params</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>${maven.compiler.source}</source>
- <target>${maven.compiler.target}</target>
- <compilerArgument>-Xlint</compilerArgument>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>geoentitylinker-addon</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <name>Apache OpenNLP GeoentityLinker Addon</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-tools</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>6.0.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>6.0.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ <version>6.0.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.spatial4j</groupId>
+ <artifactId>spatial4j</artifactId>
+ <version>0.4.1</version>
+ <type>jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerArgument>-Xlint</compilerArgument>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundary.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundary.java
index a01b0bb..15859c1 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundary.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundary.java
@@ -1,138 +1,138 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.util.Objects;
-
-/**
- * Stores an admin boundary down to the US county level. Only US places from the
- * USGS Gazetteer will have county level info
- */
-public class AdminBoundary {
-
- private static final String NO_DATA_FOUND_VALUE = "NO_DATA_FOUND";
- private final String countryCode;
- private final String provinceCode;
- private final String provinceName;
- private final String countryName;
- private final String countyName;
- private final String countyCode;
- private final String countryRegex;
- private final String provinceRegex;
- private final String countyRegex;
-
-
- public AdminBoundary(String countryCode, String countryName, String provinceCode, String provinceName, String countyCode, String countyName,
- String countryRegex, String provinceRegex, String countyRegex) {
- this.countryCode = countryCode;
- this.provinceCode = provinceCode;
- this.provinceName = provinceName;
- this.countryName = countryName;
- this.countyName = countyName.equals("") ? NO_DATA_FOUND_VALUE : countyName;
- this.countyCode = countyCode.equals("") ? NO_DATA_FOUND_VALUE : countyCode;
- this.countryRegex = countryRegex;
- this.provinceRegex = provinceRegex;
- this.countyRegex = countyRegex;
- }
-
- public String getCountryCode() {
- return countryCode;
- }
-
- public String getProvCode() {
- return provinceCode;
- }
-
- public String getProvinceName() {
- return provinceName;
- }
-
- public String getCountryName() {
- return countryName;
- }
-
- public String getCountyName() {
- return countyName;
- }
-
- public String getCountyCode() {
- return countyCode;
- }
-
- @Override
- public String toString() {
- return "AdminBoundary{" + "countryCode=" + countryCode + ", provinceCode=" + provinceCode + ", provinceName=" + provinceName + ", countryName=" + countryName + ", countyName=" + countyName + ", countyCode=" + countyCode + '}';
- }
-
- @Override
- public int hashCode() {
- int hash = 7;
- hash = 11 * hash + Objects.hashCode(this.countryCode);
- hash = 11 * hash + Objects.hashCode(this.provinceCode);
- hash = 11 * hash + Objects.hashCode(this.provinceName);
- hash = 11 * hash + Objects.hashCode(this.countryName);
- hash = 11 * hash + Objects.hashCode(this.countyName);
- hash = 11 * hash + Objects.hashCode(this.countyCode);
- return hash;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- final AdminBoundary other = (AdminBoundary) obj;
- if (!Objects.equals(this.countryCode, other.countryCode)) {
- return false;
- }
- if (!Objects.equals(this.provinceCode, other.provinceCode)) {
- return false;
- }
- if (!Objects.equals(this.provinceName, other.provinceName)) {
- return false;
- }
- if (!Objects.equals(this.countryName, other.countryName)) {
- return false;
- }
- if (!Objects.equals(this.countyName, other.countyName)) {
- return false;
- }
- if (!Objects.equals(this.countyCode, other.countyCode)) {
- return false;
- }
- return true;
- }
-
- public String getProvinceCode() {
- return provinceCode;
- }
-
- public String getCountryRegex() {
- return countryRegex;
- }
-
- public String getProvinceRegex() {
- return provinceRegex;
- }
-
- public String getCountyRegex() {
- return countyRegex;
- }
-
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.util.Objects;
+
+/**
+ * Stores an admin boundary down to the US county level. Only US places from the
+ * USGS Gazetteer will have county level info
+ */
+public class AdminBoundary {
+
+ private static final String NO_DATA_FOUND_VALUE = "NO_DATA_FOUND";
+ private final String countryCode;
+ private final String provinceCode;
+ private final String provinceName;
+ private final String countryName;
+ private final String countyName;
+ private final String countyCode;
+ private final String countryRegex;
+ private final String provinceRegex;
+ private final String countyRegex;
+
+
+ public AdminBoundary(String countryCode, String countryName, String provinceCode, String provinceName, String countyCode, String countyName,
+ String countryRegex, String provinceRegex, String countyRegex) {
+ this.countryCode = countryCode;
+ this.provinceCode = provinceCode;
+ this.provinceName = provinceName;
+ this.countryName = countryName;
+ this.countyName = countyName.equals("") ? NO_DATA_FOUND_VALUE : countyName;
+ this.countyCode = countyCode.equals("") ? NO_DATA_FOUND_VALUE : countyCode;
+ this.countryRegex = countryRegex;
+ this.provinceRegex = provinceRegex;
+ this.countyRegex = countyRegex;
+ }
+
+ public String getCountryCode() {
+ return countryCode;
+ }
+
+ public String getProvCode() {
+ return provinceCode;
+ }
+
+ public String getProvinceName() {
+ return provinceName;
+ }
+
+ public String getCountryName() {
+ return countryName;
+ }
+
+ public String getCountyName() {
+ return countyName;
+ }
+
+ public String getCountyCode() {
+ return countyCode;
+ }
+
+ @Override
+ public String toString() {
+ return "AdminBoundary{" + "countryCode=" + countryCode + ", provinceCode=" + provinceCode + ", provinceName=" + provinceName + ", countryName=" + countryName + ", countyName=" + countyName + ", countyCode=" + countyCode + '}';
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 7;
+ hash = 11 * hash + Objects.hashCode(this.countryCode);
+ hash = 11 * hash + Objects.hashCode(this.provinceCode);
+ hash = 11 * hash + Objects.hashCode(this.provinceName);
+ hash = 11 * hash + Objects.hashCode(this.countryName);
+ hash = 11 * hash + Objects.hashCode(this.countyName);
+ hash = 11 * hash + Objects.hashCode(this.countyCode);
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final AdminBoundary other = (AdminBoundary) obj;
+ if (!Objects.equals(this.countryCode, other.countryCode)) {
+ return false;
+ }
+ if (!Objects.equals(this.provinceCode, other.provinceCode)) {
+ return false;
+ }
+ if (!Objects.equals(this.provinceName, other.provinceName)) {
+ return false;
+ }
+ if (!Objects.equals(this.countryName, other.countryName)) {
+ return false;
+ }
+ if (!Objects.equals(this.countyName, other.countyName)) {
+ return false;
+ }
+ if (!Objects.equals(this.countyCode, other.countyCode)) {
+ return false;
+ }
+ return true;
+ }
+
+ public String getProvinceCode() {
+ return provinceCode;
+ }
+
+ public String getCountryRegex() {
+ return countryRegex;
+ }
+
+ public String getProvinceRegex() {
+ return provinceRegex;
+ }
+
+ public String getCountyRegex() {
+ return countyRegex;
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContext.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContext.java
index beadf4e..85553c3 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContext.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContext.java
@@ -1,153 +1,153 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-public class AdminBoundaryContext {
-
- private final Map<String, Set<Integer>> countryMentions;
- private final Map<String, Set<Integer>> provMentions;
- private final Map<String, Set<Integer>> countyMentions;
- private final Set<String> countryHits;
- private final Set<String> provHits;
- private final Set<String> countyHits;
- private final Map<String, String> countryRefMap;
- private final Map<String, String> countryRegexMap;
- private final Map<String, String> countyRegexMap;
- private final Map<String, String> provinceRegexMap;
- private final Map<String, Map<String, String>> provRefMap;
- private final Map<String, Map<String, String>> countyRefMap;
- private final Set<String> whereClauses;
- private final Map<String, Set<String>> nameCodesMap;
-
- public AdminBoundaryContext(Map<String, Set<Integer>> countryMentions,
- Map<String, Set<Integer>> provMentions,
- Map<String, Set<Integer>> countyMentions,
- Set<String> countryHits,
- Set<String> provHits,
- Set<String> countyHits,
- Map<String, String> countryRefMap,
- Map<String, Map<String, String>> provRefMap,
- Map<String, Map<String, String>> countyRefMap, Map<String, Set<String>> nameCodesMap, Map<String, String> countryRegexMap, Map<String, String> provinceRegexMap,
- Map<String, String> countyRegexMap) {
- this.countryMentions = countryMentions;
- this.provMentions = provMentions;
- this.countyMentions = countyMentions;
- this.countryHits = countryHits;
- this.provHits = provHits;
- this.countyHits = countyHits;
- this.countryRefMap = countryRefMap;
- this.provRefMap = provRefMap;
- this.countyRefMap = countyRefMap;
- this.whereClauses = setWhereClauses();
- this.nameCodesMap = nameCodesMap;
- this.countryRegexMap = countryRegexMap;
- this.provinceRegexMap = provinceRegexMap;
- this.countyRegexMap = countyRegexMap;
- }
-
- public Map<String, Set<String>> getNameCodesMap() {
- return nameCodesMap;
- }
-
- public Map<String, Set<Integer>> getCountryMentions() {
- return countryMentions;
- }
-
- public Map<String, Set<Integer>> getProvMentions() {
- return provMentions;
- }
-
- public Map<String, Set<Integer>> getCountyMentions() {
- return countyMentions;
- }
-
- public Set<String> getCountryHits() {
- return countryHits;
- }
-
- public Set<String> getProvHits() {
- return provHits;
- }
-
- public Set<String> getCountyHits() {
- return countyHits;
- }
-
- public Map<String, String> getCountryRefMap() {
- return countryRefMap;
- }
-
- public Map<String, Map<String, String>> getProvRefMap() {
- return provRefMap;
- }
-
- public Map<String, Map<String, String>> getCountyRefMap() {
- return countyRefMap;
- }
-
- public Set<String> getWhereClauses() {
- return whereClauses;
- }
-
- private Set<String> setWhereClauses() {
- Set<String> clauses = new HashSet<>();
- for (String countryCode : this.getCountryHits()) {
- String gazType = countryCode.equalsIgnoreCase("us") ? " AND gazsource:usgs" : " AND gazsource:geonames";
- if (countryCode.toLowerCase().matches(".*rg[0-9].*")) {
- gazType = " AND gazsource:region";
- }
- Map<String, String> provsForCountry = this.getProvRefMap().get(countryCode);
- if (provsForCountry == null) {
- provsForCountry = new HashMap<>();
- }
- Map<String, String> provs = new HashMap<>();
-
-// if (!provsForCountry.isEmpty()) {
-// for (String pcode : provsForCountry.keySet()) {
-// if (this.getProvHits().contains(pcode)) {
-// provs.put(pcode, provsForCountry.get(pcode));
-//
-// clauses.add(" countrycode:" + countryCode + " AND admincode:" + pcode + gazType);
-//
-// }
-// }
-// }
- if (provs.isEmpty()) {
- //got a country with no mentioned provs
- clauses.add(" countrycode:" + countryCode + gazType);
- }
- }
- return clauses;
- }
-
- public Map<String, String> getCountryRegexMap() {
- return countryRegexMap;
- }
-
- public Map<String, String> getCountyRegexMap() {
- return countyRegexMap;
- }
-
- public Map<String, String> getProvinceRegexMap() {
- return provinceRegexMap;
- }
-
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class AdminBoundaryContext {
+
+ private final Map<String, Set<Integer>> countryMentions;
+ private final Map<String, Set<Integer>> provMentions;
+ private final Map<String, Set<Integer>> countyMentions;
+ private final Set<String> countryHits;
+ private final Set<String> provHits;
+ private final Set<String> countyHits;
+ private final Map<String, String> countryRefMap;
+ private final Map<String, String> countryRegexMap;
+ private final Map<String, String> countyRegexMap;
+ private final Map<String, String> provinceRegexMap;
+ private final Map<String, Map<String, String>> provRefMap;
+ private final Map<String, Map<String, String>> countyRefMap;
+ private final Set<String> whereClauses;
+ private final Map<String, Set<String>> nameCodesMap;
+
+ public AdminBoundaryContext(Map<String, Set<Integer>> countryMentions,
+ Map<String, Set<Integer>> provMentions,
+ Map<String, Set<Integer>> countyMentions,
+ Set<String> countryHits,
+ Set<String> provHits,
+ Set<String> countyHits,
+ Map<String, String> countryRefMap,
+ Map<String, Map<String, String>> provRefMap,
+ Map<String, Map<String, String>> countyRefMap, Map<String, Set<String>> nameCodesMap, Map<String, String> countryRegexMap, Map<String, String> provinceRegexMap,
+ Map<String, String> countyRegexMap) {
+ this.countryMentions = countryMentions;
+ this.provMentions = provMentions;
+ this.countyMentions = countyMentions;
+ this.countryHits = countryHits;
+ this.provHits = provHits;
+ this.countyHits = countyHits;
+ this.countryRefMap = countryRefMap;
+ this.provRefMap = provRefMap;
+ this.countyRefMap = countyRefMap;
+ this.whereClauses = setWhereClauses();
+ this.nameCodesMap = nameCodesMap;
+ this.countryRegexMap = countryRegexMap;
+ this.provinceRegexMap = provinceRegexMap;
+ this.countyRegexMap = countyRegexMap;
+ }
+
+ public Map<String, Set<String>> getNameCodesMap() {
+ return nameCodesMap;
+ }
+
+ public Map<String, Set<Integer>> getCountryMentions() {
+ return countryMentions;
+ }
+
+ public Map<String, Set<Integer>> getProvMentions() {
+ return provMentions;
+ }
+
+ public Map<String, Set<Integer>> getCountyMentions() {
+ return countyMentions;
+ }
+
+ public Set<String> getCountryHits() {
+ return countryHits;
+ }
+
+ public Set<String> getProvHits() {
+ return provHits;
+ }
+
+ public Set<String> getCountyHits() {
+ return countyHits;
+ }
+
+ public Map<String, String> getCountryRefMap() {
+ return countryRefMap;
+ }
+
+ public Map<String, Map<String, String>> getProvRefMap() {
+ return provRefMap;
+ }
+
+ public Map<String, Map<String, String>> getCountyRefMap() {
+ return countyRefMap;
+ }
+
+ public Set<String> getWhereClauses() {
+ return whereClauses;
+ }
+
+ private Set<String> setWhereClauses() {
+ Set<String> clauses = new HashSet<>();
+ for (String countryCode : this.getCountryHits()) {
+ String gazType = countryCode.equalsIgnoreCase("us") ? " AND gazsource:usgs" : " AND gazsource:geonames";
+ if (countryCode.toLowerCase().matches(".*rg[0-9].*")) {
+ gazType = " AND gazsource:region";
+ }
+ Map<String, String> provsForCountry = this.getProvRefMap().get(countryCode);
+ if (provsForCountry == null) {
+ provsForCountry = new HashMap<>();
+ }
+ Map<String, String> provs = new HashMap<>();
+
+// if (!provsForCountry.isEmpty()) {
+// for (String pcode : provsForCountry.keySet()) {
+// if (this.getProvHits().contains(pcode)) {
+// provs.put(pcode, provsForCountry.get(pcode));
+//
+// clauses.add(" countrycode:" + countryCode + " AND admincode:" + pcode + gazType);
+//
+// }
+// }
+// }
+ if (provs.isEmpty()) {
+ //got a country with no mentioned provs
+ clauses.add(" countrycode:" + countryCode + gazType);
+ }
+ }
+ return clauses;
+ }
+
+ public Map<String, String> getCountryRegexMap() {
+ return countryRegexMap;
+ }
+
+ public Map<String, String> getCountyRegexMap() {
+ return countyRegexMap;
+ }
+
+ public Map<String, String> getProvinceRegexMap() {
+ return provinceRegexMap;
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContextGenerator.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContextGenerator.java
index a6741fe..166cfca 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContextGenerator.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContextGenerator.java
@@ -1,412 +1,412 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.logging.Level;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Finds instances of country mentions in a String, typically a document text.
- * Used to boost or degrade scoring of linked geo entities
- *
- */
-public class AdminBoundaryContextGenerator {
-
- private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private List<CountryContextEntry> countrydata;
- private Map<String, Set<String>> nameCodesMap = new HashMap<>();
- private final Map<String, Set<Integer>> countryMentions = new HashMap<>();
-
- Map<String, String> countryRegexMap = new HashMap<>();
- Map<String, String> provinceRegexMap = new HashMap<>();
- Map<String, String> countyRegexMap = new HashMap<>();
-
- private final Set<CountryContextEntry> countryHits = new HashSet<>();
- private final EntityLinkerProperties properties;
- private final List<AdminBoundary> adminBoundaryData= new ArrayList<>();
- private final Set<AdminBoundary> adminBoundaryHits = new HashSet<>();
- private AdminBoundaryContext context;
-
- public AdminBoundaryContext getContext(String text) {
- context = null;
- nameCodesMap.clear();
- context = process(text);
-
- return context;
- }
-
- private final Set<String> countryHitSet = new HashSet<>();
- private final Map<String, String> countryMap = new HashMap<>();
- private final Map<String, Map<String, String>> provMap = new HashMap<>();
- private final Map<String, Map<String, String>> countyMap = new HashMap<>();
-
- private Map<String, Set<Integer>> provMentions = new HashMap<>();
- private Map<String, Set<Integer>> countyMentions = new HashMap<>();
-
- private final Set<String> provHits = new HashSet<>();
- private final Set<String> countyHits = new HashSet<>();
-
- public static void main(String[] args) {
- try {
- AdminBoundaryContextGenerator countryContext
- = new AdminBoundaryContextGenerator(new EntityLinkerProperties(new File("C:\\Temp\\gaz_data\\newCountryContextfile.txt")));
-
- AdminBoundaryContext c = countryContext.process("This artcle is about fairfax county virginia in the north of florida in the united states. It is also about Moscow and atlanta. Hillsborough county florida is a nice place. Eastern Africa people are cool.");
- System.out.println(c);
- } catch (Exception ex) {
- java.util.logging.Logger.getLogger(AdminBoundaryContextGenerator.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
-
- public AdminBoundaryContextGenerator(EntityLinkerProperties properties) throws IOException {
- this.properties = properties;
- if (countrydata == null) {
- String path = this.properties.getProperty("opennlp.geoentitylinker.countrycontext.filepath", "");
- if (path == null || path.trim().isEmpty()) {
- throw new IOException("missing country context data configuration. Property opennlp.geoentitylinker.countrycontext.filepath must have a valid path value in entitylinker properties file");
- }
- File countryContextFile = new File(path);
- if (countryContextFile == null || !countryContextFile.exists()) {
- throw new IOException("missing country context file");
- }
- //countrydata = getCountryContextFromFile(countryContextFile);
- getContextFromFile(countryContextFile);
- if (adminBoundaryData.isEmpty()) {
- throw new IOException("missing country context data");
- }
- }
- }
-
- public Map<String, Set<Integer>> getCountryMentions() {
- return countryMentions;
- }
-
- /**
- * @return returns the last set of hits after calling regexFind
- */
- public Set<CountryContextEntry> getCountryHits() {
- return countryHits;
- }
-
- /**
- * @return returns the last name to codes map after calling regexFind
- */
- public Map<String, Set<String>> getNameCodesMap() {
- return nameCodesMap;
- }
-
- public void setNameCodesMap(Map<String, Set<String>> nameCodesMap) {
- this.nameCodesMap = nameCodesMap;
- }
-
- private void reset() {
- this.nameCodesMap.clear();
- this.countryHitSet.clear();
- this.countryHits.clear();
- this.countryMentions.clear();
- this.provHits.clear();
- this.provMentions.clear();
- this.countyHits.clear();
- this.countyMentions.clear();
- this.adminBoundaryHits.clear();
- }
-
- /**
- * Finds indicators of countries, provinces, and cities, as per the USGS and
- * Geonames gazetteers. The results of this are used to score toponymns
- * downstream. The full text of a document should be passed in here.
- *
- * @param text the full text of the document (block of text).
- */
- private AdminBoundaryContext process(String text) {
- try {
- reset();
- Map<String, Set<Integer>> countryhitMap = regexfind(text, countryMap, countryHitSet, "country");
- if (!countryhitMap.isEmpty()) {
- for (String cc : countryhitMap.keySet()) {
- Map<String, String> provsForCc = provMap.get(cc);
- if (provsForCc != null) {
- provMentions.putAll(regexfind(text, provsForCc, provHits, "province"));
- if (provMentions != null) {
- for (String prov : provMentions.keySet()) {
- Map<String, String> get = countyMap.get(prov);
- if (get != null) {
- countyMentions.putAll(regexfind(text, get, countyHits, "province"));
- }
- }
- }
- }
- }
- } else {
- for (Map<String, String> provsForCc : provMap.values()) {
- if (provsForCc != null) {
- provMentions = regexfind(text, provsForCc, provHits, "province");
- if (provMentions != null) {
- for (String prov : provMentions.keySet()) {
- //fake a country hit based on a province hit... this gets fuzzy
- String cc = prov.split("\\.")[0];
- if (!countryhitMap.containsKey(cc)) {
- countryhitMap.put(cc, provMentions.get(prov));
- countryHitSet.add(cc);
- } else {
- countryhitMap.get(cc).addAll(provMentions.get(prov));
- }
- Map<String, String> get = countyMap.get(prov);
- if (get != null) {
- countyMentions = regexfind(text, get, countyHits, "oounty");
- }
- }
- }
- }
- }
- }
-
- Map<String, String> countryRefMap = new HashMap<>();
-
- for (String c : countryHitSet) {
- String countryName = countryMap.get(c);
- if (countryName != null) {
- countryRefMap.put(c, countryName);
- }
- }
-
- return new AdminBoundaryContext(countryhitMap, provMentions, countyMentions, countryHitSet, provHits, countyHits,
- countryRefMap, provMap, countyMap, nameCodesMap, countryRegexMap, provinceRegexMap, countyRegexMap);
- } catch (Exception e) {
- LOG.error(e.getLocalizedMessage(), e);
- }
- return null;
- }
-
- /**
- * discovers indicators of admin boundary data using regex.
- *
- * @param docText the full text
- * @param lookupMap a map to use to find names. the key=a location code, the
- * value is an actual name.
- * @param hitsRef a reference to a set that stores the hits by id
- */
- private Map<String, Set<Integer>> regexfind(String docText, Map<String, String> lookupMap, Set<String> hitsRef, String locationType) {
- Map<String, Set<Integer>> mentions = new HashMap<>();
- if (lookupMap == null) {
- return mentions;
- }
- try {
-
- for (String entry : lookupMap.keySet()) {
-
- String name = lookupMap.get(entry).toLowerCase();
- if (name == null) {
- continue;
- }
- switch (locationType) {
- case "country":
- if (this.countryRegexMap.containsKey(entry)) {
- name = countryRegexMap.get(entry);
- }
- break;
-
- case "province":
- if (this.provinceRegexMap.containsKey(entry)) {
- name = provinceRegexMap.get(entry);
- }
- break;
- case "county":
- if (this.countyRegexMap.containsKey(entry)) {
- name = countyRegexMap.get(entry);
- }
- break;
- }
- name = "(^|[^\\p{L}\\p{Nd}])" + name.replace(", the", "") + "([^\\p{L}\\p{Nd}]|$)";
- Pattern regex = Pattern.compile(name, Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
- Matcher rs = regex.matcher(docText);
- String code = entry.toLowerCase();
- code = code.trim().replace("", "");
- boolean found = false;
- while (rs.find()) {
- found = true;
- Integer start = rs.start();
- String hit = rs.group().toLowerCase().trim();
- hit = hit.replaceAll("\\.|,|;|\\?|!|\\\\|/|\"|'|=|-|&", "");
- if (mentions.containsKey(code)) {
- mentions.get(code).add(start);
- } else {
- Set<Integer> newset = new HashSet<>();
- newset.add(start);
- mentions.put(code, newset);
- }
- if (!hit.equals("")) {
- if (this.nameCodesMap.containsKey(hit)) {
- nameCodesMap.get(hit).add(code);
- } else {
- HashSet<String> newset = new HashSet<>();
- newset.add(code);
- nameCodesMap.put(hit, newset);
- }
- }
-
- }
- if (found) {
- hitsRef.add(code);
-
- }
- }
-
- } catch (Exception ex) {
- LOG.error(ex.getLocalizedMessage(), ex);
- }
-
- return mentions;
- }
-
- private void getContextFromFile(File countryContextFile) {
- if (this.adminBoundaryData != null && !this.adminBoundaryData.isEmpty()) {
- return;
- }
-
- BufferedReader reader;
- try {
- reader = new BufferedReader(new FileReader(countryContextFile));
- String line;
- int lineNum = 0;
- while ((line = reader.readLine()) != null) {
- String[] values = line.split("\t");
- if (lineNum == 0) {
- lineNum++;
- continue;
- //skip column name headers
- }
- if (values.length == 9) {
- AdminBoundary entry = new AdminBoundary(
- values[0].toLowerCase().trim().replace("", ""),
- values[3].toLowerCase().trim(),
- values[1].toLowerCase().trim(),
- values[4].toLowerCase().trim(),
- values[2].toLowerCase().trim(),
- values[5].toLowerCase().trim(),
- values[6].toLowerCase().trim(),
- values[7].toLowerCase().trim(),
- values[8].toLowerCase().trim());
- this.adminBoundaryData.add(entry);
- } else {
- throw new IllegalArgumentException("Improperly formatted file");
- }
-
- }
- reader.close();
- } catch (IOException ex) {
- LOG.error(ex.getLocalizedMessage(), ex);
- }
-
- loadMaps(this.adminBoundaryData);
-
- }
-
- private void loadMaps(List<AdminBoundary> boundaries) {
- for (AdminBoundary adm : boundaries) {
- if (!adm.getCountryCode().equals("null")) {
- countryMap.put(adm.getCountryCode(), adm.getCountryName());
- if (countryRegexMap.containsKey(adm.getCountryCode())) {
- String currentRegex = countryRegexMap.get(adm.getCountryCode());
- if (currentRegex.length() > adm.getCountryRegex().length()) {
- // the longest one wins if they are not all the same for each entry in the file
- countryRegexMap.put(adm.getCountryCode(), currentRegex);
- }//else do nothing
- } else {
- countryRegexMap.put(adm.getCountryCode(), adm.getCountryRegex());
- }
-
- if (!adm.getProvCode().equals("null")) {
- Map<String, String> provs = provMap.get(adm.getCountryCode());
- if (provs == null) {
- provs = new HashMap<>();
- }
- //if (!provs.containsKey(adm.getProvCode())) {
- provs.put(adm.getCountryCode() + "." + adm.getProvCode(), adm.getProvinceName());
- provMap.put(adm.getCountryCode(), provs);
- // }
-
- if (!adm.getCountyCode().equalsIgnoreCase("no_data_found") && !adm.getCountyName().equalsIgnoreCase("no_data_found")) {
- Map<String, String> counties = countyMap.get(adm.getCountryCode() + "." + adm.getProvCode());
- if (counties == null) {
- counties = new HashMap<>();
- } // if (!counties.containsKey(adm.getCountyCode())) {
- String countyid = adm.getCountryCode() + "." + adm.getProvCode() + "." + adm.getCountyCode();
- counties.put(countyid, adm.getCountyName());
- countyMap.put(adm.getCountryCode() + "." + adm.getProvCode(), counties);
- // }
-
- }
-
- }
- }
- }
- fillProvRegexMap();
- fillCountyRegexMap();
- }
-
- private void fillProvRegexMap() {
- this.provinceRegexMap = new HashMap<>();
- // this.adminBoundaryData
- for (AdminBoundary adm : adminBoundaryData) {
-
- if (provinceRegexMap.containsKey(adm.getProvCode())) {
- String currentRegex = provinceRegexMap.get(adm.getProvCode());
- if (currentRegex.length() > adm.getProvinceRegex().length()) {
- // the longest one wins if they are not all the same for each entry in the file
- provinceRegexMap.put(adm.getProvCode(), currentRegex);
- }//else do nothing
- } else {
- provinceRegexMap.put(adm.getProvCode(), adm.getProvinceRegex());
- }
- }
- }
-
- private void fillCountyRegexMap() {
- this.countyRegexMap = new HashMap<>();
- // this.adminBoundaryData
- for (AdminBoundary adm : adminBoundaryData) {
-
- if (countyRegexMap.containsKey(adm.getCountyCode())) {
- String currentRegex = countyRegexMap.get(adm.getCountyCode());
- if (currentRegex.length() > adm.getCountyRegex().length()) {
- // the longest one wins if they are not all the same for each entry in the file
- countyRegexMap.put(adm.getCountyCode(), currentRegex);
- }//else do nothing
- } else {
- countyRegexMap.put(adm.getCountyCode(), adm.getCountyRegex());
- }
- }
-
- }
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Finds instances of country mentions in a String, typically a document text.
+ * Used to boost or degrade scoring of linked geo entities
+ *
+ */
+public class AdminBoundaryContextGenerator {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private List<CountryContextEntry> countrydata;
+ private Map<String, Set<String>> nameCodesMap = new HashMap<>();
+ private final Map<String, Set<Integer>> countryMentions = new HashMap<>();
+
+ Map<String, String> countryRegexMap = new HashMap<>();
+ Map<String, String> provinceRegexMap = new HashMap<>();
+ Map<String, String> countyRegexMap = new HashMap<>();
+
+ private final Set<CountryContextEntry> countryHits = new HashSet<>();
+ private final EntityLinkerProperties properties;
+ private final List<AdminBoundary> adminBoundaryData= new ArrayList<>();
+ private final Set<AdminBoundary> adminBoundaryHits = new HashSet<>();
+ private AdminBoundaryContext context;
+
+ public AdminBoundaryContext getContext(String text) {
+ context = null;
+ nameCodesMap.clear();
+ context = process(text);
+
+ return context;
+ }
+
+ private final Set<String> countryHitSet = new HashSet<>();
+ private final Map<String, String> countryMap = new HashMap<>();
+ private final Map<String, Map<String, String>> provMap = new HashMap<>();
+ private final Map<String, Map<String, String>> countyMap = new HashMap<>();
+
+ private Map<String, Set<Integer>> provMentions = new HashMap<>();
+ private Map<String, Set<Integer>> countyMentions = new HashMap<>();
+
+ private final Set<String> provHits = new HashSet<>();
+ private final Set<String> countyHits = new HashSet<>();
+
+ public static void main(String[] args) {
+ try {
+ AdminBoundaryContextGenerator countryContext
+ = new AdminBoundaryContextGenerator(new EntityLinkerProperties(new File("C:\\Temp\\gaz_data\\newCountryContextfile.txt")));
+
+ AdminBoundaryContext c = countryContext.process("This artcle is about fairfax county virginia in the north of florida in the united states. It is also about Moscow and atlanta. Hillsborough county florida is a nice place. Eastern Africa people are cool.");
+ System.out.println(c);
+ } catch (Exception ex) {
+ java.util.logging.Logger.getLogger(AdminBoundaryContextGenerator.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ }
+
+ public AdminBoundaryContextGenerator(EntityLinkerProperties properties) throws IOException {
+ this.properties = properties;
+ if (countrydata == null) {
+ String path = this.properties.getProperty("opennlp.geoentitylinker.countrycontext.filepath", "");
+ if (path == null || path.trim().isEmpty()) {
+ throw new IOException("missing country context data configuration. Property opennlp.geoentitylinker.countrycontext.filepath must have a valid path value in entitylinker properties file");
+ }
+ File countryContextFile = new File(path);
+ if (countryContextFile == null || !countryContextFile.exists()) {
+ throw new IOException("missing country context file");
+ }
+ //countrydata = getCountryContextFromFile(countryContextFile);
+ getContextFromFile(countryContextFile);
+ if (adminBoundaryData.isEmpty()) {
+ throw new IOException("missing country context data");
+ }
+ }
+ }
+
+ public Map<String, Set<Integer>> getCountryMentions() {
+ return countryMentions;
+ }
+
+ /**
+ * @return returns the last set of hits after calling regexFind
+ */
+ public Set<CountryContextEntry> getCountryHits() {
+ return countryHits;
+ }
+
+ /**
+ * @return returns the last name to codes map after calling regexFind
+ */
+ public Map<String, Set<String>> getNameCodesMap() {
+ return nameCodesMap;
+ }
+
+ public void setNameCodesMap(Map<String, Set<String>> nameCodesMap) {
+ this.nameCodesMap = nameCodesMap;
+ }
+
+ private void reset() {
+ this.nameCodesMap.clear();
+ this.countryHitSet.clear();
+ this.countryHits.clear();
+ this.countryMentions.clear();
+ this.provHits.clear();
+ this.provMentions.clear();
+ this.countyHits.clear();
+ this.countyMentions.clear();
+ this.adminBoundaryHits.clear();
+ }
+
+ /**
+ * Finds indicators of countries, provinces, and cities, as per the USGS and
+ * Geonames gazetteers. The results of this are used to score toponymns
+ * downstream. The full text of a document should be passed in here.
+ *
+ * @param text the full text of the document (block of text).
+ */
+ private AdminBoundaryContext process(String text) {
+ try {
+ reset();
+ Map<String, Set<Integer>> countryhitMap = regexfind(text, countryMap, countryHitSet, "country");
+ if (!countryhitMap.isEmpty()) {
+ for (String cc : countryhitMap.keySet()) {
+ Map<String, String> provsForCc = provMap.get(cc);
+ if (provsForCc != null) {
+ provMentions.putAll(regexfind(text, provsForCc, provHits, "province"));
+ if (provMentions != null) {
+ for (String prov : provMentions.keySet()) {
+ Map<String, String> get = countyMap.get(prov);
+ if (get != null) {
+ countyMentions.putAll(regexfind(text, get, countyHits, "province"));
+ }
+ }
+ }
+ }
+ }
+ } else {
+ for (Map<String, String> provsForCc : provMap.values()) {
+ if (provsForCc != null) {
+ provMentions = regexfind(text, provsForCc, provHits, "province");
+ if (provMentions != null) {
+ for (String prov : provMentions.keySet()) {
+ //fake a country hit based on a province hit... this gets fuzzy
+ String cc = prov.split("\\.")[0];
+ if (!countryhitMap.containsKey(cc)) {
+ countryhitMap.put(cc, provMentions.get(prov));
+ countryHitSet.add(cc);
+ } else {
+ countryhitMap.get(cc).addAll(provMentions.get(prov));
+ }
+ Map<String, String> get = countyMap.get(prov);
+ if (get != null) {
+ countyMentions = regexfind(text, get, countyHits, "oounty");
+ }
+ }
+ }
+ }
+ }
+ }
+
+ Map<String, String> countryRefMap = new HashMap<>();
+
+ for (String c : countryHitSet) {
+ String countryName = countryMap.get(c);
+ if (countryName != null) {
+ countryRefMap.put(c, countryName);
+ }
+ }
+
+ return new AdminBoundaryContext(countryhitMap, provMentions, countyMentions, countryHitSet, provHits, countyHits,
+ countryRefMap, provMap, countyMap, nameCodesMap, countryRegexMap, provinceRegexMap, countyRegexMap);
+ } catch (Exception e) {
+ LOG.error(e.getLocalizedMessage(), e);
+ }
+ return null;
+ }
+
+ /**
+ * discovers indicators of admin boundary data using regex.
+ *
+ * @param docText the full text
+ * @param lookupMap a map to use to find names. the key=a location code, the
+ * value is an actual name.
+ * @param hitsRef a reference to a set that stores the hits by id
+ */
+ private Map<String, Set<Integer>> regexfind(String docText, Map<String, String> lookupMap, Set<String> hitsRef, String locationType) {
+ Map<String, Set<Integer>> mentions = new HashMap<>();
+ if (lookupMap == null) {
+ return mentions;
+ }
+ try {
+
+ for (String entry : lookupMap.keySet()) {
+
+ String name = lookupMap.get(entry).toLowerCase();
+ if (name == null) {
+ continue;
+ }
+ switch (locationType) {
+ case "country":
+ if (this.countryRegexMap.containsKey(entry)) {
+ name = countryRegexMap.get(entry);
+ }
+ break;
+
+ case "province":
+ if (this.provinceRegexMap.containsKey(entry)) {
+ name = provinceRegexMap.get(entry);
+ }
+ break;
+ case "county":
+ if (this.countyRegexMap.containsKey(entry)) {
+ name = countyRegexMap.get(entry);
+ }
+ break;
+ }
+ name = "(^|[^\\p{L}\\p{Nd}])" + name.replace(", the", "") + "([^\\p{L}\\p{Nd}]|$)";
+ Pattern regex = Pattern.compile(name, Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+ Matcher rs = regex.matcher(docText);
+ String code = entry.toLowerCase();
+ code = code.trim().replace("", "");
+ boolean found = false;
+ while (rs.find()) {
+ found = true;
+ Integer start = rs.start();
+ String hit = rs.group().toLowerCase().trim();
+ hit = hit.replaceAll("\\.|,|;|\\?|!|\\\\|/|\"|'|=|-|&", "");
+ if (mentions.containsKey(code)) {
+ mentions.get(code).add(start);
+ } else {
+ Set<Integer> newset = new HashSet<>();
+ newset.add(start);
+ mentions.put(code, newset);
+ }
+ if (!hit.equals("")) {
+ if (this.nameCodesMap.containsKey(hit)) {
+ nameCodesMap.get(hit).add(code);
+ } else {
+ HashSet<String> newset = new HashSet<>();
+ newset.add(code);
+ nameCodesMap.put(hit, newset);
+ }
+ }
+
+ }
+ if (found) {
+ hitsRef.add(code);
+
+ }
+ }
+
+ } catch (Exception ex) {
+ LOG.error(ex.getLocalizedMessage(), ex);
+ }
+
+ return mentions;
+ }
+
+ private void getContextFromFile(File countryContextFile) {
+ if (this.adminBoundaryData != null && !this.adminBoundaryData.isEmpty()) {
+ return;
+ }
+
+ BufferedReader reader;
+ try {
+ reader = new BufferedReader(new FileReader(countryContextFile));
+ String line;
+ int lineNum = 0;
+ while ((line = reader.readLine()) != null) {
+ String[] values = line.split("\t");
+ if (lineNum == 0) {
+ lineNum++;
+ continue;
+ //skip column name headers
+ }
+ if (values.length == 9) {
+ AdminBoundary entry = new AdminBoundary(
+ values[0].toLowerCase().trim().replace("", ""),
+ values[3].toLowerCase().trim(),
+ values[1].toLowerCase().trim(),
+ values[4].toLowerCase().trim(),
+ values[2].toLowerCase().trim(),
+ values[5].toLowerCase().trim(),
+ values[6].toLowerCase().trim(),
+ values[7].toLowerCase().trim(),
+ values[8].toLowerCase().trim());
+ this.adminBoundaryData.add(entry);
+ } else {
+ throw new IllegalArgumentException("Improperly formatted file");
+ }
+
+ }
+ reader.close();
+ } catch (IOException ex) {
+ LOG.error(ex.getLocalizedMessage(), ex);
+ }
+
+ loadMaps(this.adminBoundaryData);
+
+ }
+
+ private void loadMaps(List<AdminBoundary> boundaries) {
+ for (AdminBoundary adm : boundaries) {
+ if (!adm.getCountryCode().equals("null")) {
+ countryMap.put(adm.getCountryCode(), adm.getCountryName());
+ if (countryRegexMap.containsKey(adm.getCountryCode())) {
+ String currentRegex = countryRegexMap.get(adm.getCountryCode());
+ if (currentRegex.length() > adm.getCountryRegex().length()) {
+ // the longest one wins if they are not all the same for each entry in the file
+ countryRegexMap.put(adm.getCountryCode(), currentRegex);
+ }//else do nothing
+ } else {
+ countryRegexMap.put(adm.getCountryCode(), adm.getCountryRegex());
+ }
+
+ if (!adm.getProvCode().equals("null")) {
+ Map<String, String> provs = provMap.get(adm.getCountryCode());
+ if (provs == null) {
+ provs = new HashMap<>();
+ }
+ //if (!provs.containsKey(adm.getProvCode())) {
+ provs.put(adm.getCountryCode() + "." + adm.getProvCode(), adm.getProvinceName());
+ provMap.put(adm.getCountryCode(), provs);
+ // }
+
+ if (!adm.getCountyCode().equalsIgnoreCase("no_data_found") && !adm.getCountyName().equalsIgnoreCase("no_data_found")) {
+ Map<String, String> counties = countyMap.get(adm.getCountryCode() + "." + adm.getProvCode());
+ if (counties == null) {
+ counties = new HashMap<>();
+ } // if (!counties.containsKey(adm.getCountyCode())) {
+ String countyid = adm.getCountryCode() + "." + adm.getProvCode() + "." + adm.getCountyCode();
+ counties.put(countyid, adm.getCountyName());
+ countyMap.put(adm.getCountryCode() + "." + adm.getProvCode(), counties);
+ // }
+
+ }
+
+ }
+ }
+ }
+ fillProvRegexMap();
+ fillCountyRegexMap();
+ }
+
+ private void fillProvRegexMap() {
+ this.provinceRegexMap = new HashMap<>();
+ // this.adminBoundaryData
+ for (AdminBoundary adm : adminBoundaryData) {
+
+ if (provinceRegexMap.containsKey(adm.getProvCode())) {
+ String currentRegex = provinceRegexMap.get(adm.getProvCode());
+ if (currentRegex.length() > adm.getProvinceRegex().length()) {
+ // the longest one wins if they are not all the same for each entry in the file
+ provinceRegexMap.put(adm.getProvCode(), currentRegex);
+ }//else do nothing
+ } else {
+ provinceRegexMap.put(adm.getProvCode(), adm.getProvinceRegex());
+ }
+ }
+ }
+
+ private void fillCountyRegexMap() {
+ this.countyRegexMap = new HashMap<>();
+ // this.adminBoundaryData
+ for (AdminBoundary adm : adminBoundaryData) {
+
+ if (countyRegexMap.containsKey(adm.getCountyCode())) {
+ String currentRegex = countyRegexMap.get(adm.getCountyCode());
+ if (currentRegex.length() > adm.getCountyRegex().length()) {
+ // the longest one wins if they are not all the same for each entry in the file
+ countyRegexMap.put(adm.getCountyCode(), currentRegex);
+ }//else do nothing
+ } else {
+ countyRegexMap.put(adm.getCountyCode(), adm.getCountyRegex());
+ }
+ }
+
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/CountryContextEntry.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/CountryContextEntry.java
index 324bd0f..04039e9 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/CountryContextEntry.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/CountryContextEntry.java
@@ -1,118 +1,118 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.util.Objects;
-
-/**
- *Stores a tuple from the opennlp.geoentitylinker.countrycontext.txt file, which is used to find country mentions in document text.
- *
- */
-public class CountryContextEntry {
- /*
- * rc,cc1, full_name_nd_ro,dsg
- */
-
- private String rc;
- private String cc1;
- private String full_name_nd_ro;
- private String dsg;
- private String provCode;
- public CountryContextEntry() {
- }
-
- public CountryContextEntry(String rc, String cc1, String full_name_nd_ro, String dsg) {
- this.rc = rc;
- this.cc1 = cc1;
- this.full_name_nd_ro = full_name_nd_ro;
- this.dsg = dsg;
- }
-
- public String getProvCode() {
- return provCode;
- }
-
- public void setProvCode(String provCode) {
- this.provCode = provCode;
- }
-
- public String getRc() {
- return rc;
- }
-
- public void setRc(String rc) {
- this.rc = rc;
- }
-
- public String getCc1() {
- return cc1;
- }
-
- public void setCc1(String cc1) {
- this.cc1 = cc1;
- }
-
- public String getFull_name_nd_ro() {
- return full_name_nd_ro;
- }
-
- public void setFull_name_nd_ro(String full_name_nd_ro) {
- this.full_name_nd_ro = full_name_nd_ro;
- }
-
- public String getDsg() {
- return dsg;
- }
-
- public void setDsg(String dsg) {
- this.dsg = dsg;
- }
-
- @Override
- public int hashCode() {
- int hash = 7;
- hash = 17 * hash + Objects.hashCode(this.rc);
- hash = 17 * hash + Objects.hashCode(this.cc1);
- hash = 17 * hash + Objects.hashCode(this.full_name_nd_ro);
- hash = 17 * hash + Objects.hashCode(this.dsg);
- return hash;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- final CountryContextEntry other = (CountryContextEntry) obj;
- if (!Objects.equals(this.rc, other.rc)) {
- return false;
- }
- if (!Objects.equals(this.cc1, other.cc1)) {
- return false;
- }
- if (!Objects.equals(this.full_name_nd_ro, other.full_name_nd_ro)) {
- return false;
- }
- if (!Objects.equals(this.dsg, other.dsg)) {
- return false;
- }
- return true;
- }
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.util.Objects;
+
+/**
+ *Stores a tuple from the opennlp.geoentitylinker.countrycontext.txt file, which is used to find country mentions in document text.
+ *
+ */
+public class CountryContextEntry {
+ /*
+ * rc,cc1, full_name_nd_ro,dsg
+ */
+
+ private String rc;
+ private String cc1;
+ private String full_name_nd_ro;
+ private String dsg;
+ private String provCode;
+ public CountryContextEntry() {
+ }
+
+ public CountryContextEntry(String rc, String cc1, String full_name_nd_ro, String dsg) {
+ this.rc = rc;
+ this.cc1 = cc1;
+ this.full_name_nd_ro = full_name_nd_ro;
+ this.dsg = dsg;
+ }
+
+ public String getProvCode() {
+ return provCode;
+ }
+
+ public void setProvCode(String provCode) {
+ this.provCode = provCode;
+ }
+
+ public String getRc() {
+ return rc;
+ }
+
+ public void setRc(String rc) {
+ this.rc = rc;
+ }
+
+ public String getCc1() {
+ return cc1;
+ }
+
+ public void setCc1(String cc1) {
+ this.cc1 = cc1;
+ }
+
+ public String getFull_name_nd_ro() {
+ return full_name_nd_ro;
+ }
+
+ public void setFull_name_nd_ro(String full_name_nd_ro) {
+ this.full_name_nd_ro = full_name_nd_ro;
+ }
+
+ public String getDsg() {
+ return dsg;
+ }
+
+ public void setDsg(String dsg) {
+ this.dsg = dsg;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 7;
+ hash = 17 * hash + Objects.hashCode(this.rc);
+ hash = 17 * hash + Objects.hashCode(this.cc1);
+ hash = 17 * hash + Objects.hashCode(this.full_name_nd_ro);
+ hash = 17 * hash + Objects.hashCode(this.dsg);
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final CountryContextEntry other = (CountryContextEntry) obj;
+ if (!Objects.equals(this.rc, other.rc)) {
+ return false;
+ }
+ if (!Objects.equals(this.cc1, other.cc1)) {
+ return false;
+ }
+ if (!Objects.equals(this.full_name_nd_ro, other.full_name_nd_ro)) {
+ return false;
+ }
+ if (!Objects.equals(this.dsg, other.dsg)) {
+ return false;
+ }
+ return true;
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerEntry.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerEntry.java
index 56e6eef..b7ef387 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerEntry.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerEntry.java
@@ -1,173 +1,173 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import opennlp.tools.entitylinker.BaseLink;
-
-/**
- * Stores a minimal amount of information from a geographic placenames gazetteer.
- */
-public class GazetteerEntry extends BaseLink {
-
- private Double latitude;
- private Double longitude;
- private String source;
- private String indexID;
- private Map<String, String> indexData = new HashMap<>();
- private String countryCode;
- private String provinceCode;
- private String hierarchy;
-
- public GazetteerEntry(String parentID, String itemID, String itemName, String itemType) {
- super(parentID, itemID, itemName, itemType);
- }
-
- /**
- * @return returns the id from the lucene document
- */
- public String getIndexID() {
- return indexID;
- }
- /**
- * sets the id from the lucene document
- */
- public void setIndexID(String indexID) {
- this.indexID = indexID;
- }
-
- /**
- * @return Retrieves the latitude from the gazetteer
- */
- public Double getLatitude() {
- return latitude;
- }
-
- /**
- * sets the latitude from the gazetteer
- */
- public void setLatitude(Double latitude) {
- this.latitude = latitude;
- }
-
- /**
- * @return Retrieves the longitude from the gaz
- */
- public Double getLongitude() {
- return longitude;
- }
-
- /**
- * sets the longitude from the gaz
- *
- * @param longitude
- */
- public void setLongitude(Double longitude) {
- this.longitude = longitude;
- }
-
- /**
- * @return Retrieves the source of the gazetteer data
- */
- public String getSource() {
- return source;
- }
-
- /**
- * sets the source (the source of the gazetteer data)
- *
- * @param source
- */
- public void setSource(String source) {
- this.source = source;
- }
-
- /**
- * @return Retrieves all the other fields in the gazetteer in the form of a map
- */
- public Map<String, String> getIndexData() {
- return indexData;
- }
-
- /**
- * sets the other fields in the gazetteer in the form of a map
- *
- * @param indexData stores all fields in the index as fieldname:value
- */
- public void setIndexData(Map<String, String> indexData) {
- this.indexData = indexData;
- }
-
- @Override
- public String toString() {
-
- return super.toString() + "\n\t\tGazateerEntry\n" + "\t\tlatitude=" + latitude + ", \n\t\tlongitude=" + longitude + ", \n\t\tsource=" + source + ", \n\t\tindexID=" + indexID + ",\n\t\tindexData=" + indexData + "\n";
- }
-
- @Override
- public int hashCode() {
- int hash = 5;
- hash = 71 * hash + Objects.hashCode(this.source);
- hash = 71 * hash + Objects.hashCode(this.indexID);
- return hash;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- final GazetteerEntry other = (GazetteerEntry) obj;
- if (!Objects.equals(this.source, other.source)) {
- return false;
- }
- if (!Objects.equals(this.indexID, other.indexID)) {
- return false;
- }
- return true;
- }
-
-
- public String getCountryCode() {
- return countryCode;
- }
-
- public void setCountryCode(String countryCode) {
- this.countryCode = countryCode;
- }
-
- public String getProvinceCode() {
- return provinceCode;
- }
-
- public void setProvinceCode(String provinceCode) {
- this.provinceCode = provinceCode;
- }
-
- public String getHierarchy() {
- return hierarchy;
- }
-
- public void setHierarchy(String hierarchy) {
- this.hierarchy = hierarchy;
- }
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import opennlp.tools.entitylinker.BaseLink;
+
+/**
+ * Stores a minimal amount of information from a geographic placenames gazetteer.
+ */
+public class GazetteerEntry extends BaseLink {
+
+ private Double latitude;
+ private Double longitude;
+ private String source;
+ private String indexID;
+ private Map<String, String> indexData = new HashMap<>();
+ private String countryCode;
+ private String provinceCode;
+ private String hierarchy;
+
+ public GazetteerEntry(String parentID, String itemID, String itemName, String itemType) {
+ super(parentID, itemID, itemName, itemType);
+ }
+
+ /**
+ * @return returns the id from the lucene document
+ */
+ public String getIndexID() {
+ return indexID;
+ }
+ /**
+ * sets the id from the lucene document
+ */
+ public void setIndexID(String indexID) {
+ this.indexID = indexID;
+ }
+
+ /**
+ * @return Retrieves the latitude from the gazetteer
+ */
+ public Double getLatitude() {
+ return latitude;
+ }
+
+ /**
+ * sets the latitude from the gazetteer
+ */
+ public void setLatitude(Double latitude) {
+ this.latitude = latitude;
+ }
+
+ /**
+ * @return Retrieves the longitude from the gaz
+ */
+ public Double getLongitude() {
+ return longitude;
+ }
+
+ /**
+ * sets the longitude from the gaz
+ *
+ * @param longitude
+ */
+ public void setLongitude(Double longitude) {
+ this.longitude = longitude;
+ }
+
+ /**
+ * @return Retrieves the source of the gazetteer data
+ */
+ public String getSource() {
+ return source;
+ }
+
+ /**
+ * sets the source (the source of the gazetteer data)
+ *
+ * @param source
+ */
+ public void setSource(String source) {
+ this.source = source;
+ }
+
+ /**
+ * @return Retrieves all the other fields in the gazetteer in the form of a map
+ */
+ public Map<String, String> getIndexData() {
+ return indexData;
+ }
+
+ /**
+ * sets the other fields in the gazetteer in the form of a map
+ *
+ * @param indexData stores all fields in the index as fieldname:value
+ */
+ public void setIndexData(Map<String, String> indexData) {
+ this.indexData = indexData;
+ }
+
+ @Override
+ public String toString() {
+
+ return super.toString() + "\n\t\tGazateerEntry\n" + "\t\tlatitude=" + latitude + ", \n\t\tlongitude=" + longitude + ", \n\t\tsource=" + source + ", \n\t\tindexID=" + indexID + ",\n\t\tindexData=" + indexData + "\n";
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 5;
+ hash = 71 * hash + Objects.hashCode(this.source);
+ hash = 71 * hash + Objects.hashCode(this.indexID);
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final GazetteerEntry other = (GazetteerEntry) obj;
+ if (!Objects.equals(this.source, other.source)) {
+ return false;
+ }
+ if (!Objects.equals(this.indexID, other.indexID)) {
+ return false;
+ }
+ return true;
+ }
+
+
+ public String getCountryCode() {
+ return countryCode;
+ }
+
+ public void setCountryCode(String countryCode) {
+ this.countryCode = countryCode;
+ }
+
+ public String getProvinceCode() {
+ return provinceCode;
+ }
+
+ public void setProvinceCode(String provinceCode) {
+ this.provinceCode = provinceCode;
+ }
+
+ public String getHierarchy() {
+ return hierarchy;
+ }
+
+ public void setHierarchy(String hierarchy) {
+ this.hierarchy = hierarchy;
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearchCache.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearchCache.java
index ac5b01e..7d13c81 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearchCache.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearchCache.java
@@ -1,49 +1,49 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- *
- * Caches gazateer query results statically. Clears itself if more than 10000 results are cached.
- */
-public class GazetteerSearchCache {
-
- private static Map<String, ArrayList<GazetteerEntry>> gazCache = new HashMap<>();
-
-/**
- * returns the cached entries. Returns null if the query does not exist in the cache
- * @param searchString
- * @return
- */
- public static synchronized ArrayList<GazetteerEntry> get(String searchString) {
- return gazCache.get(searchString);
- }
-
- public static synchronized void put(String searchString, ArrayList<GazetteerEntry> hits) {
- if (gazCache.size() > 10000) {
- gazCache.clear();
- }
- if (!gazCache.containsKey(searchString)) {
- gazCache.put(searchString, hits);
- }
- }
-
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ *
+ * Caches gazateer query results statically. Clears itself if more than 10000 results are cached.
+ */
+public class GazetteerSearchCache {
+
+ private static Map<String, ArrayList<GazetteerEntry>> gazCache = new HashMap<>();
+
+/**
+ * returns the cached entries. Returns null if the query does not exist in the cache
+ * @param searchString
+ * @return
+ */
+ public static synchronized ArrayList<GazetteerEntry> get(String searchString) {
+ return gazCache.get(searchString);
+ }
+
+ public static synchronized void put(String searchString, ArrayList<GazetteerEntry> hits) {
+ if (gazCache.size() > 10000) {
+ gazCache.clear();
+ }
+ if (!gazCache.containsKey(searchString)) {
+ gazCache.put(searchString, hits);
+ }
+ }
+
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearcher.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearcher.java
index 06e0365..4f5ad5d 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearcher.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearcher.java
@@ -1,251 +1,251 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.queryparser.classic.ParseException;
-
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
-import org.apache.lucene.analysis.util.CharArraySet;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- * Searches Gazetteers stored in a MMapDirectory Lucene index. The structure of
- * these indices are based on loading the indexes using the GazetteerIndexer
- *
- */
-public class GazetteerSearcher {
-
- private final String REGEX_CLEAN = "[^\\p{L}\\p{Nd}]";
- private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private double scoreCutoff = .70;
- private final boolean doubleQuoteAllSearchTerms = false;
- private boolean useHierarchyField = false;
-
- private final EntityLinkerProperties properties;
-
- private Directory opennlpIndex;//= new MMapDirectory(new File(indexloc));
- private IndexReader opennlpReader;// = DirectoryReader.open(geonamesIndex);
- private IndexSearcher opennlpSearcher;// = new IndexSearcher(geonamesReader);
- private Analyzer opennlpAnalyzer;
-
- public static void main(String[] args) {
- try {
- boolean b = true;
- new GazetteerSearcher(new EntityLinkerProperties(new File("c:\\temp\\entitylinker.properties"))).find("alabama", 5, " countrycode:us AND gazsource:usgs");
- } catch (IOException ex) {
- LOG.error(ex.getLocalizedMessage(), ex);
- }
- }
-
- public GazetteerSearcher(EntityLinkerProperties properties) throws IOException {
- this.properties = properties;
- init();
- }
-
- /**
- * Searches the single lucene index that includes the location hierarchy.
- *
- * @param searchString the location name to search for
- * @param rowsReturned how many index entries to return (top N...)
- * @param whereClause the conditional statement that defines the index type
- * and the country oode.
- * @return
- */
- public ArrayList<GazetteerEntry> find(String searchString, int rowsReturned, String whereClause) {
- ArrayList<GazetteerEntry> linkedData = new ArrayList<>();
- searchString = cleanInput(searchString);
- if (searchString.isEmpty()) {
- return linkedData;
- }
- try {
- /*
- * build the search string Sometimes no country context is found. In this
- * case the code variables will be empty strings
- */
- String placeNameQueryString = "placename:(" + searchString.toLowerCase() + ") " + "AND " + whereClause;
- if (searchString.trim().contains(" ") && useHierarchyField) {
- placeNameQueryString = "(placename:(" + searchString.toLowerCase() + ") AND hierarchy:(" + formatForHierarchy(searchString) + "))"
- + " AND " + whereClause;
- }
-
- /*
- * check the cache and go no further if the records already exist
- */
- ArrayList<GazetteerEntry> get = GazetteerSearchCache.get(placeNameQueryString);
- if (get != null) {
-
- return get;
- }
- /*
- * search the placename
- */
- QueryParser parser = new QueryParser(placeNameQueryString, opennlpAnalyzer);
- Query q = parser.parse(placeNameQueryString);
- //Filter filter = new QueryWrapperFilter(new QueryParser(Version.LUCENE_48, whereClause, opennlpAnalyzer).parse(whereClause));
-
- TopDocs bestDocs = opennlpSearcher.search(q, rowsReturned);
- Double maxscore = 0d;
- for (int i = 0; i < bestDocs.scoreDocs.length; ++i) {
- int docId = bestDocs.scoreDocs[i].doc;
- double sc = bestDocs.scoreDocs[i].score;
- if (maxscore.compareTo(sc) < 0) {
- maxscore = sc;
- }
- Document d = opennlpSearcher.doc(docId);
- List<IndexableField> fields = d.getFields();
-
- String lat = d.get("latitude");
- String lon = d.get("longitude");
- String placename = d.get("placename");
- String parentid = d.get("countrycode").toLowerCase();
- String provid = d.get("admincode");
- String itemtype = d.get("loctype");
- String source = d.get("gazsource");
- String hier = d.get("hierarchy");
-
- GazetteerEntry ge = new GazetteerEntry(parentid, String.valueOf(docId), placename, itemtype);
- ge.getScoreMap().put("lucene", sc);
- ge.setIndexID(String.valueOf(docId));
- ge.setSource(source);
- ge.setLatitude(Double.valueOf(lat));
- ge.setLongitude(Double.valueOf(lon));
- ge.setProvinceCode(provid);
- ge.setCountryCode(parentid);
- ge.setHierarchy(hier);
- for (IndexableField field : fields) {
- ge.getIndexData().put(field.name(), d.get(field.name()));
- }
-
- /*
- * only want hits above the levenshtein thresh. This should be a low
- * thresh due to the use of the hierarchy field in the index
- */
- // if (normLev > scoreCutoff) {
- if (ge.getItemParentID().equalsIgnoreCase(parentid) || parentid.equalsIgnoreCase("")) {
- //make sure we don't produce a duplicate
- if (!linkedData.contains(ge)) {
- linkedData.add(ge);
- /*
- * add the records to the cache for this query
- */
- GazetteerSearchCache.put(placeNameQueryString, linkedData);
- }
- }
- }
-
- } catch (IOException | ParseException ex) {
- LOG.error(ex.getLocalizedMessage(), ex);
- }
-
- return linkedData;
- }
-
- /**
- * Replaces any noise chars with a space, and depending on configuration adds
- * double quotes to the string
- *
- * @param input
- * @return
- */
- private String cleanInput(String input) {
- String output = input.replaceAll(REGEX_CLEAN, " ").trim();
- output = output.replace(" ", " ");
- if (doubleQuoteAllSearchTerms) {
- return "\"" + output + "\"";
- } else {
- return output;
- }
-
- }
-
- private void init() throws IOException {
-
- if (opennlpIndex == null) {
- String indexloc = properties.getProperty("opennlp.geoentitylinker.gaz", "");
- if (indexloc.equals("")) {
- LOG.error("Opennlp combined Gaz directory location not found!");
- }
-
- opennlpIndex = new MMapDirectory(Paths.get(indexloc));
- opennlpReader = DirectoryReader.open(opennlpIndex);
- opennlpSearcher = new IndexSearcher(opennlpReader);
- opennlpAnalyzer
- = //new StandardAnalyzer(Version.LUCENE_48, new CharArraySet(Version.LUCENE_48, new ArrayList(), true));
- new StandardAnalyzer(new CharArraySet(new ArrayList<>(), true));
- Map<String, Analyzer> analyMap = new HashMap<>();
-
- analyMap.put("countrycode", new KeywordAnalyzer());
- analyMap.put("admincode", new KeywordAnalyzer());
- analyMap.put("loctype", new KeywordAnalyzer());
- analyMap.put("countycode", new KeywordAnalyzer());
- analyMap.put("gazsource", new KeywordAnalyzer());
-
- opennlpAnalyzer
- = new PerFieldAnalyzerWrapper(opennlpAnalyzer, analyMap);
-
- String cutoff = properties.getProperty("opennlp.geoentitylinker.gaz.lucenescore.min", String.valueOf(scoreCutoff));
- String usehierarchy = properties.getProperty("opennlp.geoentitylinker.gaz.hierarchyfield", String.valueOf("0"));
- if (cutoff != null && !cutoff.isEmpty()) {
- scoreCutoff = Double.parseDouble(cutoff);
- }
- if (usehierarchy != null && !usehierarchy.isEmpty()) {
- useHierarchyField = Boolean.parseBoolean(usehierarchy);
- }
- // opennlp.geoentitylinker.gaz.doublequote=false
- //opennlp.geoentitylinker.gaz.hierarchyfield=false
-
- }
- }
-
- private String formatForHierarchy(String searchTerm) {
- String[] parts = cleanInput(searchTerm).split(" ");
- String out = "";
- if (parts.length != 0) {
- for (String string : parts) {
- out += string + " AND ";
- }
- out = out.substring(0, out.lastIndexOf(" AND "));
- } else {
- out = cleanInput(searchTerm);
- }
- return out;
- }
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.queryparser.classic.ParseException;
+
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * Searches Gazetteers stored in a MMapDirectory Lucene index. The structure of
+ * these indices are based on loading the indexes using the GazetteerIndexer
+ *
+ */
+public class GazetteerSearcher {
+
+ private final String REGEX_CLEAN = "[^\\p{L}\\p{Nd}]";
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private double scoreCutoff = .70;
+ private final boolean doubleQuoteAllSearchTerms = false;
+ private boolean useHierarchyField = false;
+
+ private final EntityLinkerProperties properties;
+
+ private Directory opennlpIndex;//= new MMapDirectory(new File(indexloc));
+ private IndexReader opennlpReader;// = DirectoryReader.open(geonamesIndex);
+ private IndexSearcher opennlpSearcher;// = new IndexSearcher(geonamesReader);
+ private Analyzer opennlpAnalyzer;
+
+ public static void main(String[] args) {
+ try {
+ boolean b = true;
+ new GazetteerSearcher(new EntityLinkerProperties(new File("c:\\temp\\entitylinker.properties"))).find("alabama", 5, " countrycode:us AND gazsource:usgs");
+ } catch (IOException ex) {
+ LOG.error(ex.getLocalizedMessage(), ex);
+ }
+ }
+
+ public GazetteerSearcher(EntityLinkerProperties properties) throws IOException {
+ this.properties = properties;
+ init();
+ }
+
+ /**
+ * Searches the single lucene index that includes the location hierarchy.
+ *
+ * @param searchString the location name to search for
+ * @param rowsReturned how many index entries to return (top N...)
+ * @param whereClause the conditional statement that defines the index type
+ * and the country oode.
+ * @return
+ */
+ public ArrayList<GazetteerEntry> find(String searchString, int rowsReturned, String whereClause) {
+ ArrayList<GazetteerEntry> linkedData = new ArrayList<>();
+ searchString = cleanInput(searchString);
+ if (searchString.isEmpty()) {
+ return linkedData;
+ }
+ try {
+ /*
+ * build the search string Sometimes no country context is found. In this
+ * case the code variables will be empty strings
+ */
+ String placeNameQueryString = "placename:(" + searchString.toLowerCase() + ") " + "AND " + whereClause;
+ if (searchString.trim().contains(" ") && useHierarchyField) {
+ placeNameQueryString = "(placename:(" + searchString.toLowerCase() + ") AND hierarchy:(" + formatForHierarchy(searchString) + "))"
+ + " AND " + whereClause;
+ }
+
+ /*
+ * check the cache and go no further if the records already exist
+ */
+ ArrayList<GazetteerEntry> get = GazetteerSearchCache.get(placeNameQueryString);
+ if (get != null) {
+
+ return get;
+ }
+ /*
+ * search the placename
+ */
+ QueryParser parser = new QueryParser(placeNameQueryString, opennlpAnalyzer);
+ Query q = parser.parse(placeNameQueryString);
+ //Filter filter = new QueryWrapperFilter(new QueryParser(Version.LUCENE_48, whereClause, opennlpAnalyzer).parse(whereClause));
+
+ TopDocs bestDocs = opennlpSearcher.search(q, rowsReturned);
+ Double maxscore = 0d;
+ for (int i = 0; i < bestDocs.scoreDocs.length; ++i) {
+ int docId = bestDocs.scoreDocs[i].doc;
+ double sc = bestDocs.scoreDocs[i].score;
+ if (maxscore.compareTo(sc) < 0) {
+ maxscore = sc;
+ }
+ Document d = opennlpSearcher.doc(docId);
+ List<IndexableField> fields = d.getFields();
+
+ String lat = d.get("latitude");
+ String lon = d.get("longitude");
+ String placename = d.get("placename");
+ String parentid = d.get("countrycode").toLowerCase();
+ String provid = d.get("admincode");
+ String itemtype = d.get("loctype");
+ String source = d.get("gazsource");
+ String hier = d.get("hierarchy");
+
+ GazetteerEntry ge = new GazetteerEntry(parentid, String.valueOf(docId), placename, itemtype);
+ ge.getScoreMap().put("lucene", sc);
+ ge.setIndexID(String.valueOf(docId));
+ ge.setSource(source);
+ ge.setLatitude(Double.valueOf(lat));
+ ge.setLongitude(Double.valueOf(lon));
+ ge.setProvinceCode(provid);
+ ge.setCountryCode(parentid);
+ ge.setHierarchy(hier);
+ for (IndexableField field : fields) {
+ ge.getIndexData().put(field.name(), d.get(field.name()));
+ }
+
+ /*
+ * only want hits above the levenshtein thresh. This should be a low
+ * thresh due to the use of the hierarchy field in the index
+ */
+ // if (normLev > scoreCutoff) {
+ if (ge.getItemParentID().equalsIgnoreCase(parentid) || parentid.equalsIgnoreCase("")) {
+ //make sure we don't produce a duplicate
+ if (!linkedData.contains(ge)) {
+ linkedData.add(ge);
+ /*
+ * add the records to the cache for this query
+ */
+ GazetteerSearchCache.put(placeNameQueryString, linkedData);
+ }
+ }
+ }
+
+ } catch (IOException | ParseException ex) {
+ LOG.error(ex.getLocalizedMessage(), ex);
+ }
+
+ return linkedData;
+ }
+
+ /**
+ * Replaces any noise chars with a space, and depending on configuration adds
+ * double quotes to the string
+ *
+ * @param input
+ * @return
+ */
+ private String cleanInput(String input) {
+ String output = input.replaceAll(REGEX_CLEAN, " ").trim();
+ output = output.replace(" ", " ");
+ if (doubleQuoteAllSearchTerms) {
+ return "\"" + output + "\"";
+ } else {
+ return output;
+ }
+
+ }
+
+ private void init() throws IOException {
+
+ if (opennlpIndex == null) {
+ String indexloc = properties.getProperty("opennlp.geoentitylinker.gaz", "");
+ if (indexloc.equals("")) {
+ LOG.error("Opennlp combined Gaz directory location not found!");
+ }
+
+ opennlpIndex = new MMapDirectory(Paths.get(indexloc));
+ opennlpReader = DirectoryReader.open(opennlpIndex);
+ opennlpSearcher = new IndexSearcher(opennlpReader);
+ opennlpAnalyzer
+ = //new StandardAnalyzer(Version.LUCENE_48, new CharArraySet(Version.LUCENE_48, new ArrayList(), true));
+ new StandardAnalyzer(new CharArraySet(new ArrayList<>(), true));
+ Map<String, Analyzer> analyMap = new HashMap<>();
+
+ analyMap.put("countrycode", new KeywordAnalyzer());
+ analyMap.put("admincode", new KeywordAnalyzer());
+ analyMap.put("loctype", new KeywordAnalyzer());
+ analyMap.put("countycode", new KeywordAnalyzer());
+ analyMap.put("gazsource", new KeywordAnalyzer());
+
+ opennlpAnalyzer
+ = new PerFieldAnalyzerWrapper(opennlpAnalyzer, analyMap);
+
+ String cutoff = properties.getProperty("opennlp.geoentitylinker.gaz.lucenescore.min", String.valueOf(scoreCutoff));
+ String usehierarchy = properties.getProperty("opennlp.geoentitylinker.gaz.hierarchyfield", String.valueOf("0"));
+ if (cutoff != null && !cutoff.isEmpty()) {
+ scoreCutoff = Double.parseDouble(cutoff);
+ }
+ if (usehierarchy != null && !usehierarchy.isEmpty()) {
+ useHierarchyField = Boolean.parseBoolean(usehierarchy);
+ }
+ // opennlp.geoentitylinker.gaz.doublequote=false
+ //opennlp.geoentitylinker.gaz.hierarchyfield=false
+
+ }
+ }
+
+ private String formatForHierarchy(String searchTerm) {
+ String[] parts = cleanInput(searchTerm).split(" ");
+ String out = "";
+ if (parts.length != 0) {
+ for (String string : parts) {
+ out += string + " AND ";
+ }
+ out = out.substring(0, out.lastIndexOf(" AND "));
+ } else {
+ out = cleanInput(searchTerm);
+ }
+ return out;
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GeoEntityLinker.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GeoEntityLinker.java
index 36de509..7fa49be 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GeoEntityLinker.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GeoEntityLinker.java
@@ -1,220 +1,220 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.io.IOException;
-import opennlp.addons.geoentitylinker.scoring.ModelBasedScorer;
-import opennlp.addons.geoentitylinker.scoring.LinkedEntityScorer;
-import opennlp.addons.geoentitylinker.scoring.CountryProximityScorer;
-import opennlp.addons.geoentitylinker.scoring.GeoHashBinningScorer;
-import opennlp.addons.geoentitylinker.scoring.FuzzyStringMatchScorer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import opennlp.addons.geoentitylinker.scoring.PlacetypeScorer;
-import opennlp.addons.geoentitylinker.scoring.ProvinceProximityScorer;
-import opennlp.tools.entitylinker.BaseLink;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.EntityLinker;
-
-/**
- * Links location entities to the USGS and GeoNames gazetteers, and uses several
- * scoring techniques to enable resolution. The gazetteers are stored in lucene
- * indexes. The indexes can be built using the GeoEntityLinkerSetupUtils class
- * in this same package.
- */
-public class GeoEntityLinker implements EntityLinker<LinkedSpan> {
-
- private static Integer topN = 2;
- private AdminBoundaryContextGenerator countryContext;
- private EntityLinkerProperties linkerProperties;
- private GazetteerSearcher gazateerSearcher;
- private final List<LinkedEntityScorer<AdminBoundaryContext>> scorers = new ArrayList<>();
-
- @Override
- public List<LinkedSpan> find(String doctext, Span[] sentences, Span[][] tokensBySentence, Span[][] namesBySentence) {
- ArrayList<LinkedSpan> spans = new ArrayList<>();
-
- if (linkerProperties == null) {
- throw new IllegalArgumentException("EntityLinkerProperties cannot be null");
- }
- //countryMentions = countryContext.regexfind(doctext);
- AdminBoundaryContext context = countryContext.getContext(doctext);
- for (int s = 0; s < sentences.length; s++) {
- Span[] names = namesBySentence[s];
-
- Span[] tokenSpans = tokensBySentence[s];
- String[] tokens = Span.spansToStrings(tokenSpans, sentences[s].getCoveredText(doctext));
-
- String[] matches = Span.spansToStrings(names, tokens);
-
- for (int i = 0; i < matches.length; i++) {
-
- ArrayList<BaseLink> geoNamesEntries = new ArrayList<>();
- if (!context.getWhereClauses().isEmpty()) {
- for (String whereclause : context.getWhereClauses()) {
- ArrayList<GazetteerEntry> find = gazateerSearcher.find(matches[i], topN, whereclause);
- for (GazetteerEntry gazetteerEntry : find) {
- if (!geoNamesEntries.contains(gazetteerEntry)) {
- geoNamesEntries.add(gazetteerEntry);
- }
- }
-
- }
- } else {//this means there were no where clauses generated so the where clause will default to look at the entire index
- ArrayList<GazetteerEntry> find = gazateerSearcher.find(matches[i], topN, " gaztype:usgs geonames regions ");
- for (GazetteerEntry gazetteerEntry : find) {
- if (!geoNamesEntries.contains(gazetteerEntry)) {
- geoNamesEntries.add(gazetteerEntry);
- }
- }
- }
- if (geoNamesEntries.isEmpty()) {
- continue;
- }
- /*
- * Normalize the returned scores for this name... this will assist the
- * sort
- */
- if (!spans.isEmpty()) {
-
- Double maxscore = 0d;
- for (BaseLink gazetteerEntry : geoNamesEntries) {
- Double deNormScore = gazetteerEntry.getScoreMap().get("lucene");
- if (deNormScore.compareTo(maxscore) > 0) {
- maxscore = deNormScore;
- }
- }
- for (BaseLink gazetteerEntry : geoNamesEntries) {
- Double deNormScore = gazetteerEntry.getScoreMap().get("lucene");
- Double normalize = normalize(deNormScore, 0d, maxscore);
- gazetteerEntry.getScoreMap().put("normlucene", normalize);
- }
- }
- LinkedSpan<BaseLink> newspan = new LinkedSpan<>(geoNamesEntries, names[i], 0);
- newspan.setSearchTerm(matches[i]);
- newspan.setLinkedEntries(geoNamesEntries);
- newspan.setSentenceid(s);
- spans.add(newspan);
- }
-
- }
-
- if (!scorers.isEmpty()) {
- for (LinkedEntityScorer scorer : scorers) {
- scorer.score(spans, doctext, sentences, linkerProperties, context);
- }
- }
- /*
- * sort the data with the best score on top based on the sum of the scores
- * below from the score map for each baselink object
- */
- for (LinkedSpan<BaseLink> s : spans) {
- ArrayList<BaseLink> linkedData = s.getLinkedEntries();
- linkedData.sort(Collections.reverseOrder((o1, o2) -> {
- Map<String, Double> o1scoreMap = o1.getScoreMap();
- Map<String, Double> o2scoreMap = o2.getScoreMap();
- if (o1scoreMap.size() != o2scoreMap.size()) {
- return 0;
- }
- double sumo1 = 0d;
- double sumo2 = 0d;
- for (String object : o1scoreMap.keySet()) {
- if (object.equals("typescore")
- || object.equals("countrycontext")
- || object.equals("placenamedicecoef")
- || object.equals("provincecontext")
- || object.equals("geohashbin")
- || object.equals("normlucene")) {
- sumo1 += o1scoreMap.get(object);
- sumo2 += o2scoreMap.get(object);
- }
- }
-
- return Double.compare(sumo1,
- sumo2);
- }));
- //prune the list to topN
- Iterator<BaseLink> iterator = linkedData.iterator();
- int n = 0;
- while (iterator.hasNext()) {
- if (n >= topN) {
- iterator.remove();
- }
- iterator.next();
- n++;
- }
- }
-
- return spans;
- }
-
- /**
- * transposes a value within one range to a relative value in a different
- * range. Used to normalize distances in this class.
- *
- * @param valueToNormalize the value to place within the new range
- * @param minimum the min of the set to be transposed
- * @param maximum the max of the set to be transposed
- * @return
- */
- private Double normalize(Double valueToNormalize, double minimum, double maximum) {
- double d = ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
- d = Double.isNaN(d) ? 0d : d;
- return d;
- }
-
- private void loadScorers() {
- if (scorers.isEmpty()) {
- scorers.add(new ProvinceProximityScorer());
- scorers.add(new GeoHashBinningScorer());
- scorers.add(new CountryProximityScorer());
- scorers.add(new ModelBasedScorer());
- scorers.add(new FuzzyStringMatchScorer());
- scorers.add(new PlacetypeScorer());
- }
- }
-
- @Override
- public void init(EntityLinkerProperties properties) throws IOException {
-
- this.linkerProperties = properties;
- countryContext = new AdminBoundaryContextGenerator(this.linkerProperties);
- gazateerSearcher = new GazetteerSearcher(this.linkerProperties);
- String rowsRetStr = this.linkerProperties.getProperty("opennlp.geoentitylinker.gaz.rowsreturned", "2");
- int rws;
- try {
- rws = Integer.parseInt(rowsRetStr);
- } catch (NumberFormatException e) {
- rws = 2;
- }
- topN = rws;
- loadScorers();
-
- }
-
- @Override
- public List<LinkedSpan> find(String doctext, Span[] sentences, Span[][] tokensBySentence,
- Span[][] namesBySentence, int sentenceIndex) {
- throw new UnsupportedOperationException("The GeoEntityLinker requires the entire document "
- + "for proper scoring. This method is unsupported");
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.io.IOException;
+import opennlp.addons.geoentitylinker.scoring.ModelBasedScorer;
+import opennlp.addons.geoentitylinker.scoring.LinkedEntityScorer;
+import opennlp.addons.geoentitylinker.scoring.CountryProximityScorer;
+import opennlp.addons.geoentitylinker.scoring.GeoHashBinningScorer;
+import opennlp.addons.geoentitylinker.scoring.FuzzyStringMatchScorer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import opennlp.addons.geoentitylinker.scoring.PlacetypeScorer;
+import opennlp.addons.geoentitylinker.scoring.ProvinceProximityScorer;
+import opennlp.tools.entitylinker.BaseLink;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.EntityLinker;
+
+/**
+ * Links location entities to the USGS and GeoNames gazetteers, and uses several
+ * scoring techniques to enable resolution. The gazetteers are stored in lucene
+ * indexes. The indexes can be built using the GeoEntityLinkerSetupUtils class
+ * in this same package.
+ */
+public class GeoEntityLinker implements EntityLinker<LinkedSpan> {
+
+ private static Integer topN = 2;
+ private AdminBoundaryContextGenerator countryContext;
+ private EntityLinkerProperties linkerProperties;
+ private GazetteerSearcher gazateerSearcher;
+ private final List<LinkedEntityScorer<AdminBoundaryContext>> scorers = new ArrayList<>();
+
+ @Override
+ public List<LinkedSpan> find(String doctext, Span[] sentences, Span[][] tokensBySentence, Span[][] namesBySentence) {
+ ArrayList<LinkedSpan> spans = new ArrayList<>();
+
+ if (linkerProperties == null) {
+ throw new IllegalArgumentException("EntityLinkerProperties cannot be null");
+ }
+ //countryMentions = countryContext.regexfind(doctext);
+ AdminBoundaryContext context = countryContext.getContext(doctext);
+ for (int s = 0; s < sentences.length; s++) {
+ Span[] names = namesBySentence[s];
+
+ Span[] tokenSpans = tokensBySentence[s];
+ String[] tokens = Span.spansToStrings(tokenSpans, sentences[s].getCoveredText(doctext));
+
+ String[] matches = Span.spansToStrings(names, tokens);
+
+ for (int i = 0; i < matches.length; i++) {
+
+ ArrayList<BaseLink> geoNamesEntries = new ArrayList<>();
+ if (!context.getWhereClauses().isEmpty()) {
+ for (String whereclause : context.getWhereClauses()) {
+ ArrayList<GazetteerEntry> find = gazateerSearcher.find(matches[i], topN, whereclause);
+ for (GazetteerEntry gazetteerEntry : find) {
+ if (!geoNamesEntries.contains(gazetteerEntry)) {
+ geoNamesEntries.add(gazetteerEntry);
+ }
+ }
+
+ }
+ } else {//this means there were no where clauses generated so the where clause will default to look at the entire index
+ ArrayList<GazetteerEntry> find = gazateerSearcher.find(matches[i], topN, " gaztype:usgs geonames regions ");
+ for (GazetteerEntry gazetteerEntry : find) {
+ if (!geoNamesEntries.contains(gazetteerEntry)) {
+ geoNamesEntries.add(gazetteerEntry);
+ }
+ }
+ }
+ if (geoNamesEntries.isEmpty()) {
+ continue;
+ }
+ /*
+ * Normalize the returned scores for this name... this will assist the
+ * sort
+ */
+ if (!spans.isEmpty()) {
+
+ Double maxscore = 0d;
+ for (BaseLink gazetteerEntry : geoNamesEntries) {
+ Double deNormScore = gazetteerEntry.getScoreMap().get("lucene");
+ if (deNormScore.compareTo(maxscore) > 0) {
+ maxscore = deNormScore;
+ }
+ }
+ for (BaseLink gazetteerEntry : geoNamesEntries) {
+ Double deNormScore = gazetteerEntry.getScoreMap().get("lucene");
+ Double normalize = normalize(deNormScore, 0d, maxscore);
+ gazetteerEntry.getScoreMap().put("normlucene", normalize);
+ }
+ }
+ LinkedSpan<BaseLink> newspan = new LinkedSpan<>(geoNamesEntries, names[i], 0);
+ newspan.setSearchTerm(matches[i]);
+ newspan.setLinkedEntries(geoNamesEntries);
+ newspan.setSentenceid(s);
+ spans.add(newspan);
+ }
+
+ }
+
+ if (!scorers.isEmpty()) {
+ for (LinkedEntityScorer scorer : scorers) {
+ scorer.score(spans, doctext, sentences, linkerProperties, context);
+ }
+ }
+ /*
+ * sort the data with the best score on top based on the sum of the scores
+ * below from the score map for each baselink object
+ */
+ for (LinkedSpan<BaseLink> s : spans) {
+ ArrayList<BaseLink> linkedData = s.getLinkedEntries();
+ linkedData.sort(Collections.reverseOrder((o1, o2) -> {
+ Map<String, Double> o1scoreMap = o1.getScoreMap();
+ Map<String, Double> o2scoreMap = o2.getScoreMap();
+ if (o1scoreMap.size() != o2scoreMap.size()) {
+ return 0;
+ }
+ double sumo1 = 0d;
+ double sumo2 = 0d;
+ for (String object : o1scoreMap.keySet()) {
+ if (object.equals("typescore")
+ || object.equals("countrycontext")
+ || object.equals("placenamedicecoef")
+ || object.equals("provincecontext")
+ || object.equals("geohashbin")
+ || object.equals("normlucene")) {
+ sumo1 += o1scoreMap.get(object);
+ sumo2 += o2scoreMap.get(object);
+ }
+ }
+
+ return Double.compare(sumo1,
+ sumo2);
+ }));
+ //prune the list to topN
+ Iterator<BaseLink> iterator = linkedData.iterator();
+ int n = 0;
+ while (iterator.hasNext()) {
+ if (n >= topN) {
+ iterator.remove();
+ }
+ iterator.next();
+ n++;
+ }
+ }
+
+ return spans;
+ }
+
+ /**
+ * transposes a value within one range to a relative value in a different
+ * range. Used to normalize distances in this class.
+ *
+ * @param valueToNormalize the value to place within the new range
+ * @param minimum the min of the set to be transposed
+ * @param maximum the max of the set to be transposed
+ * @return
+ */
+ private Double normalize(Double valueToNormalize, double minimum, double maximum) {
+ double d = ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
+ d = Double.isNaN(d) ? 0d : d;
+ return d;
+ }
+
+ private void loadScorers() {
+ if (scorers.isEmpty()) {
+ scorers.add(new ProvinceProximityScorer());
+ scorers.add(new GeoHashBinningScorer());
+ scorers.add(new CountryProximityScorer());
+ scorers.add(new ModelBasedScorer());
+ scorers.add(new FuzzyStringMatchScorer());
+ scorers.add(new PlacetypeScorer());
+ }
+ }
+
+ @Override
+ public void init(EntityLinkerProperties properties) throws IOException {
+
+ this.linkerProperties = properties;
+ countryContext = new AdminBoundaryContextGenerator(this.linkerProperties);
+ gazateerSearcher = new GazetteerSearcher(this.linkerProperties);
+ String rowsRetStr = this.linkerProperties.getProperty("opennlp.geoentitylinker.gaz.rowsreturned", "2");
+ int rws;
+ try {
+ rws = Integer.parseInt(rowsRetStr);
+ } catch (NumberFormatException e) {
+ rws = 2;
+ }
+ topN = rws;
+ loadScorers();
+
+ }
+
+ @Override
+ public List<LinkedSpan> find(String doctext, Span[] sentences, Span[][] tokensBySentence,
+ Span[][] namesBySentence, int sentenceIndex) {
+ throw new UnsupportedOperationException("The GeoEntityLinker requires the entire document "
+ + "for proper scoring. This method is unsupported");
+ }
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GazetteerIndexer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GazetteerIndexer.java
index 66e7bfe..d41b6ff 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GazetteerIndexer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GazetteerIndexer.java
@@ -1,210 +1,210 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.indexing;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.util.CharArraySet;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
-
-/**
- * Creates two lucene indexes, geonames and usgs for use in GeoEntityLinker.
- */
-public class GazetteerIndexer {
-
- public static void main(String[] args) {
-
- if (args.length != 8) {
- System.out.println("Usage: GazetteerIndexer geonamesData geoNamesCountryInfo geonamesAdmin1CodesASCII "
- + "usgsDataFile usgsGovUnitsFile outputIndexDir outputCountryContextFile regionsFile");
- System.out.println();
- System.out.println("The GazetteerIndexer.index methods javadoc explains how to retrieve the data files.");
- return;
- }
-
- File geonamesData = new File(args[0]);
- File geoNamesCountryInfo = new File(args[1]);
- File geonamesAdmin1CodesASCII = new File(args[2]);
- File usgsDataFile = new File(args[3]);
- File usgsGovUnitsFile = new File(args[4]);
- File outputIndexDir = new File(args[5]);
- File outputCountryContextFile = new File(args[6]);
- File regionsFile = new File(args[7]);
-
- try {
- GazetteerIndexer i = new GazetteerIndexer();
- i.index(geonamesData,
- geoNamesCountryInfo,
- geonamesAdmin1CodesASCII,
- usgsDataFile,
- usgsGovUnitsFile,
- outputIndexDir,
- outputCountryContextFile,
- regionsFile);
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
-
- public GazetteerIndexer() {
-
- }
-
- public interface Separable {
-
- String getSeparator();
- }
-
- public enum GazType implements Separable {
-
- GEONAMES {
- @Override
- public String toString() {
- return "/opennlp_geoentitylinker_geonames_idx";
- }
-
- @Override
- public String getSeparator() {
- return "\t";
- }
- },
- USGS {
- @Override
- public String toString() {
- return "/opennlp_geoentitylinker_usgsgaz_idx";
- }
-
- @Override
- public String getSeparator() {
- return "\\|";
- }
- }
- }
-
- /**
- *
- * @param geonamesData the actual Geonames gazetteer data downloaded from
- * <a href="https://download.geonames.org/export/dump/">here</a>, then click on this link
- * 'allCountries.zip'
- *
- * @param geoNamesCountryInfo the countryinfo lookup table that can be
- * downloaded from
- * <a href="https://download.geonames.org/export/dump/countryInfo.txt">here</a>. You'll need to
- * copy the page into a file or scrape it
- *
- * @param geonamesAdmin1CodesASCII The lookup data for the province names for
- * each place found
- * <a href="https://download.geonames.org/export/dump/admin1CodesASCII.txt">here</a>, highlight the
- * table view, and copy results into a text file. Make sure the tab delimited
- * format is maintained.
- *
- * @param usgsDataFile the actual USGS gazetteer downloaded from
- * <a href="https://geonames.usgs.gov/domestic/download_data.htm">here</a> click on the
- * national_file####.zip link to get all the most recent features
- *
- * @param usgsGovUnitsFile go to
- * <a href="https://geonames.usgs.gov/domestic/download_data.htm">here</a> in the section titled
- * "Topical Gazetteers -- File Format" click on the dropdown list and select
- * "Government Units". The downloaded file is what you need for this param.
- *
- * @param outputIndexDir where you want the final index. Must be a directory, not an actual file.
- *
- * @param outputCountryContextFile The output countrycontext file. This is a
- * very important file used inside the GeoEntityLinker to assist in toponym
- * resolution.
- *
- * @param regionsFile this file contains a list of regions in the following
- * format: tab delimited text with index 0 as the name of the region, index 1
- * as the longitude, and index 2 as the latitude
- *
- * @throws Exception
- */
- public void index(File geonamesData, File geoNamesCountryInfo, File geonamesAdmin1CodesASCII, File usgsDataFile,
- File usgsGovUnitsFile, File outputIndexDir, File outputCountryContextFile, File regionsFile)
- throws Exception {
- if (!outputIndexDir.isDirectory()) {
- throw new IllegalArgumentException("outputIndexDir must be a directory.");
- }
- if (!geonamesData.exists()) {
- throw new FileNotFoundException("geonames data file does not exist");
- }
- if (!geoNamesCountryInfo.exists()) {
- throw new FileNotFoundException("geoNamesCountryCodes data file does not exist");
- }
- if (!geonamesAdmin1CodesASCII.exists()) {
- throw new FileNotFoundException("geonamesAdmin1CodesASCII data file does not exist");
- }
-
- if (!usgsDataFile.exists()) {
- throw new FileNotFoundException("usgsDataFile data file does not exist");
- }
- if (!usgsGovUnitsFile.exists()) {
- throw new FileNotFoundException("usgsGovUnitsFile data file does not exist");
- }
- if (!outputIndexDir.exists()) {
- throw new FileNotFoundException("outputIndexDir data file does not exist");
- }
- if (!regionsFile.exists()) {
- throw new FileNotFoundException("regionsFile data file does not exist");
- }
-
- String indexloc = outputIndexDir.getPath() + "/opennlp_geoentitylinker_gazetteer";
- Directory index = new MMapDirectory(Paths.get(indexloc));
- Analyzer a = new StandardAnalyzer(new CharArraySet(new ArrayList<>(), true));
- Map<String, Analyzer> analyMap = new HashMap<>();
-
- analyMap.put("countrycode", new KeywordAnalyzer());
- analyMap.put("admincode", new KeywordAnalyzer());
- analyMap.put("loctype", new KeywordAnalyzer());
- analyMap.put("countycode", new KeywordAnalyzer());
- analyMap.put("gazsource", new KeywordAnalyzer());
-
- PerFieldAnalyzerWrapper aWrapper = new PerFieldAnalyzerWrapper(a, analyMap);
- IndexWriterConfig config = new IndexWriterConfig(aWrapper);
- try (IndexWriter w = new IndexWriter(index, config)) {
- //write the column headers for the countryContextFile
- try (FileWriter countryContextFileWriter = new FileWriter(outputCountryContextFile, false)) {
- String colNamesForCountryContextFile = "countrycode\tprovcode\tcountycode\tcountryname\tprovincename\tcountyname\tcountryregex\tprovregex\tcountyregex\n";
- countryContextFileWriter.write(colNamesForCountryContextFile);
- countryContextFileWriter.flush();
- }
-
- USGSProcessor.process(usgsGovUnitsFile, usgsDataFile, outputCountryContextFile, w);
- GeonamesProcessor.process(geoNamesCountryInfo, geonamesAdmin1CodesASCII, geonamesData, outputCountryContextFile, w);
-
- RegionProcessor.process(regionsFile, outputCountryContextFile, w);
- w.commit();
- }
-
- System.out.println("\nIndexing complete. Be sure to add '" + indexloc + "' and context file '" +
- outputCountryContextFile.getPath() + "' to entitylinker.properties file");
- }
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.indexing;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+
+/**
+ * Creates two lucene indexes, geonames and usgs for use in GeoEntityLinker.
+ */
+public class GazetteerIndexer {
+
+ public static void main(String[] args) {
+
+ if (args.length != 8) {
+ System.out.println("Usage: GazetteerIndexer geonamesData geoNamesCountryInfo geonamesAdmin1CodesASCII "
+ + "usgsDataFile usgsGovUnitsFile outputIndexDir outputCountryContextFile regionsFile");
+ System.out.println();
+ System.out.println("The GazetteerIndexer.index methods javadoc explains how to retrieve the data files.");
+ return;
+ }
+
+ File geonamesData = new File(args[0]);
+ File geoNamesCountryInfo = new File(args[1]);
+ File geonamesAdmin1CodesASCII = new File(args[2]);
+ File usgsDataFile = new File(args[3]);
+ File usgsGovUnitsFile = new File(args[4]);
+ File outputIndexDir = new File(args[5]);
+ File outputCountryContextFile = new File(args[6]);
+ File regionsFile = new File(args[7]);
+
+ try {
+ GazetteerIndexer i = new GazetteerIndexer();
+ i.index(geonamesData,
+ geoNamesCountryInfo,
+ geonamesAdmin1CodesASCII,
+ usgsDataFile,
+ usgsGovUnitsFile,
+ outputIndexDir,
+ outputCountryContextFile,
+ regionsFile);
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ public GazetteerIndexer() {
+
+ }
+
+ public interface Separable {
+
+ String getSeparator();
+ }
+
+ public enum GazType implements Separable {
+
+ GEONAMES {
+ @Override
+ public String toString() {
+ return "/opennlp_geoentitylinker_geonames_idx";
+ }
+
+ @Override
+ public String getSeparator() {
+ return "\t";
+ }
+ },
+ USGS {
+ @Override
+ public String toString() {
+ return "/opennlp_geoentitylinker_usgsgaz_idx";
+ }
+
+ @Override
+ public String getSeparator() {
+ return "\\|";
+ }
+ }
+ }
+
+ /**
+ *
+ * @param geonamesData the actual Geonames gazetteer data downloaded from
+ * <a href="https://download.geonames.org/export/dump/">here</a>, then click on this link
+ * 'allCountries.zip'
+ *
+ * @param geoNamesCountryInfo the countryinfo lookup table that can be
+ * downloaded from
+ * <a href="https://download.geonames.org/export/dump/countryInfo.txt">here</a>. You'll need to
+ * copy the page into a file or scrape it
+ *
+ * @param geonamesAdmin1CodesASCII The lookup data for the province names for
+ * each place found
+ * <a href="https://download.geonames.org/export/dump/admin1CodesASCII.txt">here</a>, highlight the
+ * table view, and copy results into a text file. Make sure the tab delimited
+ * format is maintained.
+ *
+ * @param usgsDataFile the actual USGS gazetteer downloaded from
+ * <a href="https://geonames.usgs.gov/domestic/download_data.htm">here</a> click on the
+ * national_file####.zip link to get all the most recent features
+ *
+ * @param usgsGovUnitsFile go to
+ * <a href="https://geonames.usgs.gov/domestic/download_data.htm">here</a> in the section titled
+ * "Topical Gazetteers -- File Format" click on the dropdown list and select
+ * "Government Units". The downloaded file is what you need for this param.
+ *
+ * @param outputIndexDir where you want the final index. Must be a directory, not an actual file.
+ *
+ * @param outputCountryContextFile The output countrycontext file. This is a
+ * very important file used inside the GeoEntityLinker to assist in toponym
+ * resolution.
+ *
+ * @param regionsFile this file contains a list of regions in the following
+ * format: tab delimited text with index 0 as the name of the region, index 1
+ * as the longitude, and index 2 as the latitude
+ *
+ * @throws Exception
+ */
+ public void index(File geonamesData, File geoNamesCountryInfo, File geonamesAdmin1CodesASCII, File usgsDataFile,
+ File usgsGovUnitsFile, File outputIndexDir, File outputCountryContextFile, File regionsFile)
+ throws Exception {
+ if (!outputIndexDir.isDirectory()) {
+ throw new IllegalArgumentException("outputIndexDir must be a directory.");
+ }
+ if (!geonamesData.exists()) {
+ throw new FileNotFoundException("geonames data file does not exist");
+ }
+ if (!geoNamesCountryInfo.exists()) {
+ throw new FileNotFoundException("geoNamesCountryCodes data file does not exist");
+ }
+ if (!geonamesAdmin1CodesASCII.exists()) {
+ throw new FileNotFoundException("geonamesAdmin1CodesASCII data file does not exist");
+ }
+
+ if (!usgsDataFile.exists()) {
+ throw new FileNotFoundException("usgsDataFile data file does not exist");
+ }
+ if (!usgsGovUnitsFile.exists()) {
+ throw new FileNotFoundException("usgsGovUnitsFile data file does not exist");
+ }
+ if (!outputIndexDir.exists()) {
+ throw new FileNotFoundException("outputIndexDir data file does not exist");
+ }
+ if (!regionsFile.exists()) {
+ throw new FileNotFoundException("regionsFile data file does not exist");
+ }
+
+ String indexloc = outputIndexDir.getPath() + "/opennlp_geoentitylinker_gazetteer";
+ Directory index = new MMapDirectory(Paths.get(indexloc));
+ Analyzer a = new StandardAnalyzer(new CharArraySet(new ArrayList<>(), true));
+ Map<String, Analyzer> analyMap = new HashMap<>();
+
+ analyMap.put("countrycode", new KeywordAnalyzer());
+ analyMap.put("admincode", new KeywordAnalyzer());
+ analyMap.put("loctype", new KeywordAnalyzer());
+ analyMap.put("countycode", new KeywordAnalyzer());
+ analyMap.put("gazsource", new KeywordAnalyzer());
+
+ PerFieldAnalyzerWrapper aWrapper = new PerFieldAnalyzerWrapper(a, analyMap);
+ IndexWriterConfig config = new IndexWriterConfig(aWrapper);
+ try (IndexWriter w = new IndexWriter(index, config)) {
+ //write the column headers for the countryContextFile
+ try (FileWriter countryContextFileWriter = new FileWriter(outputCountryContextFile, false)) {
+ String colNamesForCountryContextFile = "countrycode\tprovcode\tcountycode\tcountryname\tprovincename\tcountyname\tcountryregex\tprovregex\tcountyregex\n";
+ countryContextFileWriter.write(colNamesForCountryContextFile);
+ countryContextFileWriter.flush();
+ }
+
+ USGSProcessor.process(usgsGovUnitsFile, usgsDataFile, outputCountryContextFile, w);
+ GeonamesProcessor.process(geoNamesCountryInfo, geonamesAdmin1CodesASCII, geonamesData, outputCountryContextFile, w);
+
+ RegionProcessor.process(regionsFile, outputCountryContextFile, w);
+ w.commit();
+ }
+
+ System.out.println("\nIndexing complete. Be sure to add '" + indexloc + "' and context file '" +
+ outputCountryContextFile.getPath() + "' to entitylinker.properties file");
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesFileDownloader.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesFileDownloader.java
index 7d27e3f..58d680c 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesFileDownloader.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesFileDownloader.java
@@ -1,120 +1,120 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.indexing;
-
-import java.io.BufferedOutputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URL;
-import java.util.Enumeration;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
-
-public class GeonamesFileDownloader {
-
- final static int size = 1024;
- private static final String ALL_COUNTRIES = "http://download.geonames.org/export/dump/ZM.zip";
- private static final String COUNTRY_INFO = "";
- private static final String ADM1_LOOKUP = "";
-
- public static void main(String[] args) {
- downloadGeonamesFiles(COUNTRY_INFO, "c:\\temp\\gazetteers");
- }
-
- public static void downloadGeonamesFiles(String outputFileName, String outputDir) {
- String fileDownload = fileDownload(ALL_COUNTRIES, outputDir);
-
- unzipMyZip(fileDownload, outputDir);
-
- fileDownload(COUNTRY_INFO, outputDir);
- fileDownload(ADM1_LOOKUP, outputDir);
-
- }
-
- public static void writeFile(InputStream in, OutputStream out)
- throws IOException {
- byte[] buffer = new byte[1024];
- int len;
-
- while ((len = in.read(buffer)) != 0) {
- out.write(buffer, 0, len);
- }
-
- in.close();
- out.close();
- }
-
- public static void unzipMyZip(String zipFileName, String directoryToExtractTo) {
- Enumeration<? extends ZipEntry> entriesEnum;
- ZipFile zip;
- try {
- zip = new ZipFile(zipFileName);
- entriesEnum = zip.entries();
- while (entriesEnum.hasMoreElements()) {
- ZipEntry entry = entriesEnum.nextElement();
- InputStream is = zip.getInputStream(entry); // get the input stream
- OutputStream os = new FileOutputStream(zipFileName.replace("\\.zip", ".txt"));
- byte[] buf = new byte[4096];
- int r;
- while ((r = is.read(buf)) != -1) {
- os.write(buf, 0, r);
- }
- os.close();
- is.close();
- }
- } catch (IOException ioe) {
- System.err.println("Some Exception Occurred:");
- ioe.printStackTrace();
- }
- }
-
- public static String fileUrl(String fAddress, String localFileName, String destDir) {
- String filename = destDir + "\\" + localFileName;
- try (InputStream is = new URL(fAddress).openConnection().getInputStream();
- OutputStream outStream = new BufferedOutputStream(new FileOutputStream(destDir + "\\" + localFileName))) {
-
- byte[] buf = new byte[size];
- int byteRead, byteWritten = 0;
- while ((byteRead = is.read(buf)) != -1) {
- outStream.write(buf, 0, byteRead);
- byteWritten += byteRead;
- }
- System.out.println("Downloaded Successfully.");
- System.out.println("File name:\"" + localFileName + "\"\nNo ofbytes :" + byteWritten);
- } catch (Exception e) {
- e.printStackTrace();
- }
- return filename;
- }
-
- public static String fileDownload(String fAddress, String destinationDir) {
- int slashIndex = fAddress.lastIndexOf('/');
- int periodIndex = fAddress.lastIndexOf('.');
-
- String fileName = fAddress.substring(slashIndex + 1);
- String retFileName = "";
- if (periodIndex >= 1 && slashIndex >= 0
- && slashIndex < fAddress.length() - 1) {
- retFileName = fileUrl(fAddress, fileName, destinationDir);
- } else {
- System.err.println("path or file name.");
- }
- return retFileName;
- }
-
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.indexing;
+
+import java.io.BufferedOutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.Enumeration;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+public class GeonamesFileDownloader {
+
+ final static int size = 1024;
+ private static final String ALL_COUNTRIES = "http://download.geonames.org/export/dump/ZM.zip";
+ private static final String COUNTRY_INFO = "";
+ private static final String ADM1_LOOKUP = "";
+
+ public static void main(String[] args) {
+ downloadGeonamesFiles(COUNTRY_INFO, "c:\\temp\\gazetteers");
+ }
+
+ public static void downloadGeonamesFiles(String outputFileName, String outputDir) {
+ String fileDownload = fileDownload(ALL_COUNTRIES, outputDir);
+
+ unzipMyZip(fileDownload, outputDir);
+
+ fileDownload(COUNTRY_INFO, outputDir);
+ fileDownload(ADM1_LOOKUP, outputDir);
+
+ }
+
+ public static void writeFile(InputStream in, OutputStream out)
+ throws IOException {
+ byte[] buffer = new byte[1024];
+ int len;
+
+ while ((len = in.read(buffer)) != 0) {
+ out.write(buffer, 0, len);
+ }
+
+ in.close();
+ out.close();
+ }
+
+ public static void unzipMyZip(String zipFileName, String directoryToExtractTo) {
+ Enumeration<? extends ZipEntry> entriesEnum;
+ ZipFile zip;
+ try {
+ zip = new ZipFile(zipFileName);
+ entriesEnum = zip.entries();
+ while (entriesEnum.hasMoreElements()) {
+ ZipEntry entry = entriesEnum.nextElement();
+ InputStream is = zip.getInputStream(entry); // get the input stream
+ OutputStream os = new FileOutputStream(zipFileName.replace("\\.zip", ".txt"));
+ byte[] buf = new byte[4096];
+ int r;
+ while ((r = is.read(buf)) != -1) {
+ os.write(buf, 0, r);
+ }
+ os.close();
+ is.close();
+ }
+ } catch (IOException ioe) {
+ System.err.println("Some Exception Occurred:");
+ ioe.printStackTrace();
+ }
+ }
+
+ public static String fileUrl(String fAddress, String localFileName, String destDir) {
+ String filename = destDir + "\\" + localFileName;
+ try (InputStream is = new URL(fAddress).openConnection().getInputStream();
+ OutputStream outStream = new BufferedOutputStream(new FileOutputStream(destDir + "\\" + localFileName))) {
+
+ byte[] buf = new byte[size];
+ int byteRead, byteWritten = 0;
+ while ((byteRead = is.read(buf)) != -1) {
+ outStream.write(buf, 0, byteRead);
+ byteWritten += byteRead;
+ }
+ System.out.println("Downloaded Successfully.");
+ System.out.println("File name:\"" + localFileName + "\"\nNo ofbytes :" + byteWritten);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return filename;
+ }
+
+ public static String fileDownload(String fAddress, String destinationDir) {
+ int slashIndex = fAddress.lastIndexOf('/');
+ int periodIndex = fAddress.lastIndexOf('.');
+
+ String fileName = fAddress.substring(slashIndex + 1);
+ String retFileName = "";
+ if (periodIndex >= 1 && slashIndex >= 0
+ && slashIndex < fAddress.length() - 1) {
+ retFileName = fileUrl(fAddress, fileName, destinationDir);
+ } else {
+ System.err.println("path or file name.");
+ }
+ return retFileName;
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesProcessor.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesProcessor.java
index f560d33..c58c92d 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesProcessor.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesProcessor.java
@@ -1,296 +1,296 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.indexing;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import opennlp.addons.geoentitylinker.AdminBoundary;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexWriter;
-
-public class GeonamesProcessor {
-
- public static void process(File countryCodesLookupFile, File adm1CodesLookupFile, File geonamesGazetteerFile, File outputCountryContextFile, IndexWriter w) throws Exception {
- Map<String, String> countryCodes = getCountryCodes(countryCodesLookupFile);
-
- Map<String, AdminBoundary> adm1s = getProvData(adm1CodesLookupFile, countryCodes);
- // List<AdminBoundary> adm2s = getCountryContextFromFile(new File("C:\\temp\\gazetteers\\geonamesdata\\admin2Codes.txt"));
- //admin2Codes.txt
-
- readFile(geonamesGazetteerFile, GazetteerIndexer.GazType.GEONAMES, adm1s, countryCodes, w);
- //now append to the coutnry context file
- writeCountryContextFile(outputCountryContextFile, adm1s);
-
- }
-
- public GeonamesProcessor() {
- }
-
- private static Map<String, AdminBoundary> getProvData(File adm1CodesLookupFile, Map<String, String> ccodes) {
- System.out.println("Attempting to read geonames province data from: " + adm1CodesLookupFile.getPath());
-
- Map<String, AdminBoundary> outmap = new HashMap<>();
- BufferedReader reader;
- Set<String> nullcodes = new HashSet<>();
- try {
-
- reader = new BufferedReader(new FileReader(adm1CodesLookupFile));
- int i = 0;
- String line;
- while ((line = reader.readLine()) != null) {
-
- // String line = reader.readLine();
- String[] values = line.split("\t");
- if (values.length != 4) {
- throw new IOException("improperly formatted province lookup file");
- }
- String ccode = values[0].toLowerCase();
-
- String[] split = ccode.split("\\.");
- String pcode = "";
- if (split.length == 2) {
- //System.out.println(split);
- ccode = split[0];
- pcode = split[1];
- }
-
- String pname = values[2];
-
- if (ccode.matches("[0-9].*")) {
- String code = ccode;
- ccode = pcode;
- pcode = code;
- }
-
- String cname = ccodes.get(ccode);
-
- if (cname == null) {
- nullcodes.add(ccode);
- }
- AdminBoundary data = new AdminBoundary(ccode, cname, pcode, pname, "NO_DATA_FOUND", "NO_DATA_FOUND", cname, pname, "NO_DATA_FOUND");
- // System.out.println(data);
- outmap.put(ccode + "." + pcode, data);
-
- }
- System.out.println("INFO: there were " + nullcodes.size() + " null prov codes. This is due to inconsistencies in reference data.");
- reader.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- System.out.println("Successfully read geonames province data from: " + adm1CodesLookupFile.getPath());
-
- return outmap;
-
- }
-
- private static Map<String, String> getCountryCodes(File countryContextFile) {
- Map<String, String> ccs = new HashMap<>();
- BufferedReader reader;
- try {
-
- reader = new BufferedReader(new FileReader(countryContextFile));
- int i = 0;
- String line;
- boolean start = false;
- while ((line = reader.readLine()) != null) {
- if (!line.toLowerCase().startsWith("#iso\t") && !start) {
-
- continue;
- } else {
- start = true;
- }
- String[] values = line.split("\t");
-
- String ccode = values[0].toLowerCase();//this is the 2 digit ISO code
- String cname = values[4].toLowerCase();
- if (!ccode.equals("")) {
- ccs.put(ccode, cname);
- }
-
- }
- reader.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- ccs.put("SS", "South Sudan");
- ccs.put("CS", "Kosovo");
- return ccs;
-
- }
-
- public static void writeCountryContextFile(File outfile, Map<String, AdminBoundary> adms) {
- // FileWriter writer = null;
- try (FileWriter writer = new FileWriter(outfile, true)) {
- BufferedWriter bw = new BufferedWriter(writer);
-
- for (String admKey : adms.keySet()) {
- AdminBoundary adm = adms.get(admKey);
- if (adm == null) {
- continue;
- }
- String province = adm.getProvinceName();
- String country = adm.getCountryName();
-
- String line = adm.getCountryCode() + "\t" + adm.getProvCode() + "\t" + "\t" + country + "\t" + province + "\t" + "\t" + "(" + country + ")" + "\t"
- + adm.getProvinceName() + "\t" + adm.getCountyName() + "\n";
- bw.write(line);
- // System.out.println(line);
-
- }
-
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- System.out.println("successfully wrote Geonames entries to country oontext file");
- }
-
- /**
- *
- * @param gazateerInputData the Geonames allCounties.txt file
- * @param type the types of gaz entry, usgs, geonames, or regions
- * @param adms the province info
- * @param countrycodes the country code info
- * @param w the lucene index writer
- * @throws Exception
- */
- public static void readFile(File gazateerInputData, GazetteerIndexer.GazType type, Map<String, AdminBoundary> adms, Map<String, String> countrycodes, IndexWriter w) throws Exception {
-
- BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData));
- String[] boosts = "ADM1 ADM1H ADM2 ADM2H ADM3 ADM3H ADM4 ADM4H ADM5 ADMD ADMDH PCLD PCLH PCLI PCLIX TERR PCLIX PPL PPLA PPLA2 PPLA3 PPLA4 PPLC PPLCH PPLF PPLG PPLH PPLL PPLQ PPLR PPLS PPLX STLMT".split(" ");
- Map<String, Float> boostMap = new HashMap<>();
- for (String boost : boosts) {
- boostMap.put(boost.toLowerCase(), 10f);
- }
- String[] fieldStrings = new String[]{
- "geonameid",
- "name",
- "asciiname",
- "alternatenames",
- "latitude",
- "longitude",
- "feature_class",
- "feature_code",
- "country code",
- "cc2",
- "admin1_code",
- "admin2_code",
- "admin3_code",
- "admin4_code",
- "population",
- "elevation",
- "dem ",
- "timezone",
- "modification_date"};
-
- List<String> fields = Arrays.asList(fieldStrings);
- int counter = 0;
- System.out.println("reading gazetteer data from file...........");
- String line;
- while ((line = reader.readLine()) != null) {
- String[] values = line.split(type.getSeparator());
-
- Document doc = new Document();
- String admincode = values[10].toLowerCase();
- String ccode = values[8].toLowerCase();
- if (ccode.contains(",")) {
- String[] codes = ccode.split(",");
- if (codes.length > 0) {
- ccode = codes[0];
- }
- }
- AdminBoundary adm = adms.get(ccode + "." + admincode);
-
- String placeName = values[2];
- String lat = values[4];
- String lon = values[5];
- String dsg = values[7].toLowerCase();
-
- String id = values[0];
- String concatIndexEntry;
- String countryname;
- if (adm != null) {
- concatIndexEntry = adm.getCountryName() + ", " + adm.getProvinceName() + ", " + placeName;
- countryname = adm.getCountryName();
- } else {
- //there is no admin info, but we can still use the countrycode to concat the country name
- String n = countrycodes.get(ccode);
- countryname = n;
- if (n != null) {
- concatIndexEntry = n + ", " + placeName;
- } else {
- ///don't want a single token hierarchy entry.
- concatIndexEntry = "";
- }
- }
- if (ccode == null) {
- System.out.println("naughty country code");
- }
- for (int i = 0; i < fields.size() - 1; i++) {
- doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
-
- }
- if (dsg.equals("pcli")) {
- System.out.println("placename: " + placeName + " RESET TO: " + countryname);
- placeName = countryname;
- }
- /*
- * add standard fields to the index
- */
- doc.add(new TextField("hierarchy", concatIndexEntry, Field.Store.YES));
- doc.add(new TextField("placename", placeName, Field.Store.YES));
- // doc.add(new TextField("countryname", countryname, Field.Store.YES));
- //System.out.println(placeName);
-
- doc.add(new TextField("latitude", lat, Field.Store.YES));
- doc.add(new TextField("longitude", lon, Field.Store.YES));
- doc.add(new StringField("loctype", dsg, Field.Store.YES));
- doc.add(new StringField("admincode", (ccode + "." + admincode).toLowerCase(), Field.Store.YES));
- doc.add(new StringField("countrycode", ccode.toLowerCase(), Field.Store.YES));
- doc.add(new StringField("countycode", "", Field.Store.YES));
- doc.add(new StringField("locid", id, Field.Store.YES));
- placeName = placeName.replace("republic of", "").replace("federative", "");
- if (id.equals("3175395")) {
- System.out.println(placeName);
- }
- doc.add(new StringField("gazsource", "geonames", Field.Store.YES));
-
- w.addDocument(doc);
-
- counter++;
- if (counter % 100000 == 0) {
- w.commit();
- System.out.println(counter + " .........Geonames entries committed to index..............");
- }
-
- }
-
- System.out.println("Completed indexing geonames gaz! index name is: " + type.toString());
- }
-
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.indexing;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import opennlp.addons.geoentitylinker.AdminBoundary;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+
+public class GeonamesProcessor {
+
+ public static void process(File countryCodesLookupFile, File adm1CodesLookupFile, File geonamesGazetteerFile, File outputCountryContextFile, IndexWriter w) throws Exception {
+ Map<String, String> countryCodes = getCountryCodes(countryCodesLookupFile);
+
+ Map<String, AdminBoundary> adm1s = getProvData(adm1CodesLookupFile, countryCodes);
+ // List<AdminBoundary> adm2s = getCountryContextFromFile(new File("C:\\temp\\gazetteers\\geonamesdata\\admin2Codes.txt"));
+ //admin2Codes.txt
+
+ readFile(geonamesGazetteerFile, GazetteerIndexer.GazType.GEONAMES, adm1s, countryCodes, w);
+ //now append to the coutnry context file
+ writeCountryContextFile(outputCountryContextFile, adm1s);
+
+ }
+
+ public GeonamesProcessor() {
+ }
+
+ private static Map<String, AdminBoundary> getProvData(File adm1CodesLookupFile, Map<String, String> ccodes) {
+ System.out.println("Attempting to read geonames province data from: " + adm1CodesLookupFile.getPath());
+
+ Map<String, AdminBoundary> outmap = new HashMap<>();
+ BufferedReader reader;
+ Set<String> nullcodes = new HashSet<>();
+ try {
+
+ reader = new BufferedReader(new FileReader(adm1CodesLookupFile));
+ int i = 0;
+ String line;
+ while ((line = reader.readLine()) != null) {
+
+ // String line = reader.readLine();
+ String[] values = line.split("\t");
+ if (values.length != 4) {
+ throw new IOException("improperly formatted province lookup file");
+ }
+ String ccode = values[0].toLowerCase();
+
+ String[] split = ccode.split("\\.");
+ String pcode = "";
+ if (split.length == 2) {
+ //System.out.println(split);
+ ccode = split[0];
+ pcode = split[1];
+ }
+
+ String pname = values[2];
+
+ if (ccode.matches("[0-9].*")) {
+ String code = ccode;
+ ccode = pcode;
+ pcode = code;
+ }
+
+ String cname = ccodes.get(ccode);
+
+ if (cname == null) {
+ nullcodes.add(ccode);
+ }
+ AdminBoundary data = new AdminBoundary(ccode, cname, pcode, pname, "NO_DATA_FOUND", "NO_DATA_FOUND", cname, pname, "NO_DATA_FOUND");
+ // System.out.println(data);
+ outmap.put(ccode + "." + pcode, data);
+
+ }
+ System.out.println("INFO: there were " + nullcodes.size() + " null prov codes. This is due to inconsistencies in reference data.");
+ reader.close();
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ System.out.println("Successfully read geonames province data from: " + adm1CodesLookupFile.getPath());
+
+ return outmap;
+
+ }
+
+ private static Map<String, String> getCountryCodes(File countryContextFile) {
+ Map<String, String> ccs = new HashMap<>();
+ BufferedReader reader;
+ try {
+
+ reader = new BufferedReader(new FileReader(countryContextFile));
+ int i = 0;
+ String line;
+ boolean start = false;
+ while ((line = reader.readLine()) != null) {
+ if (!line.toLowerCase().startsWith("#iso\t") && !start) {
+
+ continue;
+ } else {
+ start = true;
+ }
+ String[] values = line.split("\t");
+
+ String ccode = values[0].toLowerCase();//this is the 2 digit ISO code
+ String cname = values[4].toLowerCase();
+ if (!ccode.equals("")) {
+ ccs.put(ccode, cname);
+ }
+
+ }
+ reader.close();
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ ccs.put("SS", "South Sudan");
+ ccs.put("CS", "Kosovo");
+ return ccs;
+
+ }
+
+ public static void writeCountryContextFile(File outfile, Map<String, AdminBoundary> adms) {
+ // FileWriter writer = null;
+ try (FileWriter writer = new FileWriter(outfile, true)) {
+ BufferedWriter bw = new BufferedWriter(writer);
+
+ for (String admKey : adms.keySet()) {
+ AdminBoundary adm = adms.get(admKey);
+ if (adm == null) {
+ continue;
+ }
+ String province = adm.getProvinceName();
+ String country = adm.getCountryName();
+
+ String line = adm.getCountryCode() + "\t" + adm.getProvCode() + "\t" + "\t" + country + "\t" + province + "\t" + "\t" + "(" + country + ")" + "\t"
+ + adm.getProvinceName() + "\t" + adm.getCountyName() + "\n";
+ bw.write(line);
+ // System.out.println(line);
+
+ }
+
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ System.out.println("successfully wrote Geonames entries to country oontext file");
+ }
+
+ /**
+ *
+ * @param gazateerInputData the Geonames allCounties.txt file
+ * @param type the types of gaz entry, usgs, geonames, or regions
+ * @param adms the province info
+ * @param countrycodes the country code info
+ * @param w the lucene index writer
+ * @throws Exception
+ */
+ public static void readFile(File gazateerInputData, GazetteerIndexer.GazType type, Map<String, AdminBoundary> adms, Map<String, String> countrycodes, IndexWriter w) throws Exception {
+
+ BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData));
+ String[] boosts = "ADM1 ADM1H ADM2 ADM2H ADM3 ADM3H ADM4 ADM4H ADM5 ADMD ADMDH PCLD PCLH PCLI PCLIX TERR PCLIX PPL PPLA PPLA2 PPLA3 PPLA4 PPLC PPLCH PPLF PPLG PPLH PPLL PPLQ PPLR PPLS PPLX STLMT".split(" ");
+ Map<String, Float> boostMap = new HashMap<>();
+ for (String boost : boosts) {
+ boostMap.put(boost.toLowerCase(), 10f);
+ }
+ String[] fieldStrings = new String[]{
+ "geonameid",
+ "name",
+ "asciiname",
+ "alternatenames",
+ "latitude",
+ "longitude",
+ "feature_class",
+ "feature_code",
+ "country code",
+ "cc2",
+ "admin1_code",
+ "admin2_code",
+ "admin3_code",
+ "admin4_code",
+ "population",
+ "elevation",
+ "dem ",
+ "timezone",
+ "modification_date"};
+
+ List<String> fields = Arrays.asList(fieldStrings);
+ int counter = 0;
+ System.out.println("reading gazetteer data from file...........");
+ String line;
+ while ((line = reader.readLine()) != null) {
+ String[] values = line.split(type.getSeparator());
+
+ Document doc = new Document();
+ String admincode = values[10].toLowerCase();
+ String ccode = values[8].toLowerCase();
+ if (ccode.contains(",")) {
+ String[] codes = ccode.split(",");
+ if (codes.length > 0) {
+ ccode = codes[0];
+ }
+ }
+ AdminBoundary adm = adms.get(ccode + "." + admincode);
+
+ String placeName = values[2];
+ String lat = values[4];
+ String lon = values[5];
+ String dsg = values[7].toLowerCase();
+
+ String id = values[0];
+ String concatIndexEntry;
+ String countryname;
+ if (adm != null) {
+ concatIndexEntry = adm.getCountryName() + ", " + adm.getProvinceName() + ", " + placeName;
+ countryname = adm.getCountryName();
+ } else {
+ //there is no admin info, but we can still use the countrycode to concat the country name
+ String n = countrycodes.get(ccode);
+ countryname = n;
+ if (n != null) {
+ concatIndexEntry = n + ", " + placeName;
+ } else {
+ ///don't want a single token hierarchy entry.
+ concatIndexEntry = "";
+ }
+ }
+ if (ccode == null) {
+ System.out.println("naughty country code");
+ }
+ for (int i = 0; i < fields.size() - 1; i++) {
+ doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
+
+ }
+ if (dsg.equals("pcli")) {
+ System.out.println("placename: " + placeName + " RESET TO: " + countryname);
+ placeName = countryname;
+ }
+ /*
+ * add standard fields to the index
+ */
+ doc.add(new TextField("hierarchy", concatIndexEntry, Field.Store.YES));
+ doc.add(new TextField("placename", placeName, Field.Store.YES));
+ // doc.add(new TextField("countryname", countryname, Field.Store.YES));
+ //System.out.println(placeName);
+
+ doc.add(new TextField("latitude", lat, Field.Store.YES));
+ doc.add(new TextField("longitude", lon, Field.Store.YES));
+ doc.add(new StringField("loctype", dsg, Field.Store.YES));
+ doc.add(new StringField("admincode", (ccode + "." + admincode).toLowerCase(), Field.Store.YES));
+ doc.add(new StringField("countrycode", ccode.toLowerCase(), Field.Store.YES));
+ doc.add(new StringField("countycode", "", Field.Store.YES));
+ doc.add(new StringField("locid", id, Field.Store.YES));
+ placeName = placeName.replace("republic of", "").replace("federative", "");
+ if (id.equals("3175395")) {
+ System.out.println(placeName);
+ }
+ doc.add(new StringField("gazsource", "geonames", Field.Store.YES));
+
+ w.addDocument(doc);
+
+ counter++;
+ if (counter % 100000 == 0) {
+ w.commit();
+ System.out.println(counter + " .........Geonames entries committed to index..............");
+ }
+
+ }
+
+ System.out.println("Completed indexing geonames gaz! index name is: " + type.toString());
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/RegionProcessor.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/RegionProcessor.java
index f1d5d9f..c6b2d0c 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/RegionProcessor.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/RegionProcessor.java
@@ -1,115 +1,115 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.indexing;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexWriter;
-
-public class RegionProcessor {
-
- public static void main(String[] args) {
- RegionProcessor.process(new File("C:\\temp\\gazetteers\\regions.txt"), new File("C:\\temp\\gazetteers\\testRegionContext.txt"), null);
- }
-
- /**
- *
- * @param regionsFile the file that stores Region references. the format of
- * this file is tab delimitted text with index 0 as the name of the region,
- * index 1 as the longitude, and index 2 as the latitude
- * @param outputCountryContextfile this is the country context files shared by
- * all indexing processors
- * @param w
- */
- public static void process(File regionsFile, File outputCountryContextfile, IndexWriter w) {
- try {
- readFile(regionsFile, outputCountryContextfile, w);
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
-
- public static void readFile(File gazateerInputData, File outputCountryContextfile, IndexWriter w) throws Exception {
- List<String> ccfileentries = new ArrayList<>();
- BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData));
- List<String> fields = new ArrayList<>();
- int counter = 0;
- System.out.println("reading gazetteer data from Regions file...........");
- String line;
- while ((line = reader.readLine()) != null) {
-
- String[] values = line.split("\t");
- if (counter == 0) {
-
- } else {
- Document doc = new Document();
- for (int i = 0; i < fields.size() - 1; i++) {
- doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
- }
- String placeName = values[0];
- String lat = values[2];
- String lon = values[1];
- String dsg = "region";
- String id = "rg" + counter;
-
- String hierarchy = placeName;
-
- doc.add(new TextField("hierarchy", hierarchy, Field.Store.YES));
- doc.add(new TextField("placename", placeName, Field.Store.YES));
- doc.add(new StringField("latitude", lat, Field.Store.YES));
- doc.add(new StringField("longitude", lon, Field.Store.YES));
- doc.add(new StringField("loctype", dsg, Field.Store.YES));
- doc.add(new StringField("admincode", "", Field.Store.YES));
- doc.add(new StringField("countrycode", id, Field.Store.YES));
- doc.add(new StringField("countycode", "", Field.Store.YES));
-
- doc.add(new StringField("locid", id, Field.Store.YES));
- doc.add(new StringField("gazsource", "region", Field.Store.YES));
- //countrycontext file format
- // US KY 131 United States Kentucky Leslie
-
- ccfileentries.add(id + "\t" + id + "\t" + id + "\t" + placeName + "\t" + "NO_DATA_FOUND" + "\t" + "NO_DATA_FOUND" + "\t" + "("
- + placeName + ")" + "\t" + "NO_DATA_FOUND" + "\t" + "NO_DATA_FOUND" + "\n");
- if (w != null) {
- w.addDocument(doc);
- }
- }
- counter++;
-
- }
- if (w != null) {
- w.commit();
- }
- FileWriter writer = new FileWriter(outputCountryContextfile, true);
- BufferedWriter bw = new BufferedWriter(writer);
- for (String string : ccfileentries) {
- bw.write(string);
- }
- System.out.println("successfully wrote Region entries to country oontext file");
- bw.close();
- System.out.println("Completed indexing regions!");
- }
-
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.indexing;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+
+public class RegionProcessor {
+
+ public static void main(String[] args) {
+ RegionProcessor.process(new File("C:\\temp\\gazetteers\\regions.txt"), new File("C:\\temp\\gazetteers\\testRegionContext.txt"), null);
+ }
+
+ /**
+ *
+ * @param regionsFile the file that stores Region references. the format of
+ * this file is tab delimitted text with index 0 as the name of the region,
+ * index 1 as the longitude, and index 2 as the latitude
+ * @param outputCountryContextfile this is the country context files shared by
+ * all indexing processors
+ * @param w
+ */
+ public static void process(File regionsFile, File outputCountryContextfile, IndexWriter w) {
+ try {
+ readFile(regionsFile, outputCountryContextfile, w);
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ public static void readFile(File gazateerInputData, File outputCountryContextfile, IndexWriter w) throws Exception {
+ List<String> ccfileentries = new ArrayList<>();
+ BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData));
+ List<String> fields = new ArrayList<>();
+ int counter = 0;
+ System.out.println("reading gazetteer data from Regions file...........");
+ String line;
+ while ((line = reader.readLine()) != null) {
+
+ String[] values = line.split("\t");
+ if (counter == 0) {
+
+ } else {
+ Document doc = new Document();
+ for (int i = 0; i < fields.size() - 1; i++) {
+ doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
+ }
+ String placeName = values[0];
+ String lat = values[2];
+ String lon = values[1];
+ String dsg = "region";
+ String id = "rg" + counter;
+
+ String hierarchy = placeName;
+
+ doc.add(new TextField("hierarchy", hierarchy, Field.Store.YES));
+ doc.add(new TextField("placename", placeName, Field.Store.YES));
+ doc.add(new StringField("latitude", lat, Field.Store.YES));
+ doc.add(new StringField("longitude", lon, Field.Store.YES));
+ doc.add(new StringField("loctype", dsg, Field.Store.YES));
+ doc.add(new StringField("admincode", "", Field.Store.YES));
+ doc.add(new StringField("countrycode", id, Field.Store.YES));
+ doc.add(new StringField("countycode", "", Field.Store.YES));
+
+ doc.add(new StringField("locid", id, Field.Store.YES));
+ doc.add(new StringField("gazsource", "region", Field.Store.YES));
+ //countrycontext file format
+ // US KY 131 United States Kentucky Leslie
+
+ ccfileentries.add(id + "\t" + id + "\t" + id + "\t" + placeName + "\t" + "NO_DATA_FOUND" + "\t" + "NO_DATA_FOUND" + "\t" + "("
+ + placeName + ")" + "\t" + "NO_DATA_FOUND" + "\t" + "NO_DATA_FOUND" + "\n");
+ if (w != null) {
+ w.addDocument(doc);
+ }
+ }
+ counter++;
+
+ }
+ if (w != null) {
+ w.commit();
+ }
+ FileWriter writer = new FileWriter(outputCountryContextfile, true);
+ BufferedWriter bw = new BufferedWriter(writer);
+ for (String string : ccfileentries) {
+ bw.write(string);
+ }
+ System.out.println("successfully wrote Region entries to country oontext file");
+ bw.close();
+ System.out.println("Completed indexing regions!");
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/USGSProcessor.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/USGSProcessor.java
index 588befb..46e6200 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/USGSProcessor.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/USGSProcessor.java
@@ -1,250 +1,250 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.indexing;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import opennlp.addons.geoentitylinker.AdminBoundary;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-
-import org.apache.lucene.index.IndexWriter;
-
-public class USGSProcessor {
-
- public static void main(String[] args) {
- try {
- Map<String, AdminBoundary> provData = getProvData(new File("C:\\temp\\gazetteers\\usgsdata\\GOVT_UNITS_20140601.txt"), GazetteerIndexer.GazType.USGS);
- process(new File("C:\\temp\\gazetteers\\usgsdata\\GOVT_UNITS_20140601.txt"), new File("C:\\temp\\gazetteers\\usgsdata\\NationalFile_20140601.txt"), null, null);
- } catch (Exception ex) {
- Logger.getLogger(USGSProcessor.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
-
- public static void process(File lookupData, File usgsGazDataFile, File outputCountryContextfile, IndexWriter w) throws Exception {
- Map<String, AdminBoundary> provData = getProvData(lookupData, GazetteerIndexer.GazType.USGS);
- readFile(usgsGazDataFile, w, GazetteerIndexer.GazType.USGS, provData);
- writeCountryContextFile(outputCountryContextfile, provData);
- }
-
- public static void readFile(File gazateerInputData, IndexWriter w, GazetteerIndexer.GazType type, Map<String, AdminBoundary> lookupMap) throws Exception {
-
- Map<String, StateCentroid> states = new HashMap<>();
- try (BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData))) {
-
- List<String> fields = new ArrayList<>();
- int counter = 0;
- System.out.println("reading gazetteer data from USGS file...........");
- String line;
- while ((line = reader.readLine()) != null) {
-
- String[] values = line.split(type.getSeparator());
- if (counter == 0) {
- for (String columnName : values) {
- fields.add(columnName.replace("»¿", "").trim());
- }
-
- } else {
- Document doc = new Document();
- for (int i = 0; i < fields.size() - 1; i++) {
- doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
- }
- String placeName = values[1];
- String lat = values[9];
- String lon = values[10];
- String dsg = values[2];
- String id = values[0];
-
- String ccode = values[6];
- String admincode = values[3];
- AdminBoundary get = lookupMap.get(admincode + "." + ccode);
- String countyname = "";
- if (get == null) {
- System.out.println("null...continuing to index" + " ccode: " + ccode + " , admincode: " + admincode + " , placename: " + placeName);
- continue;
-
- }
- String countyCode = get.getCountyCode();
-
- if (!get.getCountyName().equals("NO_DATA_FOUND_VALUE")) {
- countyname = get.getCountyName();
- }
- if (!get.getCountyCode().equals("NO_DATA_FOUND_VALUE")) {
- countyCode = get.getCountyCode();
- }
- String hierarchy = get.getCountryName() + ", " + get.getProvinceName() + ", " + countyname + ", " + placeName;
-
- if (states.containsKey(get.getProvinceName())) {
- StateCentroid entry = states.get(get.getProvinceName());
- entry.count++;
- entry.latSum += Double.parseDouble(lat);
- entry.longSum += Double.parseDouble(lon);
- } else {
- StateCentroid centroid = new StateCentroid();
- centroid.statecode = get.getProvCode();
- centroid.count = 1;
- centroid.latSum = Double.parseDouble(lat);
- centroid.longSum = Double.parseDouble(lon);
- states.put(get.getProvinceName(), centroid);
- }
-
- doc.add(new TextField("hierarchy", hierarchy, Field.Store.YES));
- doc.add(new TextField("placename", placeName, Field.Store.YES));
- doc.add(new TextField("latitude", lat, Field.Store.YES));
- doc.add(new TextField("longitude", lon, Field.Store.YES));
- doc.add(new StringField("loctype", dsg, Field.Store.YES));
- doc.add(new StringField("admincode", (get.getCountryCode() + "." + get.getProvCode()).toLowerCase(), Field.Store.YES));
- doc.add(new StringField("countrycode", get.getCountryCode().toLowerCase(), Field.Store.YES));
- doc.add(new StringField("countycode", (get.getCountryCode() + "." + get.getProvCode() + "." + countyCode).toLowerCase(), Field.Store.YES));
-
- doc.add(new StringField("locid", id, Field.Store.YES));
- doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
- w.addDocument(doc);
- }
- counter++;
- if (counter % 100000 == 0) {
- w.commit();
- System.out.println(counter + " .........USGS entries committed to index..............");
- }
-
- }
- }
-
-
- for (String state : states.keySet()) {
- StateCentroid get = states.get(state);
- Document doc = new Document();
- doc.add(new TextField("hierarchy", "united states, " + state, Field.Store.YES));
- doc.add(new TextField("placename", state, Field.Store.YES));
- //calculate a centroid for all the points that were in the state
- doc.add(new TextField("latitude", (get.latSum / get.count) + "", Field.Store.YES));
- doc.add(new TextField("longitude", (get.longSum / get.count) + "", Field.Store.YES));
- doc.add(new StringField("loctype", "adm1", Field.Store.YES));
- doc.add(new StringField("admincode", get.statecode, Field.Store.YES));
- doc.add(new StringField("countrycode", "us", Field.Store.YES));
- doc.add(new StringField("countycode", "", Field.Store.YES));
-
- doc.add(new StringField("locid", "us_state:" + state, Field.Store.YES));
- doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
- w.addDocument(doc);
-
- // System.out.println(get.statecode + "," + (get.latSum / get.count) + "," + (get.longSum / get.count));
- }
- Document doc = new Document();
- doc.add(new TextField("hierarchy", "united states", Field.Store.YES));
- doc.add(new TextField("placename", "united states", Field.Store.YES));
- //calculate a centroid for all the points that were in the state
- doc.add(new TextField("latitude", 39.0 + "", Field.Store.YES));
- doc.add(new TextField("longitude", -103.0 + "", Field.Store.YES));
- doc.add(new StringField("loctype", "pcli", Field.Store.YES));
- doc.add(new StringField("admincode", "", Field.Store.YES));
- doc.add(new StringField("countrycode", "us", Field.Store.YES));
- doc.add(new StringField("countycode", "", Field.Store.YES));
-
- doc.add(new StringField("locid", "us_centroid" + "unitedstates", Field.Store.YES));
- doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
- //System.out.println("uscentroid," + (sumofLatSums / sumOfCounts) + "," + (sumofLonSums / sumOfCounts));
-
- w.addDocument(doc);
- w.commit();
-
- System.out.println("Completed indexing USGS gaz!");
- }
-
- private static class StateCentroid {
-
- double latSum;
- double longSum;
- String statecode;
- int count;
- }
-
- private static Map<String, AdminBoundary> getProvData(File govUnitsFile, GazetteerIndexer.GazType type) {
- System.out.println("Attempting to read USGS province (State) data from: " + govUnitsFile.getPath());
- Map<String, AdminBoundary> outmap = new HashMap<>();
-
- try (BufferedReader reader = new BufferedReader(new FileReader(govUnitsFile))) {
- int i = 0;
- String line;
- String[] fields = null;
- while ((line = reader.readLine()) != null) {
-
- String[] values = line.split(type.getSeparator());
- if (i == 0) {
- fields = values;
- i++;
- continue;
- }
- i++;
- // System.out.println(i);
- String countyCode = values[2];
- String countyName = values[3];
- String stateCode = values[5];
- String stateName = values[6];
- String countryCode = values[7];
- String countryName = values[8];
- AdminBoundary adminBoundary = new AdminBoundary(countryCode, countryName, stateCode, stateName, countyCode,
- countyName, null, null, null);
- outmap.put(stateCode + "." + countyCode, adminBoundary);
- // System.out.println(adminBoundary);
-
- }
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- System.out.println("Successfully read USGS province (State) data from: " + govUnitsFile.getPath());
-
- return outmap;
-
- }
-
- public static void writeCountryContextFile(File outfile, Map<String, AdminBoundary> adms) {
-
- try (BufferedWriter bw = new BufferedWriter(new FileWriter(outfile, true))) {
- for (String admkey : adms.keySet()) {
- AdminBoundary adm = adms.get(admkey);
- if (adm == null) {
- continue;
- }
- String province = adm.getProvinceName();
- String country = adm.getCountryName();
- /*
- * this is the standard format of the country context file... Geonames
- * data will have an empty string for the county
- */
- String line = adm.getCountryCode() + "\t" + adm.getProvCode() + "\t" + adm.getCountyCode() + "\t" + country + "\t" + province + "\t" + adm.getCountyName() + "\t"
- + "(U\\.S\\.[ $]|U\\.S\\.A\\.[ $]|United States|the US[ $]|a us[ $])" + "\t" + adm.getProvinceName() + "\t" + adm.getCountyName() + "\n";
- bw.write(line);
- /// System.out.println(line);
- }
- } catch (IOException ex) {
- Logger.getLogger(GeonamesProcessor.class.getName()).log(Level.SEVERE, null, ex);
- }
- System.out.println("successfully wrote USGS entries to country context file");
- }
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.indexing;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import opennlp.addons.geoentitylinker.AdminBoundary;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+
+import org.apache.lucene.index.IndexWriter;
+
+public class USGSProcessor {
+
+ public static void main(String[] args) {
+ try {
+ Map<String, AdminBoundary> provData = getProvData(new File("C:\\temp\\gazetteers\\usgsdata\\GOVT_UNITS_20140601.txt"), GazetteerIndexer.GazType.USGS);
+ process(new File("C:\\temp\\gazetteers\\usgsdata\\GOVT_UNITS_20140601.txt"), new File("C:\\temp\\gazetteers\\usgsdata\\NationalFile_20140601.txt"), null, null);
+ } catch (Exception ex) {
+ Logger.getLogger(USGSProcessor.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ }
+
+ public static void process(File lookupData, File usgsGazDataFile, File outputCountryContextfile, IndexWriter w) throws Exception {
+ Map<String, AdminBoundary> provData = getProvData(lookupData, GazetteerIndexer.GazType.USGS);
+ readFile(usgsGazDataFile, w, GazetteerIndexer.GazType.USGS, provData);
+ writeCountryContextFile(outputCountryContextfile, provData);
+ }
+
+ public static void readFile(File gazateerInputData, IndexWriter w, GazetteerIndexer.GazType type, Map<String, AdminBoundary> lookupMap) throws Exception {
+
+ Map<String, StateCentroid> states = new HashMap<>();
+ try (BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData))) {
+
+ List<String> fields = new ArrayList<>();
+ int counter = 0;
+ System.out.println("reading gazetteer data from USGS file...........");
+ String line;
+ while ((line = reader.readLine()) != null) {
+
+ String[] values = line.split(type.getSeparator());
+ if (counter == 0) {
+ for (String columnName : values) {
+ fields.add(columnName.replace("»¿", "").trim());
+ }
+
+ } else {
+ Document doc = new Document();
+ for (int i = 0; i < fields.size() - 1; i++) {
+ doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
+ }
+ String placeName = values[1];
+ String lat = values[9];
+ String lon = values[10];
+ String dsg = values[2];
+ String id = values[0];
+
+ String ccode = values[6];
+ String admincode = values[3];
+ AdminBoundary get = lookupMap.get(admincode + "." + ccode);
+ String countyname = "";
+ if (get == null) {
+ System.out.println("null...continuing to index" + " ccode: " + ccode + " , admincode: " + admincode + " , placename: " + placeName);
+ continue;
+
+ }
+ String countyCode = get.getCountyCode();
+
+ if (!get.getCountyName().equals("NO_DATA_FOUND_VALUE")) {
+ countyname = get.getCountyName();
+ }
+ if (!get.getCountyCode().equals("NO_DATA_FOUND_VALUE")) {
+ countyCode = get.getCountyCode();
+ }
+ String hierarchy = get.getCountryName() + ", " + get.getProvinceName() + ", " + countyname + ", " + placeName;
+
+ if (states.containsKey(get.getProvinceName())) {
+ StateCentroid entry = states.get(get.getProvinceName());
+ entry.count++;
+ entry.latSum += Double.parseDouble(lat);
+ entry.longSum += Double.parseDouble(lon);
+ } else {
+ StateCentroid centroid = new StateCentroid();
+ centroid.statecode = get.getProvCode();
+ centroid.count = 1;
+ centroid.latSum = Double.parseDouble(lat);
+ centroid.longSum = Double.parseDouble(lon);
+ states.put(get.getProvinceName(), centroid);
+ }
+
+ doc.add(new TextField("hierarchy", hierarchy, Field.Store.YES));
+ doc.add(new TextField("placename", placeName, Field.Store.YES));
+ doc.add(new TextField("latitude", lat, Field.Store.YES));
+ doc.add(new TextField("longitude", lon, Field.Store.YES));
+ doc.add(new StringField("loctype", dsg, Field.Store.YES));
+ doc.add(new StringField("admincode", (get.getCountryCode() + "." + get.getProvCode()).toLowerCase(), Field.Store.YES));
+ doc.add(new StringField("countrycode", get.getCountryCode().toLowerCase(), Field.Store.YES));
+ doc.add(new StringField("countycode", (get.getCountryCode() + "." + get.getProvCode() + "." + countyCode).toLowerCase(), Field.Store.YES));
+
+ doc.add(new StringField("locid", id, Field.Store.YES));
+ doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
+ w.addDocument(doc);
+ }
+ counter++;
+ if (counter % 100000 == 0) {
+ w.commit();
+ System.out.println(counter + " .........USGS entries committed to index..............");
+ }
+
+ }
+ }
+
+
+ for (String state : states.keySet()) {
+ StateCentroid get = states.get(state);
+ Document doc = new Document();
+ doc.add(new TextField("hierarchy", "united states, " + state, Field.Store.YES));
+ doc.add(new TextField("placename", state, Field.Store.YES));
+ //calculate a centroid for all the points that were in the state
+ doc.add(new TextField("latitude", (get.latSum / get.count) + "", Field.Store.YES));
+ doc.add(new TextField("longitude", (get.longSum / get.count) + "", Field.Store.YES));
+ doc.add(new StringField("loctype", "adm1", Field.Store.YES));
+ doc.add(new StringField("admincode", get.statecode, Field.Store.YES));
+ doc.add(new StringField("countrycode", "us", Field.Store.YES));
+ doc.add(new StringField("countycode", "", Field.Store.YES));
+
+ doc.add(new StringField("locid", "us_state:" + state, Field.Store.YES));
+ doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
+ w.addDocument(doc);
+
+ // System.out.println(get.statecode + "," + (get.latSum / get.count) + "," + (get.longSum / get.count));
+ }
+ Document doc = new Document();
+ doc.add(new TextField("hierarchy", "united states", Field.Store.YES));
+ doc.add(new TextField("placename", "united states", Field.Store.YES));
+ //calculate a centroid for all the points that were in the state
+ doc.add(new TextField("latitude", 39.0 + "", Field.Store.YES));
+ doc.add(new TextField("longitude", -103.0 + "", Field.Store.YES));
+ doc.add(new StringField("loctype", "pcli", Field.Store.YES));
+ doc.add(new StringField("admincode", "", Field.Store.YES));
+ doc.add(new StringField("countrycode", "us", Field.Store.YES));
+ doc.add(new StringField("countycode", "", Field.Store.YES));
+
+ doc.add(new StringField("locid", "us_centroid" + "unitedstates", Field.Store.YES));
+ doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
+ //System.out.println("uscentroid," + (sumofLatSums / sumOfCounts) + "," + (sumofLonSums / sumOfCounts));
+
+ w.addDocument(doc);
+ w.commit();
+
+ System.out.println("Completed indexing USGS gaz!");
+ }
+
+ private static class StateCentroid {
+
+ double latSum;
+ double longSum;
+ String statecode;
+ int count;
+ }
+
+ private static Map<String, AdminBoundary> getProvData(File govUnitsFile, GazetteerIndexer.GazType type) {
+ System.out.println("Attempting to read USGS province (State) data from: " + govUnitsFile.getPath());
+ Map<String, AdminBoundary> outmap = new HashMap<>();
+
+ try (BufferedReader reader = new BufferedReader(new FileReader(govUnitsFile))) {
+ int i = 0;
+ String line;
+ String[] fields = null;
+ while ((line = reader.readLine()) != null) {
+
+ String[] values = line.split(type.getSeparator());
+ if (i == 0) {
+ fields = values;
+ i++;
+ continue;
+ }
+ i++;
+ // System.out.println(i);
+ String countyCode = values[2];
+ String countyName = values[3];
+ String stateCode = values[5];
+ String stateName = values[6];
+ String countryCode = values[7];
+ String countryName = values[8];
+ AdminBoundary adminBoundary = new AdminBoundary(countryCode, countryName, stateCode, stateName, countyCode,
+ countyName, null, null, null);
+ outmap.put(stateCode + "." + countyCode, adminBoundary);
+ // System.out.println(adminBoundary);
+
+ }
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ System.out.println("Successfully read USGS province (State) data from: " + govUnitsFile.getPath());
+
+ return outmap;
+
+ }
+
+ public static void writeCountryContextFile(File outfile, Map<String, AdminBoundary> adms) {
+
+ try (BufferedWriter bw = new BufferedWriter(new FileWriter(outfile, true))) {
+ for (String admkey : adms.keySet()) {
+ AdminBoundary adm = adms.get(admkey);
+ if (adm == null) {
+ continue;
+ }
+ String province = adm.getProvinceName();
+ String country = adm.getCountryName();
+ /*
+ * this is the standard format of the country context file... Geonames
+ * data will have an empty string for the county
+ */
+ String line = adm.getCountryCode() + "\t" + adm.getProvCode() + "\t" + adm.getCountyCode() + "\t" + country + "\t" + province + "\t" + adm.getCountyName() + "\t"
+ + "(U\\.S\\.[ $]|U\\.S\\.A\\.[ $]|United States|the US[ $]|a us[ $])" + "\t" + adm.getProvinceName() + "\t" + adm.getCountyName() + "\n";
+ bw.write(line);
+ /// System.out.println(line);
+ }
+ } catch (IOException ex) {
+ Logger.getLogger(GeonamesProcessor.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ System.out.println("successfully wrote USGS entries to country context file");
+ }
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/CountryProximityScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/CountryProximityScorer.java
index df69fb7..b706c5f 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/CountryProximityScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/CountryProximityScorer.java
@@ -1,277 +1,277 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.regex.Pattern;
-import opennlp.addons.geoentitylinker.AdminBoundaryContext;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.BaseLink;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-
-/**
- * Scores toponyms based on their proximity to a country mention. Based on the
- * heuristic that toponymn mentions are more likely close to their parent
- * country mentions. For instance, if the toponym Berlin is mentioned near an
- * indicator of Germany, it is more likely to be Berlin Germany than Berlin
- * Connecticut (if Connecticut is mentioned further down in the article).
- */
-public class CountryProximityScorer implements LinkedEntityScorer<AdminBoundaryContext> {
-
- private Map<String, Set<String>> nameCodesMap;
- String dominantCode = "";
- private Map<String, String> regexMap = new HashMap<>();
-
- @Override
- public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
-
- regexMap = additionalContext.getCountryRegexMap();
- score(linkedSpans, additionalContext.getCountryMentions(), additionalContext.getNameCodesMap(), docText, sentenceSpans, 1000);
-
- }
-
- /**
- * Assigns a score to each BaseLink in each linkedSpan's set of N best
- * matches. Currently, the scoring indicates the probability that the toponym
- * is correct based on the country context in the document
- *
- * @param linkedData the linked spans, holds the Namefinder results, and the
- * list of BaseLink for each
- * @param countryHits all the country mentions in the document
- * @param nameCodesMap maps a country indicator name to a country code. Used
- * to determine if the namefinder found the same exact toponym the country
- * context did. If so the score is boosted due to the high probability that
- * the NameFinder actually "rediscovered" a country
- * @param docText the full text of the document...not used in this default
- * implementation
- * @param sentences the sentences that correspond to the doc text.
- * @param maxAllowedDist a constant that is used to determine which country
- * mentions, based on proximity within the text, should be used to score the
- * Named Entity.
- * @return
- */
- public List<LinkedSpan> score(List<LinkedSpan> linkedData, Map<String, Set<Integer>> countryHits, Map<String, Set<String>> nameCodesMap, String docText, Span[] sentences, Integer maxAllowedDist) {
- this.nameCodesMap = nameCodesMap;
- setDominantCode(countryHits);
- for (LinkedSpan<BaseLink> linkedspan : linkedData) {
-
- linkedspan = simpleProximityAnalysis(sentences, countryHits, linkedspan, maxAllowedDist);
- }
- return linkedData;
- }
-
- /**
- * sets class level variable to a code based on the number of mentions
- *
- * @param countryHits
- */
- private void setDominantCode(Map<String, Set<Integer>> countryHits) {
- int hits = -1;
- for (String code : countryHits.keySet()) {
- if (countryHits.get(code).size() > hits) {
- hits = countryHits.get(code).size();
- dominantCode = code;
- }
- }
- }
-
- /**
- * Generates distances from each country mention to the span's location in the
- * doc text. Ultimately an attempt to ensure that ambiguously named toponyms
- * are resolved to the correct country and coordinate.
- *
- * @param sentences
- * @param countryHits
- * @param span
- * @return
- */
- private LinkedSpan<BaseLink> simpleProximityAnalysis(Span[] sentences, Map<String, Set<Integer>> countryHits, LinkedSpan<BaseLink> span, Integer maxAllowedDistance) {
- Double score = 0.0;
- /*
- * get the index of the actual span, beginning of sentence //should generate
- * tokens from sentence and create a char offset... //could have large
- * sentences due to poor sentence detection or wonky doc text
- */
- int sentenceIdx = span.getSentenceid();
- int sentIndexInDoc = sentences[sentenceIdx].getStart();
- /*
- * create a map of all the span's proximal country mentions in the document
- * Map< countrycode, set of <distances from this NamedEntity>>
- */
- Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<>();
- //map = Map<countrycode, Set <of distances this span is from all the mentions of the code>>
- for (String cCode : countryHits.keySet()) {
- // iterate over all the regex start values and calculate an offset
- for (Integer cHit : countryHits.get(cCode)) {
- Integer absDist = Math.abs(sentIndexInDoc - cHit);
- //only include near mentions based on a heuristic
- //TODO make this a property
- // if (absDist < maxAllowedDistance) {
- if (distancesFromCodeMap.containsKey(cCode)) {
- distancesFromCodeMap.get(cCode).add(absDist);
- } else {
- HashSet<Integer> newset = new HashSet<>();
- newset.add(absDist);
- distancesFromCodeMap.put(cCode, newset);
- }
- }
- }
- //we now know how far this named entity is from every country mention in the document
-
- /*
- * the gaz matches that have a country code that have mentions in the doc
- * that are closest to the Named Entity should return the best score.
- * Analyzemap generates a likelihood score that the toponym from the gaz is
- * referring to one of the countries, i.e, Map<countrycode, prob that this
- * span is referring to the toponym form this code key>
- */
- Map<String, Double> scoreMap = analyzeMap(distancesFromCodeMap, sentences, span);
- for (BaseLink link : span.getLinkedEntries()) {
- //getItemParentId is the country code
- String spanCountryCode = link.getItemParentID();
- if (scoreMap.containsKey(spanCountryCode)) {
-
- score = scoreMap.get(spanCountryCode);
- ///does the name extracted match a country name?
- if (nameCodesMap.containsKey(link.getItemName().toLowerCase()) || regexMatch(link.getItemName(), link.getItemParentID())) {
- //if so, is it the correct country code for that name?
- if (nameCodesMap.get(link.getItemName().toLowerCase()).contains(link.getItemParentID())) {
- //boost the score because it is likely that this is the location in the text, so add 50% to the score or set to 1
- score = (score + .75) > 1.0 ? 1d : (score + .75);
-
- if (link.getItemParentID().equals(dominantCode)) {
- score = (score + .25) > 1.0 ? 1d : (score + .25);
- }
- }
- }
- }
-
- link.getScoreMap().put("countrycontext", score);
- }
- return span;
- }
-
- /**
- * takes a map of distances from the toponym to each country mention and
- * generates a map of scores for each country code. The map is then correlated
- * to the code of the BaseLink parentid for retrieval. Then the score is added
- * to the overall list.
- *
- * @param distanceMap
- * @param sentences
- * @param span
- * @return
- */
- private Map<String, Double> analyzeMap(Map<String, Set<Integer>> distanceMap, Span[] sentences, LinkedSpan<BaseLink> span) {
-
- Map<String, Double> scoreMap = new HashMap<>();
- if (distanceMap.isEmpty()) {
- return scoreMap;
- }
- TreeSet<Integer> all = new TreeSet<>();
- for (String key : distanceMap.keySet()) {
- all.addAll(distanceMap.get(key));
- }
-
- // get min max for normalization, this could be more efficient
- int min = all.first();
- int max = all.last();
- if (min == max) {
- min = 0;
- }
-
- for (String key : distanceMap.keySet()) {
- TreeSet<Double> normalizedDistances = new TreeSet<>();
- for (Integer i : distanceMap.get(key)) {
- Double norm = normalize(i, min, max);
- //reverse the normed distance so low numbers (closer) are better
- //this could be improved with a "decaying " function using an imcreaseing negative exponent
- Double reverse = Math.abs(norm - 1);
- normalizedDistances.add(reverse);
- }
-
- List<Double> doubles = new ArrayList<>(normalizedDistances);
- scoreMap.put(key, slidingDistanceAverage(doubles));
- }
- return scoreMap;
- }
-
- private boolean regexMatch(String placeName, String countryCode) {
- if (regexMap.containsKey(countryCode)) {
- String regexForCountry = regexMap.get(countryCode);
-
- Pattern p = Pattern.compile(regexForCountry,Pattern.DOTALL|Pattern.CASE_INSENSITIVE);
- return p.matcher(placeName.trim()).matches();
- }
- return false;
- }
-
- /**
- * this method is an attempt to make closer clusters of mentions group
- * together to smooth out the average, so one distant outlier does not kill
- * the score for an obviously good hit. More elegant solution is possible
- * using Math.pow, and making the score decay with distance by using an
- * increasing negative exponent (I think)
- *
- * @param normDis the normalized and sorted set of distances as a list
- * @return
- */
- private Double slidingDistanceAverage(List<Double> normDis) {
- List<Double> windowOfAverages = new ArrayList<>();
-
- if (normDis.size() < 3) {
- windowOfAverages.addAll(normDis);
- } else {
-
- for (int i = 0; i < normDis.size() - 1; i++) {
- double a = normDis.get(i);
- double b = normDis.get(i + 1);
- windowOfAverages.add((a + b) / 2);
-
- }
- }
- double sum = 0d;
- for (double d : windowOfAverages) {
- sum += d;
- }
- double result = sum / windowOfAverages.size();
- //TODO: ++ prob when large amounts of mentions for a code
- //System.out.println("avg of window:" + result);
- return result;
- }
-
- /**
- * transposes a value within one range to a relative value in a different
- * range. Used to normalize distances in this class.
- *
- * @param valueToNormalize the value to place within the new range
- * @param minimum the min of the set to be transposed
- * @param maximum the max of the set to be transposed
- * @return
- */
- private Double normalize(int valueToNormalize, int minimum, int maximum) {
- Double d = (double) ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
- d = d == null ? 0d : d;
- return d;
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Pattern;
+import opennlp.addons.geoentitylinker.AdminBoundaryContext;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.BaseLink;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ * Scores toponyms based on their proximity to a country mention. Based on the
+ * heuristic that toponymn mentions are more likely close to their parent
+ * country mentions. For instance, if the toponym Berlin is mentioned near an
+ * indicator of Germany, it is more likely to be Berlin Germany than Berlin
+ * Connecticut (if Connecticut is mentioned further down in the article).
+ */
+public class CountryProximityScorer implements LinkedEntityScorer<AdminBoundaryContext> {
+
+ private Map<String, Set<String>> nameCodesMap;
+ String dominantCode = "";
+ private Map<String, String> regexMap = new HashMap<>();
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+
+ regexMap = additionalContext.getCountryRegexMap();
+ score(linkedSpans, additionalContext.getCountryMentions(), additionalContext.getNameCodesMap(), docText, sentenceSpans, 1000);
+
+ }
+
+ /**
+ * Assigns a score to each BaseLink in each linkedSpan's set of N best
+ * matches. Currently, the scoring indicates the probability that the toponym
+ * is correct based on the country context in the document
+ *
+ * @param linkedData the linked spans, holds the Namefinder results, and the
+ * list of BaseLink for each
+ * @param countryHits all the country mentions in the document
+ * @param nameCodesMap maps a country indicator name to a country code. Used
+ * to determine if the namefinder found the same exact toponym the country
+ * context did. If so the score is boosted due to the high probability that
+ * the NameFinder actually "rediscovered" a country
+ * @param docText the full text of the document...not used in this default
+ * implementation
+ * @param sentences the sentences that correspond to the doc text.
+ * @param maxAllowedDist a constant that is used to determine which country
+ * mentions, based on proximity within the text, should be used to score the
+ * Named Entity.
+ * @return
+ */
+ public List<LinkedSpan> score(List<LinkedSpan> linkedData, Map<String, Set<Integer>> countryHits, Map<String, Set<String>> nameCodesMap, String docText, Span[] sentences, Integer maxAllowedDist) {
+ this.nameCodesMap = nameCodesMap;
+ setDominantCode(countryHits);
+ for (LinkedSpan<BaseLink> linkedspan : linkedData) {
+
+ linkedspan = simpleProximityAnalysis(sentences, countryHits, linkedspan, maxAllowedDist);
+ }
+ return linkedData;
+ }
+
+ /**
+ * sets class level variable to a code based on the number of mentions
+ *
+ * @param countryHits
+ */
+ private void setDominantCode(Map<String, Set<Integer>> countryHits) {
+ int hits = -1;
+ for (String code : countryHits.keySet()) {
+ if (countryHits.get(code).size() > hits) {
+ hits = countryHits.get(code).size();
+ dominantCode = code;
+ }
+ }
+ }
+
+ /**
+ * Generates distances from each country mention to the span's location in the
+ * doc text. Ultimately an attempt to ensure that ambiguously named toponyms
+ * are resolved to the correct country and coordinate.
+ *
+ * @param sentences
+ * @param countryHits
+ * @param span
+ * @return
+ */
+ private LinkedSpan<BaseLink> simpleProximityAnalysis(Span[] sentences, Map<String, Set<Integer>> countryHits, LinkedSpan<BaseLink> span, Integer maxAllowedDistance) {
+ Double score = 0.0;
+ /*
+ * get the index of the actual span, beginning of sentence //should generate
+ * tokens from sentence and create a char offset... //could have large
+ * sentences due to poor sentence detection or wonky doc text
+ */
+ int sentenceIdx = span.getSentenceid();
+ int sentIndexInDoc = sentences[sentenceIdx].getStart();
+ /*
+ * create a map of all the span's proximal country mentions in the document
+ * Map< countrycode, set of <distances from this NamedEntity>>
+ */
+ Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<>();
+ //map = Map<countrycode, Set <of distances this span is from all the mentions of the code>>
+ for (String cCode : countryHits.keySet()) {
+ // iterate over all the regex start values and calculate an offset
+ for (Integer cHit : countryHits.get(cCode)) {
+ Integer absDist = Math.abs(sentIndexInDoc - cHit);
+ //only include near mentions based on a heuristic
+ //TODO make this a property
+ // if (absDist < maxAllowedDistance) {
+ if (distancesFromCodeMap.containsKey(cCode)) {
+ distancesFromCodeMap.get(cCode).add(absDist);
+ } else {
+ HashSet<Integer> newset = new HashSet<>();
+ newset.add(absDist);
+ distancesFromCodeMap.put(cCode, newset);
+ }
+ }
+ }
+ //we now know how far this named entity is from every country mention in the document
+
+ /*
+ * the gaz matches that have a country code that have mentions in the doc
+ * that are closest to the Named Entity should return the best score.
+ * Analyzemap generates a likelihood score that the toponym from the gaz is
+ * referring to one of the countries, i.e, Map<countrycode, prob that this
+ * span is referring to the toponym form this code key>
+ */
+ Map<String, Double> scoreMap = analyzeMap(distancesFromCodeMap, sentences, span);
+ for (BaseLink link : span.getLinkedEntries()) {
+ //getItemParentId is the country code
+ String spanCountryCode = link.getItemParentID();
+ if (scoreMap.containsKey(spanCountryCode)) {
+
+ score = scoreMap.get(spanCountryCode);
+ ///does the name extracted match a country name?
+ if (nameCodesMap.containsKey(link.getItemName().toLowerCase()) || regexMatch(link.getItemName(), link.getItemParentID())) {
+ //if so, is it the correct country code for that name?
+ if (nameCodesMap.get(link.getItemName().toLowerCase()).contains(link.getItemParentID())) {
+ //boost the score because it is likely that this is the location in the text, so add 50% to the score or set to 1
+ score = (score + .75) > 1.0 ? 1d : (score + .75);
+
+ if (link.getItemParentID().equals(dominantCode)) {
+ score = (score + .25) > 1.0 ? 1d : (score + .25);
+ }
+ }
+ }
+ }
+
+ link.getScoreMap().put("countrycontext", score);
+ }
+ return span;
+ }
+
+ /**
+ * takes a map of distances from the toponym to each country mention and
+ * generates a map of scores for each country code. The map is then correlated
+ * to the code of the BaseLink parentid for retrieval. Then the score is added
+ * to the overall list.
+ *
+ * @param distanceMap
+ * @param sentences
+ * @param span
+ * @return
+ */
+ private Map<String, Double> analyzeMap(Map<String, Set<Integer>> distanceMap, Span[] sentences, LinkedSpan<BaseLink> span) {
+
+ Map<String, Double> scoreMap = new HashMap<>();
+ if (distanceMap.isEmpty()) {
+ return scoreMap;
+ }
+ TreeSet<Integer> all = new TreeSet<>();
+ for (String key : distanceMap.keySet()) {
+ all.addAll(distanceMap.get(key));
+ }
+
+ // get min max for normalization, this could be more efficient
+ int min = all.first();
+ int max = all.last();
+ if (min == max) {
+ min = 0;
+ }
+
+ for (String key : distanceMap.keySet()) {
+ TreeSet<Double> normalizedDistances = new TreeSet<>();
+ for (Integer i : distanceMap.get(key)) {
+ Double norm = normalize(i, min, max);
+ //reverse the normed distance so low numbers (closer) are better
+ //this could be improved with a "decaying " function using an imcreaseing negative exponent
+ Double reverse = Math.abs(norm - 1);
+ normalizedDistances.add(reverse);
+ }
+
+ List<Double> doubles = new ArrayList<>(normalizedDistances);
+ scoreMap.put(key, slidingDistanceAverage(doubles));
+ }
+ return scoreMap;
+ }
+
+ private boolean regexMatch(String placeName, String countryCode) {
+ if (regexMap.containsKey(countryCode)) {
+ String regexForCountry = regexMap.get(countryCode);
+
+ Pattern p = Pattern.compile(regexForCountry,Pattern.DOTALL|Pattern.CASE_INSENSITIVE);
+ return p.matcher(placeName.trim()).matches();
+ }
+ return false;
+ }
+
+ /**
+ * this method is an attempt to make closer clusters of mentions group
+ * together to smooth out the average, so one distant outlier does not kill
+ * the score for an obviously good hit. More elegant solution is possible
+ * using Math.pow, and making the score decay with distance by using an
+ * increasing negative exponent (I think)
+ *
+ * @param normDis the normalized and sorted set of distances as a list
+ * @return
+ */
+ private Double slidingDistanceAverage(List<Double> normDis) {
+ List<Double> windowOfAverages = new ArrayList<>();
+
+ if (normDis.size() < 3) {
+ windowOfAverages.addAll(normDis);
+ } else {
+
+ for (int i = 0; i < normDis.size() - 1; i++) {
+ double a = normDis.get(i);
+ double b = normDis.get(i + 1);
+ windowOfAverages.add((a + b) / 2);
+
+ }
+ }
+ double sum = 0d;
+ for (double d : windowOfAverages) {
+ sum += d;
+ }
+ double result = sum / windowOfAverages.size();
+ //TODO: ++ prob when large amounts of mentions for a code
+ //System.out.println("avg of window:" + result);
+ return result;
+ }
+
+ /**
+ * transposes a value within one range to a relative value in a different
+ * range. Used to normalize distances in this class.
+ *
+ * @param valueToNormalize the value to place within the new range
+ * @param minimum the min of the set to be transposed
+ * @param maximum the max of the set to be transposed
+ * @return
+ */
+ private Double normalize(int valueToNormalize, int minimum, int maximum) {
+ Double d = (double) ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
+ d = d == null ? 0d : d;
+ return d;
+ }
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/FuzzyStringMatchScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/FuzzyStringMatchScorer.java
index 64bd788..5367122 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/FuzzyStringMatchScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/FuzzyStringMatchScorer.java
@@ -1,124 +1,124 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import opennlp.addons.geoentitylinker.AdminBoundaryContext;
-import opennlp.addons.geoentitylinker.GazetteerEntry;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.BaseLink;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-
-/**
- *
- * Generates scores based on string comparisons levenstein and dice
- */
-public class FuzzyStringMatchScorer implements LinkedEntityScorer<AdminBoundaryContext> {
-
- @Override
- public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
-
- for (LinkedSpan<BaseLink> linkedSpan : linkedSpans) {
- for (BaseLink link : linkedSpan.getLinkedEntries()) {
- if (link instanceof GazetteerEntry) {
- GazetteerEntry entry = (GazetteerEntry) link;
- String hierarchy = entry.getHierarchy();
- if (hierarchy != null) {
- Double dice = getDiceCoefficient(linkedSpan.getSearchTerm().toLowerCase(), hierarchy.toLowerCase(), 2);
- link.getScoreMap().put("hierarchydicecoef", dice);
- Double ld = (double) getLevenshteinDistance(linkedSpan.getSearchTerm().toLowerCase(), hierarchy.toLowerCase());
- link.getScoreMap().put("hierarchylevenshtein", ld);
- }
- String placename = entry.getItemName().toLowerCase();
- if (placename != null) {
- Double dice = getDiceCoefficient(linkedSpan.getSearchTerm().toLowerCase(), placename, 2);
- link.getScoreMap().put("placenamedicecoef", dice);
-
- }
- }
- }
- }
-
- }
-
- /**
- * Generates a score based on an overlap of nGrams between two strings using
- * the DiceCoefficient technique.
- *
- * @param s1 first string
- * @param s2 second string
- * @param nGrams number of chars in each gram
- * @return
- */
- public double getDiceCoefficient(String s1, String s2, int nGrams) {
- if (s1.isEmpty() || s2.isEmpty()) {
- return 0d;
- }
- List<String> s1Grams = new ArrayList<>();
- List<String> s2Grams = new ArrayList<>();
- String[] split1 = s1.split("[ ,]");
- for (String token : split1) {
- if (token.trim().equals("")) {
- continue;
- }
- s1Grams.add(token);
- }
- String[] split2 = s2.split("[ ,]");
- for (String token : split2) {
- if (token.trim().equals("")) {
- continue;
- }
- s2Grams.add(token);
- }
-
- Set<String> overlap = new HashSet<>(s1Grams);
- overlap.retainAll(s2Grams);
- double totcombigrams = overlap.size();
-
- return (2 * totcombigrams) / (s1Grams.size() + s2Grams.size());
- }
-
- private int minimum(int a, int b, int c) {
- return Math.min(Math.min(a, b), c);
- }
-
- public int getLevenshteinDistance(CharSequence str1,
- CharSequence str2) {
- int[][] distance = new int[str1.length() + 1][str2.length() + 1];
-
- for (int i = 0; i <= str1.length(); i++) {
- distance[i][0] = i;
- }
- for (int j = 1; j <= str2.length(); j++) {
- distance[0][j] = j;
- }
-
- for (int i = 1; i <= str1.length(); i++) {
- for (int j = 1; j <= str2.length(); j++) {
- distance[i][j] = minimum(
- distance[i - 1][j] + 1,
- distance[i][j - 1] + 1,
- distance[i - 1][j - 1] + ((str1.charAt(i - 1) == str2.charAt(j - 1)) ? 0 : 1));
- }
- }
-
- return distance[str1.length()][str2.length()];
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import opennlp.addons.geoentitylinker.AdminBoundaryContext;
+import opennlp.addons.geoentitylinker.GazetteerEntry;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.BaseLink;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ *
+ * Generates scores based on string comparisons levenstein and dice
+ */
+public class FuzzyStringMatchScorer implements LinkedEntityScorer<AdminBoundaryContext> {
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+
+ for (LinkedSpan<BaseLink> linkedSpan : linkedSpans) {
+ for (BaseLink link : linkedSpan.getLinkedEntries()) {
+ if (link instanceof GazetteerEntry) {
+ GazetteerEntry entry = (GazetteerEntry) link;
+ String hierarchy = entry.getHierarchy();
+ if (hierarchy != null) {
+ Double dice = getDiceCoefficient(linkedSpan.getSearchTerm().toLowerCase(), hierarchy.toLowerCase(), 2);
+ link.getScoreMap().put("hierarchydicecoef", dice);
+ Double ld = (double) getLevenshteinDistance(linkedSpan.getSearchTerm().toLowerCase(), hierarchy.toLowerCase());
+ link.getScoreMap().put("hierarchylevenshtein", ld);
+ }
+ String placename = entry.getItemName().toLowerCase();
+ if (placename != null) {
+ Double dice = getDiceCoefficient(linkedSpan.getSearchTerm().toLowerCase(), placename, 2);
+ link.getScoreMap().put("placenamedicecoef", dice);
+
+ }
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Generates a score based on an overlap of nGrams between two strings using
+ * the DiceCoefficient technique.
+ *
+ * @param s1 first string
+ * @param s2 second string
+ * @param nGrams number of chars in each gram
+ * @return
+ */
+ public double getDiceCoefficient(String s1, String s2, int nGrams) {
+ if (s1.isEmpty() || s2.isEmpty()) {
+ return 0d;
+ }
+ List<String> s1Grams = new ArrayList<>();
+ List<String> s2Grams = new ArrayList<>();
+ String[] split1 = s1.split("[ ,]");
+ for (String token : split1) {
+ if (token.trim().equals("")) {
+ continue;
+ }
+ s1Grams.add(token);
+ }
+ String[] split2 = s2.split("[ ,]");
+ for (String token : split2) {
+ if (token.trim().equals("")) {
+ continue;
+ }
+ s2Grams.add(token);
+ }
+
+ Set<String> overlap = new HashSet<>(s1Grams);
+ overlap.retainAll(s2Grams);
+ double totcombigrams = overlap.size();
+
+ return (2 * totcombigrams) / (s1Grams.size() + s2Grams.size());
+ }
+
+ private int minimum(int a, int b, int c) {
+ return Math.min(Math.min(a, b), c);
+ }
+
+ public int getLevenshteinDistance(CharSequence str1,
+ CharSequence str2) {
+ int[][] distance = new int[str1.length() + 1][str2.length() + 1];
+
+ for (int i = 0; i <= str1.length(); i++) {
+ distance[i][0] = i;
+ }
+ for (int j = 1; j <= str2.length(); j++) {
+ distance[0][j] = j;
+ }
+
+ for (int i = 1; i <= str1.length(); i++) {
+ for (int j = 1; j <= str2.length(); j++) {
+ distance[i][j] = minimum(
+ distance[i - 1][j] + 1,
+ distance[i][j - 1] + 1,
+ distance[i - 1][j - 1] + ((str1.charAt(i - 1) == str2.charAt(j - 1)) ? 0 : 1));
+ }
+ }
+
+ return distance[str1.length()][str2.length()];
+ }
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/GeoHashBinningScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/GeoHashBinningScorer.java
index adf48fd..6b245d6 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/GeoHashBinningScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/GeoHashBinningScorer.java
@@ -1,61 +1,61 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import opennlp.addons.geoentitylinker.AdminBoundaryContext;
-import opennlp.addons.geoentitylinker.GazetteerEntry;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.BaseLink;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-
-/**
- * Scores toponymns based on geographic point binning. Based on the heuristic
- * that docs are generally about a small amount of locations, so one can detect
- * outliers by finding those points that are not near the majority
- *
- */
-public class GeoHashBinningScorer implements LinkedEntityScorer<AdminBoundaryContext> {
-
- private final PointClustering CLUSTERER = new PointClustering();
- private int PRECISION = 3;
-
- @Override
- public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
- List<GazetteerEntry> allGazEntries = new ArrayList<>();
-
- /*
- * collect all the gaz entry references
- */
- for (LinkedSpan<BaseLink> ls : linkedSpans) {
- for (BaseLink bl : ls.getLinkedEntries()) {
- if (bl instanceof GazetteerEntry) {
- allGazEntries.add((GazetteerEntry) bl);
- }
- }
- }
- /*
- * use the point clustering to score each hit
- */
- Map<String, List<GazetteerEntry>> cluster = CLUSTERER.cluster(allGazEntries, PRECISION);
- CLUSTERER.scoreClusters(cluster);
-
- }
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import opennlp.addons.geoentitylinker.AdminBoundaryContext;
+import opennlp.addons.geoentitylinker.GazetteerEntry;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.BaseLink;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ * Scores toponymns based on geographic point binning. Based on the heuristic
+ * that docs are generally about a small amount of locations, so one can detect
+ * outliers by finding those points that are not near the majority
+ *
+ */
+public class GeoHashBinningScorer implements LinkedEntityScorer<AdminBoundaryContext> {
+
+ private final PointClustering CLUSTERER = new PointClustering();
+ private int PRECISION = 3;
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+ List<GazetteerEntry> allGazEntries = new ArrayList<>();
+
+ /*
+ * collect all the gaz entry references
+ */
+ for (LinkedSpan<BaseLink> ls : linkedSpans) {
+ for (BaseLink bl : ls.getLinkedEntries()) {
+ if (bl instanceof GazetteerEntry) {
+ allGazEntries.add((GazetteerEntry) bl);
+ }
+ }
+ }
+ /*
+ * use the point clustering to score each hit
+ */
+ Map<String, List<GazetteerEntry>> cluster = CLUSTERER.cluster(allGazEntries, PRECISION);
+ CLUSTERER.scoreClusters(cluster);
+
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/LinkedEntityScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/LinkedEntityScorer.java
index 5741af0..8054d6f 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/LinkedEntityScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/LinkedEntityScorer.java
@@ -1,40 +1,40 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.util.List;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-
-/**
- * Structure for scoring linked entities. The Map logically represents a pair :
- * "Score type" to the "actual Score."
- * @param <T> a generic for providing additional context
- */
-public interface LinkedEntityScorer<T> {
-
-/**
- * Scores a collection of linked entities. Implementations should populate the scoreMap in the list of BaseLink for each linkedSpan
- * this method internally affects the reference to linkedSpans that was passed in
- * @param linkedSpans the spans that have been linked to some external source and have all the data they need to be scored
- * @param docText the full text of the document.
- * @param sentenceSpans the sentence spans the correspond to the document text
- * @param properties the entitylinker properties config file
- * @param additionalContext any additional data required to perform the scoring operation
- */
- void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, T additionalContext);
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.util.List;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ * Structure for scoring linked entities. The Map logically represents a pair :
+ * "Score type" to the "actual Score."
+ * @param <T> a generic for providing additional context
+ */
+public interface LinkedEntityScorer<T> {
+
+/**
+ * Scores a collection of linked entities. Implementations should populate the scoreMap in the list of BaseLink for each linkedSpan
+ * this method internally affects the reference to linkedSpans that was passed in
+ * @param linkedSpans the spans that have been linked to some external source and have all the data they need to be scored
+ * @param docText the full text of the document.
+ * @param sentenceSpans the sentence spans the correspond to the document text
+ * @param properties the entitylinker properties config file
+ * @param additionalContext any additional data required to perform the scoring operation
+ */
+ void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, T additionalContext);
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ModelBasedScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ModelBasedScorer.java
index 8d473d0..1c39855 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ModelBasedScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ModelBasedScorer.java
@@ -1,157 +1,157 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.io.File;
-import java.lang.invoke.MethodHandles;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import opennlp.addons.geoentitylinker.AdminBoundaryContext;
-import opennlp.tools.doccat.DoccatModel;
-import opennlp.tools.doccat.DocumentCategorizerME;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.BaseLink;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- * Utilizes a doccat model to score toponyms based on surrounding context
- */
-public class ModelBasedScorer implements LinkedEntityScorer<AdminBoundaryContext> {
-
- private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- DocumentCategorizerME documentCategorizerME;
- DoccatModel doccatModel;
- public static final int RADIUS = 200;
- boolean modelexists = false;
-
- @Override
- public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
- try {
- if (doccatModel == null) {
- String path = properties.getProperty("opennlp.geoentitylinker.modelbasedscorer.modelpath", "");
- if (path.equals("")) {
- return;
- }
- modelexists = true;
- doccatModel = new DoccatModel(new File(path));
- documentCategorizerME = new DocumentCategorizerME(doccatModel);
- }
- Map<Integer, String> proximalFeatures = generateProximalFeatures(linkedSpans, sentenceSpans, docText, RADIUS);
- for (Map.Entry<Integer, String> entry : proximalFeatures.entrySet()) {
- Map<String, Double> scores = this.getScore(entry.getValue());
- for (BaseLink link : (List<BaseLink>) linkedSpans.get(entry.getKey()).getLinkedEntries()) {
- double score = 0d;
- if (scores.containsKey(link.getItemParentID())) {
- score = scores.get(link.getItemParentID());
- }
- link.getScoreMap().put("countrymodel", score);
- }
- }
-
- } catch (Exception ex) {
- LOG.error(ex.getLocalizedMessage(), ex);
- }
- }
-
- /**
- * generates features using a BagOfWordsfeatureGenerator that are within the
- * radius of a mention within the doctext
- *
- * @param linkedSpans
- * @param sentenceSpans
- * @param docText
- * @param radius
- * @return a map of the index of the linked span to the string of surrounding
- * text: Map<indexofspan,surrounding text>
- */
- public Map<Integer, String> generateProximalFeatures(List<LinkedSpan> linkedSpans, Span[] sentenceSpans, String docText, int radius) {
- Map<Integer, String> featureBags = new HashMap<>();
- Map<Integer, Integer> nameMentionMap = new HashMap<>();
- /*
- * iterator over the map that contains a mapping of every country code to
- * all of its mentions in the document
- */
- for (int i = 0; i < linkedSpans.size(); i++) {
- LinkedSpan<?> span = linkedSpans.get(i);
- if (span.getLinkedEntries().isEmpty()) {
- //don't care about spans that did not get linked to anything at all; nothing to work with
- continue;
- }
- /*
- * get the sentence the name span was found in, the beginning of the
- * sentence will suffice as a centroid for feature generation around the
- * named entity
- */
- Integer mentionIdx = sentenceSpans[span.getSentenceid()].getStart();
- nameMentionMap.put(i, mentionIdx);
- }
- /*
- * now associate each span to a string that will be used for categorization
- * against the model.
- */
- for (Map.Entry<Integer, Integer> entry : nameMentionMap.entrySet()) {
- featureBags.put(entry.getKey(), getTextChunk(entry.getValue(), docText, radius));
- }
-
- return featureBags;
- }
-
- public String getTextChunk(int mentionIdx, String docText, int radius) {
- int docSize = docText.length();
- int left = 0, right = 0;
- left = (mentionIdx - radius < 0) ? 0 : mentionIdx - radius;
- right = (mentionIdx + radius > docSize) ? docSize : mentionIdx + radius;
- String chunk = "";
- if (right <= left) {
- chunk = "";
- } else {
- /*
- * don't want to chop any words in half, so take fron the first space to
- * the last space in the chunk string
- */
- chunk = docText.substring(left, right);
- if (left != 0) {
- left = chunk.indexOf(" ");
- }
- right = chunk.lastIndexOf(" ");
- /*
- * now get the substring again with only whole words
- */
- if (left < right) {
- chunk = chunk.substring(left, right);
- }
- }
-
- return chunk;
- }
-
- private Map<String, Double> getScore(String text) throws Exception {
- Map<String, Double> scoreMap = new HashMap<>();
- double[] categorize = documentCategorizerME.categorize(List.of(text).toArray(new String[0]));
- int catSize = documentCategorizerME.getNumberOfCategories();
- for (int i = 0; i < catSize; i++) {
- String category = documentCategorizerME.getCategory(i);
- scoreMap.put(category, categorize[documentCategorizerME.getIndex(category)]);
- }
- return scoreMap;
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.io.File;
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import opennlp.addons.geoentitylinker.AdminBoundaryContext;
+import opennlp.tools.doccat.DoccatModel;
+import opennlp.tools.doccat.DocumentCategorizerME;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.BaseLink;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * Utilizes a doccat model to score toponyms based on surrounding context
+ */
+public class ModelBasedScorer implements LinkedEntityScorer<AdminBoundaryContext> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ DocumentCategorizerME documentCategorizerME;
+ DoccatModel doccatModel;
+ public static final int RADIUS = 200;
+ boolean modelexists = false;
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+ try {
+ if (doccatModel == null) {
+ String path = properties.getProperty("opennlp.geoentitylinker.modelbasedscorer.modelpath", "");
+ if (path.equals("")) {
+ return;
+ }
+ modelexists = true;
+ doccatModel = new DoccatModel(new File(path));
+ documentCategorizerME = new DocumentCategorizerME(doccatModel);
+ }
+ Map<Integer, String> proximalFeatures = generateProximalFeatures(linkedSpans, sentenceSpans, docText, RADIUS);
+ for (Map.Entry<Integer, String> entry : proximalFeatures.entrySet()) {
+ Map<String, Double> scores = this.getScore(entry.getValue());
+ for (BaseLink link : (List<BaseLink>) linkedSpans.get(entry.getKey()).getLinkedEntries()) {
+ double score = 0d;
+ if (scores.containsKey(link.getItemParentID())) {
+ score = scores.get(link.getItemParentID());
+ }
+ link.getScoreMap().put("countrymodel", score);
+ }
+ }
+
+ } catch (Exception ex) {
+ LOG.error(ex.getLocalizedMessage(), ex);
+ }
+ }
+
+ /**
+ * generates features using a BagOfWordsfeatureGenerator that are within the
+ * radius of a mention within the doctext
+ *
+ * @param linkedSpans
+ * @param sentenceSpans
+ * @param docText
+ * @param radius
+ * @return a map of the index of the linked span to the string of surrounding
+ * text: Map<indexofspan,surrounding text>
+ */
+ public Map<Integer, String> generateProximalFeatures(List<LinkedSpan> linkedSpans, Span[] sentenceSpans, String docText, int radius) {
+ Map<Integer, String> featureBags = new HashMap<>();
+ Map<Integer, Integer> nameMentionMap = new HashMap<>();
+ /*
+ * iterator over the map that contains a mapping of every country code to
+ * all of its mentions in the document
+ */
+ for (int i = 0; i < linkedSpans.size(); i++) {
+ LinkedSpan<?> span = linkedSpans.get(i);
+ if (span.getLinkedEntries().isEmpty()) {
+ //don't care about spans that did not get linked to anything at all; nothing to work with
+ continue;
+ }
+ /*
+ * get the sentence the name span was found in, the beginning of the
+ * sentence will suffice as a centroid for feature generation around the
+ * named entity
+ */
+ Integer mentionIdx = sentenceSpans[span.getSentenceid()].getStart();
+ nameMentionMap.put(i, mentionIdx);
+ }
+ /*
+ * now associate each span to a string that will be used for categorization
+ * against the model.
+ */
+ for (Map.Entry<Integer, Integer> entry : nameMentionMap.entrySet()) {
+ featureBags.put(entry.getKey(), getTextChunk(entry.getValue(), docText, radius));
+ }
+
+ return featureBags;
+ }
+
+ public String getTextChunk(int mentionIdx, String docText, int radius) {
+ int docSize = docText.length();
+ int left = 0, right = 0;
+ left = (mentionIdx - radius < 0) ? 0 : mentionIdx - radius;
+ right = (mentionIdx + radius > docSize) ? docSize : mentionIdx + radius;
+ String chunk = "";
+ if (right <= left) {
+ chunk = "";
+ } else {
+ /*
+ * don't want to chop any words in half, so take fron the first space to
+ * the last space in the chunk string
+ */
+ chunk = docText.substring(left, right);
+ if (left != 0) {
+ left = chunk.indexOf(" ");
+ }
+ right = chunk.lastIndexOf(" ");
+ /*
+ * now get the substring again with only whole words
+ */
+ if (left < right) {
+ chunk = chunk.substring(left, right);
+ }
+ }
+
+ return chunk;
+ }
+
+ private Map<String, Double> getScore(String text) throws Exception {
+ Map<String, Double> scoreMap = new HashMap<>();
+ double[] categorize = documentCategorizerME.categorize(List.of(text).toArray(new String[0]));
+ int catSize = documentCategorizerME.getNumberOfCategories();
+ for (int i = 0; i < catSize; i++) {
+ String category = documentCategorizerME.getCategory(i);
+ scoreMap.put(category, categorize[documentCategorizerME.getIndex(category)]);
+ }
+ return scoreMap;
+ }
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PlacetypeScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PlacetypeScorer.java
index 4c8e151..9a3e2d1 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PlacetypeScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PlacetypeScorer.java
@@ -1,82 +1,82 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import opennlp.addons.geoentitylinker.AdminBoundaryContext;
-import opennlp.addons.geoentitylinker.GazetteerEntry;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-
-/**
- *
- * @author mgiaconia
- */
-public class PlacetypeScorer implements LinkedEntityScorer<AdminBoundaryContext> {
-
- private static final String[] boosts = "ADM1 ADM1H ADM2 ADM2H ADM3 ADM3H ADM4 ADM4H ADM5 ADMD ADMDH PCLD PCLH PCLI PCLIX TERR PCLIX PPL PPLA PPLA2 PPLA3 PPLA4 PPLC PPLCH PPLF PPLG PPLH PPLL PPLQ PPLR PPLS PPLX STLMT civil Populated_Place".split(" ");
- private Map<String, Double> boosetedTypes = new HashMap<>();
-
- public PlacetypeScorer() {
- fillMap();
- }
-
- @Override
- public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
- for (LinkedSpan<GazetteerEntry> geospan : linkedSpans) {
- ArrayList<GazetteerEntry> linkedEntries = geospan.getLinkedEntries();
- for (GazetteerEntry gazetteerEntry : linkedEntries) {
- String type = gazetteerEntry.getItemType().toLowerCase();
- Double score = getScore(type);
- if (score == null) {
- score = 0d;
- }
- gazetteerEntry.getScoreMap().put("typescore", score);
- }
- }
- }
-
- private Double getScore(String type) {
- Double ret = boosetedTypes.get(type.toLowerCase());
- return ret == null ? 0d : ret;
- }
-
- private void fillMap() {
- if (boosetedTypes.isEmpty()) {
- for (String type : boosts) {
- if (type.equals("PCLI")) {
- boosetedTypes.put(type.toLowerCase(), 1d);
- } else if ((type.startsWith("PC")|| type.startsWith("PP")) && !type.equals("PCLI")) {
- boosetedTypes.put(type.toLowerCase(), .5d);
- } else if (type.startsWith("ADM")) {
- boosetedTypes.put(type.toLowerCase(), .75d);
- }else if (type.equalsIgnoreCase("civil")){
- boosetedTypes.put(type.toLowerCase(), .9d);
- }
- if(type.equalsIgnoreCase("populated_place")){
- boosetedTypes.put("Populated Place", .75d);
- }
-
- }
- boosetedTypes.put("pplc", .9);
- }
- }
-
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import opennlp.addons.geoentitylinker.AdminBoundaryContext;
+import opennlp.addons.geoentitylinker.GazetteerEntry;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ *
+ * @author mgiaconia
+ */
+public class PlacetypeScorer implements LinkedEntityScorer<AdminBoundaryContext> {
+
+ private static final String[] boosts = "ADM1 ADM1H ADM2 ADM2H ADM3 ADM3H ADM4 ADM4H ADM5 ADMD ADMDH PCLD PCLH PCLI PCLIX TERR PCLIX PPL PPLA PPLA2 PPLA3 PPLA4 PPLC PPLCH PPLF PPLG PPLH PPLL PPLQ PPLR PPLS PPLX STLMT civil Populated_Place".split(" ");
+ private Map<String, Double> boosetedTypes = new HashMap<>();
+
+ public PlacetypeScorer() {
+ fillMap();
+ }
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+ for (LinkedSpan<GazetteerEntry> geospan : linkedSpans) {
+ ArrayList<GazetteerEntry> linkedEntries = geospan.getLinkedEntries();
+ for (GazetteerEntry gazetteerEntry : linkedEntries) {
+ String type = gazetteerEntry.getItemType().toLowerCase();
+ Double score = getScore(type);
+ if (score == null) {
+ score = 0d;
+ }
+ gazetteerEntry.getScoreMap().put("typescore", score);
+ }
+ }
+ }
+
+ private Double getScore(String type) {
+ Double ret = boosetedTypes.get(type.toLowerCase());
+ return ret == null ? 0d : ret;
+ }
+
+ private void fillMap() {
+ if (boosetedTypes.isEmpty()) {
+ for (String type : boosts) {
+ if (type.equals("PCLI")) {
+ boosetedTypes.put(type.toLowerCase(), 1d);
+ } else if ((type.startsWith("PC")|| type.startsWith("PP")) && !type.equals("PCLI")) {
+ boosetedTypes.put(type.toLowerCase(), .5d);
+ } else if (type.startsWith("ADM")) {
+ boosetedTypes.put(type.toLowerCase(), .75d);
+ }else if (type.equalsIgnoreCase("civil")){
+ boosetedTypes.put(type.toLowerCase(), .9d);
+ }
+ if(type.equalsIgnoreCase("populated_place")){
+ boosetedTypes.put("Populated Place", .75d);
+ }
+
+ }
+ boosetedTypes.put("pplc", .9);
+ }
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PointClustering.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PointClustering.java
index 8cd1440..a8677e4 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PointClustering.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PointClustering.java
@@ -1,121 +1,121 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.io.GeohashUtils;
-import com.spatial4j.core.shape.Point;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import opennlp.addons.geoentitylinker.GazetteerEntry;
-
-/**
- *
- * Clusters a list of lat long points using a simple geohashing/boxing approach
- */
-public class PointClustering {
-
- /**
- * Clusters a set of points from the gazateers. The idea is that locations
- * that matched a name that are closer to each other, the more likely the
- * toponym is to be accurate
- *
- * @param entries
- * @param precision
- * @return
- */
- public Map<String, List<GazetteerEntry>> cluster(List<GazetteerEntry> entries, int precision) {
- Map<String, List<GazetteerEntry>> map = new HashMap<>();
- for (GazetteerEntry entry : entries) {
- Double latw = entry.getLatitude();
- Double lonw = entry.getLongitude();
-
- String key = geoHash(latw, lonw).substring(0, precision);
- if (map.containsKey(key)) {
- map.get(key).add(entry);
- } else {
- List<GazetteerEntry> newlist = new ArrayList<>();
- newlist.add(entry);
- map.put(key, newlist);
- }
- }
- return map;
- }
-
- public void scoreClusters(Map<String, List<GazetteerEntry>> clusters) {
- Double min = 0d;
- Double max = -1d;
- for (String key : clusters.keySet()) {
- int size = clusters.get(key).size();
- if (size > max) {
- max = (double) size;
- }
- }
- for (String key : clusters.keySet()) {
- int size = clusters.get(key).size();
- Double score = normalize((double) size, min, max);
- for (GazetteerEntry entry : clusters.get(key)) {
- entry.getScoreMap().put("geohashbin", score);
- }
- }
-
- }
-
- /**
- * Returns a geohash based on Lucene Spatial
- *
- * @param lat the input latitude Y
- * @param lon the input longitude X
- * @return
- */
- public String geoHash(Double lat, Double lon) {
- String encodeLatLon = GeohashUtils.encodeLatLon(lat, lon);
- return encodeLatLon;
- }
-
- /**
- * Returns the X and Y point for the geohash. Element 0 is the X (longitude)
- * element 1 is the Y (latitude)
- *
- * @param geohash
- * @return
- */
- public double[] geoHashToPoint(String geohash) {
- Point decode = GeohashUtils.decode(geohash, SpatialContext.GEO);
- double[] coords = new double[]{decode.getX(), decode.getY()};
- return coords;
- }
-
- /**
- * Returns the X and Y point for the geohash. Element 0 is the X (longitude)
- * element 1 is the Y (latitude)
- *
- * @param geohash
- * @return
- */
- public String geoHashToPointStr(String geohash) {
- Point decode = GeohashUtils.decode(geohash, SpatialContext.GEO);
- String point = decode.getX() + "," + decode.getY();
- return point;
- }
-
-
- private Double normalize(Double valueToNormalize, double minimum, double maximum) {
- return ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
- }
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.io.GeohashUtils;
+import com.spatial4j.core.shape.Point;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import opennlp.addons.geoentitylinker.GazetteerEntry;
+
+/**
+ *
+ * Clusters a list of lat long points using a simple geohashing/boxing approach
+ */
+public class PointClustering {
+
+ /**
+ * Clusters a set of points from the gazateers. The idea is that locations
+ * that matched a name that are closer to each other, the more likely the
+ * toponym is to be accurate
+ *
+ * @param entries
+ * @param precision
+ * @return
+ */
+ public Map<String, List<GazetteerEntry>> cluster(List<GazetteerEntry> entries, int precision) {
+ Map<String, List<GazetteerEntry>> map = new HashMap<>();
+ for (GazetteerEntry entry : entries) {
+ Double latw = entry.getLatitude();
+ Double lonw = entry.getLongitude();
+
+ String key = geoHash(latw, lonw).substring(0, precision);
+ if (map.containsKey(key)) {
+ map.get(key).add(entry);
+ } else {
+ List<GazetteerEntry> newlist = new ArrayList<>();
+ newlist.add(entry);
+ map.put(key, newlist);
+ }
+ }
+ return map;
+ }
+
+ public void scoreClusters(Map<String, List<GazetteerEntry>> clusters) {
+ Double min = 0d;
+ Double max = -1d;
+ for (String key : clusters.keySet()) {
+ int size = clusters.get(key).size();
+ if (size > max) {
+ max = (double) size;
+ }
+ }
+ for (String key : clusters.keySet()) {
+ int size = clusters.get(key).size();
+ Double score = normalize((double) size, min, max);
+ for (GazetteerEntry entry : clusters.get(key)) {
+ entry.getScoreMap().put("geohashbin", score);
+ }
+ }
+
+ }
+
+ /**
+ * Returns a geohash based on Lucene Spatial
+ *
+ * @param lat the input latitude Y
+ * @param lon the input longitude X
+ * @return
+ */
+ public String geoHash(Double lat, Double lon) {
+ String encodeLatLon = GeohashUtils.encodeLatLon(lat, lon);
+ return encodeLatLon;
+ }
+
+ /**
+ * Returns the X and Y point for the geohash. Element 0 is the X (longitude)
+ * element 1 is the Y (latitude)
+ *
+ * @param geohash
+ * @return
+ */
+ public double[] geoHashToPoint(String geohash) {
+ Point decode = GeohashUtils.decode(geohash, SpatialContext.GEO);
+ double[] coords = new double[]{decode.getX(), decode.getY()};
+ return coords;
+ }
+
+ /**
+ * Returns the X and Y point for the geohash. Element 0 is the X (longitude)
+ * element 1 is the Y (latitude)
+ *
+ * @param geohash
+ * @return
+ */
+ public String geoHashToPointStr(String geohash) {
+ Point decode = GeohashUtils.decode(geohash, SpatialContext.GEO);
+ String point = decode.getX() + "," + decode.getY();
+ return point;
+ }
+
+
+ private Double normalize(Double valueToNormalize, double minimum, double maximum) {
+ return ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
+ }
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ProvinceProximityScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ProvinceProximityScorer.java
index a21af45..27e9354 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ProvinceProximityScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ProvinceProximityScorer.java
@@ -1,294 +1,294 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.regex.Pattern;
-import opennlp.addons.geoentitylinker.AdminBoundaryContext;
-import opennlp.addons.geoentitylinker.GazetteerEntry;
-import opennlp.tools.entitylinker.BaseLink;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-
-/**
- * Scores toponyms based on their proximity to a province mention. Based on the
- * heuristic that toponymn mentions are more likely close to their parent
- * province mentions. For instance, if the toponym Berlin is mentioned near an
- * indicator of Connecticut, it is more likely to be Berlin Connecticut than
- * Berlin Germany (if Germany did not exist in, or is mentioned further away in
- * the article).
- *
- *
- */
-public class ProvinceProximityScorer implements LinkedEntityScorer<AdminBoundaryContext> {
-
- private Map<String, Set<String>> nameCodesMap;
- String dominantCode = "";
- private Map<String, String> regexMap = new HashMap<>();
-
- @Override
- public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
- if (!additionalContext.getProvHits().isEmpty()) {
- regexMap = additionalContext.getProvinceRegexMap();
- score(linkedSpans, additionalContext.getProvMentions(), additionalContext.getNameCodesMap(), docText, sentenceSpans, 1000);
- } else {
- for (LinkedSpan<BaseLink> span : linkedSpans) {
- for (BaseLink link : span.getLinkedEntries()) {
- link.getScoreMap().put("provincecontext", 0d);
- }
- }
- }
-
- }
-
- /**
- * Assigns a score to each BaseLink in each linkedSpan's set of N best
- * matches. Currently the scoring indicates the probability that the toponym
- * is correct based on the country context in the document
- *
- * @param linkedData the linked spans, holds the Namefinder results, and the
- * list of BaseLink for each
- * @param countryHits all the country mentions in the document
- * @param nameCodesMap maps a province indicator name to a province code. Used
- * to determine if the namefinder found the same exact toponym the country
- * context did. If so the score is boosted due to the high probability that
- * the NameFinder actually "rediscovered" a country
- * @param docText the full text of the document...not used in this default
- * implementation
- * @param sentences the sentences that correspond to the doc text.
- * @param maxAllowedDist a constant that is used to determine which country
- * mentions, based on proximity within the text, should be used to score the
- * Named Entity.
- * @return
- */
- public List<LinkedSpan> score(List<LinkedSpan> linkedData, Map<String, Set<Integer>> countryHits, Map<String, Set<String>> nameCodesMap, String docText, Span[] sentences, Integer maxAllowedDist) {
- this.nameCodesMap = nameCodesMap;
- setDominantCode(countryHits);
- for (LinkedSpan<BaseLink> linkedspan : linkedData) {
-
- linkedspan = simpleProximityAnalysis(sentences, countryHits, linkedspan, maxAllowedDist);
- }
- return linkedData;
- }
-
- /**
- * sets class level variable to a code based on the number of mentions
- *
- * @param countryHits
- */
- private void setDominantCode(Map<String, Set<Integer>> countryHits) {
- int hits = -1;
- for (String code : countryHits.keySet()) {
- if (countryHits.get(code).size() > hits) {
- hits = countryHits.get(code).size();
- dominantCode = code;
- }
- }
- }
-
- /**
- * Generates distances from each country mention to the span's location in the
- * doc text. Ultimately an attempt to ensure that ambiguously named toponyms
- * are resolved to the correct country and coordinate.
- *
- * @param sentences
- * @param countryHits
- * @param span
- * @return
- */
- private LinkedSpan<BaseLink> simpleProximityAnalysis(Span[] sentences, Map<String, Set<Integer>> countryHits, LinkedSpan<BaseLink> span, Integer maxAllowedDistance) {
- Double score = 0.0;
- /*
- * get the index of the actual span, begining of sentence //should generate
- * tokens from sentence and create a char offset... //could have large
- * sentences due to poor sentence detection or wonky doc text
- */
- int sentenceIdx = span.getSentenceid();
- int sentIndexInDoc = sentences[sentenceIdx].getStart();
- /**
- * create a map of all the span's proximal country mentions in the document
- * Map< countrycode, set of <distances from this NamedEntity>>
- */
- Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<>();
- //map = Map<countrycode, Set <of distances this span is from all the mentions of the code>>
- for (String cCode : countryHits.keySet()) {
-//iterate over all the regex start values and calculate an offset
- for (Integer cHit : countryHits.get(cCode)) {
- Integer absDist = Math.abs(sentIndexInDoc - cHit);
- //only include near mentions based on a heuristic
- //TODO make this a property
- // if (absDist < maxAllowedDistance) {
- if (distancesFromCodeMap.containsKey(cCode)) {
- distancesFromCodeMap.get(cCode).add(absDist);
- } else {
- HashSet<Integer> newset = new HashSet<>();
- newset.add(absDist);
- distancesFromCodeMap.put(cCode, newset);
- }
- }
-
- //}
- }
- //we now know how far this named entity is from every country mention in the document
-
- /**
- * the gaz matches that have a country code that have mentions in the doc
- * that are closest to the Named Entity should return the best score.
- * Analyzemap generates a likelihood score that the toponym from the gaz is
- * referring to one of the countries, i.e, Map<countrycode, prob that this
- * span is referring to the toponym form this code key>
- */
- Map<String, Double> scoreMap = analyzeMap(distancesFromCodeMap, sentences, span);
- if (scoreMap.isEmpty()) {
- return span;
- }
- for (BaseLink link : span.getLinkedEntries()) {
- //getItemParentId is the country code
- GazetteerEntry entry = (GazetteerEntry) link;
- String spanCountryCode = entry.getProvinceCode();
- if (scoreMap.containsKey(spanCountryCode)) {
-
- score = scoreMap.get(spanCountryCode);
- ///does the name extracted match a province name?
- if (nameCodesMap.containsKey(link.getItemName().toLowerCase()) || regexMatch(link.getItemName(), link.getItemParentID())) {
- //if so, is it the correct country code for that name?
- if (nameCodesMap.get(entry.getItemName().toLowerCase()).contains(entry.getProvinceCode())) {
- //boost the score becuase it is likely that this is the location in the text, so add 50% to the score or set to 1
- //TODO: make this smarter
- score = (score + .75) > 1.0 ? 1d : (score + .75);
-
- if (entry.getProvinceCode().equals(dominantCode)) {
- score = (score + .25) > 1.0 ? 1d : (score + .25);
- }
- }
- }
- }
- link.getScoreMap().put("provincecontext", score);
- }
- return span;
- }
-
- private boolean regexMatch(String placeName, String countryCode) {
- if (regexMap.containsKey(countryCode)) {
- String regexForCountry = regexMap.get(countryCode);
-
- Pattern p = Pattern.compile(regexForCountry, Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
- return p.matcher(placeName.trim()).matches();
- }
- return false;
- }
-
- /**
- * takes a map of distances from the toponym to each province mention and
- * generates a map of scores for each province code. The map is then
- * correlated to the code of the BaseLink parentid for retrieval. Then the
- * score is added to the overall list.
- *
- * @param distanceMap
- * @param sentences
- * @param span
- * @return
- */
- private Map<String, Double> analyzeMap(Map<String, Set<Integer>> distanceMap, Span[] sentences, LinkedSpan<BaseLink> span) {
-
- Map<String, Double> scoreMap = new HashMap<>();
- if (distanceMap.isEmpty()) {
- return scoreMap;
- }
- TreeSet<Integer> all = new TreeSet<>();
- for (String key : distanceMap.keySet()) {
- all.addAll(distanceMap.get(key));
- }
- //get min max for normalization, this could be more efficient
-
- int min = all.first();
- int max = all.last();
- if (min == max) {
- min = 0;
- }
- for (String key : distanceMap.keySet()) {
-
- TreeSet<Double> normalizedDistances = new TreeSet<>();
- for (Integer i : distanceMap.get(key)) {
- Double norm = normalize(i, min, max);
- //reverse the normed distance so low numbers (closer) are better
- //this could be improved with a "decaying " function using an imcreaseing negative exponent
- Double reverse = Math.abs(norm - 1);
- normalizedDistances.add(reverse);
- }
-
- List<Double> doubles = new ArrayList<>(normalizedDistances);
- scoreMap.put(key, slidingDistanceAverage(doubles));
- }
- return scoreMap;
- }
-
- /**
- * this method is an attempt to make closer clusters of mentions group
- * together to smooth out the average, so one distant outlier does not kill
- * the score for an obviously good hit. More elegant solution is possible
- * using Math.pow, and making the score decay with distance by using an
- * increasing negative exponent
- *
- * @param normDis the normalized and sorted set of distances as a list
- * @return
- */
- private Double slidingDistanceAverage(List<Double> normDis) {
- List<Double> windowOfAverages = new ArrayList<>();
-
- if (normDis.size() < 3) {
- windowOfAverages.addAll(normDis);
- } else {
-
- for (int i = 0; i < normDis.size() - 1; i++) {
- double a = normDis.get(i);
- double b = normDis.get(i + 1);
- windowOfAverages.add((a + b) / 2);
-
- }
- }
- double sum = 0d;
- for (double d : windowOfAverages) {
- sum += d;
- }
- double result = sum / windowOfAverages.size();
- //TODO: ++ prob when large amounts of mentions for a code
- //System.out.println("avg of window:" + result);
- return result;
- }
-
- /**
- * transposes a value within one range to a relative value in a different
- * range. Used to normalize distances in this class.
- *
- * @param valueToNormalize the value to place within the new range
- * @param minimum the min of the set to be transposed
- * @param maximum the max of the set to be transposed
- * @return
- */
- private Double normalize(int valueToNormalize, int minimum, int maximum) {
- Double d = (double) ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
- d = d == null ? 0d : d;
- return d;
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Pattern;
+import opennlp.addons.geoentitylinker.AdminBoundaryContext;
+import opennlp.addons.geoentitylinker.GazetteerEntry;
+import opennlp.tools.entitylinker.BaseLink;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ * Scores toponyms based on their proximity to a province mention. Based on the
+ * heuristic that toponymn mentions are more likely close to their parent
+ * province mentions. For instance, if the toponym Berlin is mentioned near an
+ * indicator of Connecticut, it is more likely to be Berlin Connecticut than
+ * Berlin Germany (if Germany did not exist in, or is mentioned further away in
+ * the article).
+ *
+ *
+ */
+public class ProvinceProximityScorer implements LinkedEntityScorer<AdminBoundaryContext> {
+
+ private Map<String, Set<String>> nameCodesMap;
+ String dominantCode = "";
+ private Map<String, String> regexMap = new HashMap<>();
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+ if (!additionalContext.getProvHits().isEmpty()) {
+ regexMap = additionalContext.getProvinceRegexMap();
+ score(linkedSpans, additionalContext.getProvMentions(), additionalContext.getNameCodesMap(), docText, sentenceSpans, 1000);
+ } else {
+ for (LinkedSpan<BaseLink> span : linkedSpans) {
+ for (BaseLink link : span.getLinkedEntries()) {
+ link.getScoreMap().put("provincecontext", 0d);
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Assigns a score to each BaseLink in each linkedSpan's set of N best
+ * matches. Currently the scoring indicates the probability that the toponym
+ * is correct based on the country context in the document
+ *
+ * @param linkedData the linked spans, holds the Namefinder results, and the
+ * list of BaseLink for each
+ * @param countryHits all the country mentions in the document
+ * @param nameCodesMap maps a province indicator name to a province code. Used
+ * to determine if the namefinder found the same exact toponym the country
+ * context did. If so the score is boosted due to the high probability that
+ * the NameFinder actually "rediscovered" a country
+ * @param docText the full text of the document...not used in this default
+ * implementation
+ * @param sentences the sentences that correspond to the doc text.
+ * @param maxAllowedDist a constant that is used to determine which country
+ * mentions, based on proximity within the text, should be used to score the
+ * Named Entity.
+ * @return
+ */
+ public List<LinkedSpan> score(List<LinkedSpan> linkedData, Map<String, Set<Integer>> countryHits, Map<String, Set<String>> nameCodesMap, String docText, Span[] sentences, Integer maxAllowedDist) {
+ this.nameCodesMap = nameCodesMap;
+ setDominantCode(countryHits);
+ for (LinkedSpan<BaseLink> linkedspan : linkedData) {
+
+ linkedspan = simpleProximityAnalysis(sentences, countryHits, linkedspan, maxAllowedDist);
+ }
+ return linkedData;
+ }
+
+ /**
+ * sets class level variable to a code based on the number of mentions
+ *
+ * @param countryHits
+ */
+ private void setDominantCode(Map<String, Set<Integer>> countryHits) {
+ int hits = -1;
+ for (String code : countryHits.keySet()) {
+ if (countryHits.get(code).size() > hits) {
+ hits = countryHits.get(code).size();
+ dominantCode = code;
+ }
+ }
+ }
+
+ /**
+ * Generates distances from each country mention to the span's location in the
+ * doc text. Ultimately an attempt to ensure that ambiguously named toponyms
+ * are resolved to the correct country and coordinate.
+ *
+ * @param sentences
+ * @param countryHits
+ * @param span
+ * @return
+ */
+ private LinkedSpan<BaseLink> simpleProximityAnalysis(Span[] sentences, Map<String, Set<Integer>> countryHits, LinkedSpan<BaseLink> span, Integer maxAllowedDistance) {
+ Double score = 0.0;
+ /*
+ * get the index of the actual span, begining of sentence //should generate
+ * tokens from sentence and create a char offset... //could have large
+ * sentences due to poor sentence detection or wonky doc text
+ */
+ int sentenceIdx = span.getSentenceid();
+ int sentIndexInDoc = sentences[sentenceIdx].getStart();
+ /**
+ * create a map of all the span's proximal country mentions in the document
+ * Map< countrycode, set of <distances from this NamedEntity>>
+ */
+ Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<>();
+ //map = Map<countrycode, Set <of distances this span is from all the mentions of the code>>
+ for (String cCode : countryHits.keySet()) {
+//iterate over all the regex start values and calculate an offset
+ for (Integer cHit : countryHits.get(cCode)) {
+ Integer absDist = Math.abs(sentIndexInDoc - cHit);
+ //only include near mentions based on a heuristic
+ //TODO make this a property
+ // if (absDist < maxAllowedDistance) {
+ if (distancesFromCodeMap.containsKey(cCode)) {
+ distancesFromCodeMap.get(cCode).add(absDist);
+ } else {
+ HashSet<Integer> newset = new HashSet<>();
+ newset.add(absDist);
+ distancesFromCodeMap.put(cCode, newset);
+ }
+ }
+
+ //}
+ }
+ //we now know how far this named entity is from every country mention in the document
+
+ /**
+ * the gaz matches that have a country code that have mentions in the doc
+ * that are closest to the Named Entity should return the best score.
+ * Analyzemap generates a likelihood score that the toponym from the gaz is
+ * referring to one of the countries, i.e, Map<countrycode, prob that this
+ * span is referring to the toponym form this code key>
+ */
+ Map<String, Double> scoreMap = analyzeMap(distancesFromCodeMap, sentences, span);
+ if (scoreMap.isEmpty()) {
+ return span;
+ }
+ for (BaseLink link : span.getLinkedEntries()) {
+ //getItemParentId is the country code
+ GazetteerEntry entry = (GazetteerEntry) link;
+ String spanCountryCode = entry.getProvinceCode();
+ if (scoreMap.containsKey(spanCountryCode)) {
+
+ score = scoreMap.get(spanCountryCode);
+ ///does the name extracted match a province name?
+ if (nameCodesMap.containsKey(link.getItemName().toLowerCase()) || regexMatch(link.getItemName(), link.getItemParentID())) {
+ //if so, is it the correct country code for that name?
+ if (nameCodesMap.get(entry.getItemName().toLowerCase()).contains(entry.getProvinceCode())) {
+ //boost the score becuase it is likely that this is the location in the text, so add 50% to the score or set to 1
+ //TODO: make this smarter
+ score = (score + .75) > 1.0 ? 1d : (score + .75);
+
+ if (entry.getProvinceCode().equals(dominantCode)) {
+ score = (score + .25) > 1.0 ? 1d : (score + .25);
+ }
+ }
+ }
+ }
+ link.getScoreMap().put("provincecontext", score);
+ }
+ return span;
+ }
+
+ private boolean regexMatch(String placeName, String countryCode) {
+ if (regexMap.containsKey(countryCode)) {
+ String regexForCountry = regexMap.get(countryCode);
+
+ Pattern p = Pattern.compile(regexForCountry, Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
+ return p.matcher(placeName.trim()).matches();
+ }
+ return false;
+ }
+
+ /**
+ * takes a map of distances from the toponym to each province mention and
+ * generates a map of scores for each province code. The map is then
+ * correlated to the code of the BaseLink parentid for retrieval. Then the
+ * score is added to the overall list.
+ *
+ * @param distanceMap
+ * @param sentences
+ * @param span
+ * @return
+ */
+ private Map<String, Double> analyzeMap(Map<String, Set<Integer>> distanceMap, Span[] sentences, LinkedSpan<BaseLink> span) {
+
+ Map<String, Double> scoreMap = new HashMap<>();
+ if (distanceMap.isEmpty()) {
+ return scoreMap;
+ }
+ TreeSet<Integer> all = new TreeSet<>();
+ for (String key : distanceMap.keySet()) {
+ all.addAll(distanceMap.get(key));
+ }
+ //get min max for normalization, this could be more efficient
+
+ int min = all.first();
+ int max = all.last();
+ if (min == max) {
+ min = 0;
+ }
+ for (String key : distanceMap.keySet()) {
+
+ TreeSet<Double> normalizedDistances = new TreeSet<>();
+ for (Integer i : distanceMap.get(key)) {
+ Double norm = normalize(i, min, max);
+ //reverse the normed distance so low numbers (closer) are better
+ //this could be improved with a "decaying " function using an imcreaseing negative exponent
+ Double reverse = Math.abs(norm - 1);
+ normalizedDistances.add(reverse);
+ }
+
+ List<Double> doubles = new ArrayList<>(normalizedDistances);
+ scoreMap.put(key, slidingDistanceAverage(doubles));
+ }
+ return scoreMap;
+ }
+
+ /**
+ * this method is an attempt to make closer clusters of mentions group
+ * together to smooth out the average, so one distant outlier does not kill
+ * the score for an obviously good hit. More elegant solution is possible
+ * using Math.pow, and making the score decay with distance by using an
+ * increasing negative exponent
+ *
+ * @param normDis the normalized and sorted set of distances as a list
+ * @return
+ */
+ private Double slidingDistanceAverage(List<Double> normDis) {
+ List<Double> windowOfAverages = new ArrayList<>();
+
+ if (normDis.size() < 3) {
+ windowOfAverages.addAll(normDis);
+ } else {
+
+ for (int i = 0; i < normDis.size() - 1; i++) {
+ double a = normDis.get(i);
+ double b = normDis.get(i + 1);
+ windowOfAverages.add((a + b) / 2);
+
+ }
+ }
+ double sum = 0d;
+ for (double d : windowOfAverages) {
+ sum += d;
+ }
+ double result = sum / windowOfAverages.size();
+ //TODO: ++ prob when large amounts of mentions for a code
+ //System.out.println("avg of window:" + result);
+ return result;
+ }
+
+ /**
+ * transposes a value within one range to a relative value in a different
+ * range. Used to normalize distances in this class.
+ *
+ * @param valueToNormalize the value to place within the new range
+ * @param minimum the min of the set to be transposed
+ * @param maximum the max of the set to be transposed
+ * @return
+ */
+ private Double normalize(int valueToNormalize, int minimum, int maximum) {
+ Double d = (double) ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
+ d = d == null ? 0d : d;
+ return d;
+ }
+}
diff --git a/jwnl-addon/pom.xml b/jwnl-addon/pom.xml
index 9b0a8ed..ed01100 100644
--- a/jwnl-addon/pom.xml
+++ b/jwnl-addon/pom.xml
@@ -1,78 +1,78 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-addons</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>jwnl-addon</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Apache OpenNLP JWNL Addon</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-tools</artifactId>
- </dependency>
-
- <dependency>
- <groupId>net.sf.jwordnet</groupId>
- <artifactId>jwnl</artifactId>
- <version>1.3.3</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-engine</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-params</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>${maven.compiler.source}</source>
- <target>${maven.compiler.target}</target>
- <compilerArgument>-Xlint</compilerArgument>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>jwnl-addon</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <name>Apache OpenNLP JWNL Addon</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-tools</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>net.sf.jwordnet</groupId>
+ <artifactId>jwnl</artifactId>
+ <version>1.3.3</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerArgument>-Xlint</compilerArgument>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/modelbuilder-addon/pom.xml b/modelbuilder-addon/pom.xml
index b139d3b..c3109e3 100644
--- a/modelbuilder-addon/pom.xml
+++ b/modelbuilder-addon/pom.xml
@@ -1,72 +1,72 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-addons</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>modelbuilder-addon</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <name>Apache OpenNLP ModelBuilder Addon</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-tools</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-engine</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-params</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>${maven.compiler.source}</source>
- <target>${maven.compiler.target}</target>
- <compilerArgument>-Xlint</compilerArgument>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>modelbuilder-addon</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <name>Apache OpenNLP ModelBuilder Addon</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-tools</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerArgument>-Xlint</compilerArgument>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/DefaultModelBuilderUtil.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/DefaultModelBuilderUtil.java
index b613877..72f5d46 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/DefaultModelBuilderUtil.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/DefaultModelBuilderUtil.java
@@ -1,121 +1,121 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import java.io.File;
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-import opennlp.addons.modelbuilder.impls.FileKnownEntityProvider;
-import opennlp.addons.modelbuilder.impls.FileModelValidatorImpl;
-import opennlp.addons.modelbuilder.impls.FileSentenceProvider;
-import opennlp.addons.modelbuilder.impls.GenericModelGenerator;
-import opennlp.addons.modelbuilder.impls.GenericModelableImpl;
-
-/**
- * Utilizes the file-based implementations to produce an NER model from user defined data.
- * <p>
- * The basic processing is such read in the list of known entities
- * <ul>
- * <li>annotate the sentences based on the list of known entities,</li>
- * <li>create a model from the annotations,</li>
- * <li>perform NER with the model on the sentences,</li>
- * <li>add the NER results to the annotations,</li>
- * <li>rebuild the model,</li>
- * <li>loop</li>
- * </ul>
- */
-public class DefaultModelBuilderUtil {
-
- /**
- *
- * @param sentences a file that contains one sentence per line.
- * There should be at least 15K sentences
- * consisting of a representative sample from
- * user data
- * @param knownEntities a file consisting of a simple list of
- * unambiguous entities, one entry per line.
- * For instance, if one was trying to build a
- * person NER model then this file would be a
- * list of person names that are unambiguous
- * and are known to exist in the sentences
- * file
- * @param knownEntitiesBlacklist This file contains a list of known bad hits
- * that the NER phase of this processing might
- * catch early one before the model iterates
- * to maturity
- * @param modelOutFile the location where the model will be
- * written to
- * @param annotatedSentenceOutFile where the annotated sentences produced by
- * this process will be written to
- * @param namedEntityType the type of entity... for example, person,
- * location, organization...
- * @param iterations how many times to repeat the iterative loop
- * of annotation, model generation, and NER
- */
- public static void generateModel(File sentences, File knownEntities, File knownEntitiesBlacklist,
- File modelOutFile, File annotatedSentenceOutFile, String namedEntityType, int iterations) {
- SemiSupervisedModelGenerator modelGenerator = new GenericModelGenerator();
- BaseModelBuilderParams params = new BaseModelBuilderParams();
- params.setAnnotatedTrainingDataFile(annotatedSentenceOutFile);
- params.setSentenceFile(sentences);
- params.setEntityType(namedEntityType);
- params.setKnownEntitiesFile(knownEntities);
- params.setModelFile(modelOutFile);
- params.setKnownEntityBlacklist(knownEntitiesBlacklist);
-
- /*
- * sentence providers feed this process with user data derived sentences
- * this impl just reads line by line through a file
- */
- SentenceProvider sentenceProvider = new FileSentenceProvider();
- sentenceProvider.setParameters(params);
-
- /*
- * KnownEntityProviders provide a seed list of known entities... such as
- * Barack Obama for person, or Germany for location obviously these would
- * want to be prolific, non-ambiguous names
- */
- KnownEntityProvider knownEntityProvider = new FileKnownEntityProvider();
- knownEntityProvider.setParameters(params);
-
- /*
- * ModelGenerationValidators try to weed out bad hits by the iterations of
- * the name finder. Since this is a recursive process, with each iteration
- * the namefinder will get more and more greedy if bad entities are allowed
- * in this provides a mechanism for throwing out obviously bad hits. A good
- * impl may be to make sure a location is actually within a noun phrase
- * etc...users can make this as specific as they need for their dat and
- * their use case
- */
- ModelGenerationValidator validator = new FileModelValidatorImpl();
- validator.setParameters(params);
-
- /*
- * Modelable's write and read the annotated sentences, as well as create and
- * write the NER models
- */
- Modelable modelable = new GenericModelableImpl();
- modelable.setParameters(params);
-
- /*
- * the modelGenerator actually runs the process with a set number of
- * iterations... could be better by actually calculating the diff between
- * runs and stopping based on a thresh, but for extrememly large sentence
- * sets this may be too much.
- */
- modelGenerator.build(sentenceProvider, knownEntityProvider, validator, modelable, iterations);
-
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import java.io.File;
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+import opennlp.addons.modelbuilder.impls.FileKnownEntityProvider;
+import opennlp.addons.modelbuilder.impls.FileModelValidatorImpl;
+import opennlp.addons.modelbuilder.impls.FileSentenceProvider;
+import opennlp.addons.modelbuilder.impls.GenericModelGenerator;
+import opennlp.addons.modelbuilder.impls.GenericModelableImpl;
+
+/**
+ * Utilizes the file-based implementations to produce an NER model from user defined data.
+ * <p>
+ * The basic processing is such read in the list of known entities
+ * <ul>
+ * <li>annotate the sentences based on the list of known entities,</li>
+ * <li>create a model from the annotations,</li>
+ * <li>perform NER with the model on the sentences,</li>
+ * <li>add the NER results to the annotations,</li>
+ * <li>rebuild the model,</li>
+ * <li>loop</li>
+ * </ul>
+ */
+public class DefaultModelBuilderUtil {
+
+ /**
+ *
+ * @param sentences a file that contains one sentence per line.
+ * There should be at least 15K sentences
+ * consisting of a representative sample from
+ * user data
+ * @param knownEntities a file consisting of a simple list of
+ * unambiguous entities, one entry per line.
+ * For instance, if one was trying to build a
+ * person NER model then this file would be a
+ * list of person names that are unambiguous
+ * and are known to exist in the sentences
+ * file
+ * @param knownEntitiesBlacklist This file contains a list of known bad hits
+ * that the NER phase of this processing might
+ * catch early one before the model iterates
+ * to maturity
+ * @param modelOutFile the location where the model will be
+ * written to
+ * @param annotatedSentenceOutFile where the annotated sentences produced by
+ * this process will be written to
+ * @param namedEntityType the type of entity... for example, person,
+ * location, organization...
+ * @param iterations how many times to repeat the iterative loop
+ * of annotation, model generation, and NER
+ */
+ public static void generateModel(File sentences, File knownEntities, File knownEntitiesBlacklist,
+ File modelOutFile, File annotatedSentenceOutFile, String namedEntityType, int iterations) {
+ SemiSupervisedModelGenerator modelGenerator = new GenericModelGenerator();
+ BaseModelBuilderParams params = new BaseModelBuilderParams();
+ params.setAnnotatedTrainingDataFile(annotatedSentenceOutFile);
+ params.setSentenceFile(sentences);
+ params.setEntityType(namedEntityType);
+ params.setKnownEntitiesFile(knownEntities);
+ params.setModelFile(modelOutFile);
+ params.setKnownEntityBlacklist(knownEntitiesBlacklist);
+
+ /*
+ * sentence providers feed this process with user data derived sentences
+ * this impl just reads line by line through a file
+ */
+ SentenceProvider sentenceProvider = new FileSentenceProvider();
+ sentenceProvider.setParameters(params);
+
+ /*
+ * KnownEntityProviders provide a seed list of known entities... such as
+ * Barack Obama for person, or Germany for location obviously these would
+ * want to be prolific, non-ambiguous names
+ */
+ KnownEntityProvider knownEntityProvider = new FileKnownEntityProvider();
+ knownEntityProvider.setParameters(params);
+
+ /*
+ * ModelGenerationValidators try to weed out bad hits by the iterations of
+ * the name finder. Since this is a recursive process, with each iteration
+ * the namefinder will get more and more greedy if bad entities are allowed
+ * in this provides a mechanism for throwing out obviously bad hits. A good
+ * impl may be to make sure a location is actually within a noun phrase
+ * etc...users can make this as specific as they need for their dat and
+ * their use case
+ */
+ ModelGenerationValidator validator = new FileModelValidatorImpl();
+ validator.setParameters(params);
+
+ /*
+ * Modelable's write and read the annotated sentences, as well as create and
+ * write the NER models
+ */
+ Modelable modelable = new GenericModelableImpl();
+ modelable.setParameters(params);
+
+ /*
+ * the modelGenerator actually runs the process with a set number of
+ * iterations... could be better by actually calculating the diff between
+ * runs and stopping based on a thresh, but for extrememly large sentence
+ * sets this may be too much.
+ */
+ modelGenerator.build(sentenceProvider, knownEntityProvider, validator, modelable, iterations);
+
+ }
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/KnownEntityProvider.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/KnownEntityProvider.java
index 5135c2b..9354c50 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/KnownEntityProvider.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/KnownEntityProvider.java
@@ -1,48 +1,48 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-
-import java.util.Set;
-
-
-
-/**
- *
-Supplies a list of known entities (a list of names or locations)
- */
-public interface KnownEntityProvider extends ModelParameter<BaseModelBuilderParams> {
-
- /**
- * returns a list of known non ambiguous entities.
- * @return a set of entities
- */
- Set<String> getKnownEntities();
-
- /**
- * adds to the set of known entities. Overriding classes should hold this list in a class level set.
- * @param unambiguousEntity
- */
- void addKnownEntity(String unambiguousEntity);
-
- /**
- * defines the type of entity that the set contains, ie person, location, organization.
- * @return
- */
- String getKnownEntitiesType();
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+
+import java.util.Set;
+
+
+
+/**
+ *
+Supplies a list of known entities (a list of names or locations)
+ */
+public interface KnownEntityProvider extends ModelParameter<BaseModelBuilderParams> {
+
+ /**
+ * returns a list of known non ambiguous entities.
+ * @return a set of entities
+ */
+ Set<String> getKnownEntities();
+
+ /**
+ * adds to the set of known entities. Overriding classes should hold this list in a class level set.
+ * @param unambiguousEntity
+ */
+ void addKnownEntity(String unambiguousEntity);
+
+ /**
+ * defines the type of entity that the set contains, ie person, location, organization.
+ * @return
+ */
+ String getKnownEntitiesType();
+
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelGenerationValidator.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelGenerationValidator.java
index d05a1d5..a66f36f 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelGenerationValidator.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelGenerationValidator.java
@@ -1,35 +1,35 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-
-import java.util.Collection;
-
-/**
- *
-Validates results from the iterative namefinding
- */
-public interface ModelGenerationValidator extends ModelParameter<BaseModelBuilderParams> {
-
- Boolean validSentence(String sentence);
-
- Boolean validNamedEntity(String namedEntity);
-
-
-
- Collection<String> getBlackList();
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+
+import java.util.Collection;
+
+/**
+ *
+Validates results from the iterative namefinding
+ */
+public interface ModelGenerationValidator extends ModelParameter<BaseModelBuilderParams> {
+
+ Boolean validSentence(String sentence);
+
+ Boolean validNamedEntity(String namedEntity);
+
+
+
+ Collection<String> getBlackList();
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelParameter.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelParameter.java
index 136e775..5bc8e15 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelParameter.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelParameter.java
@@ -1,28 +1,28 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-
-/**
- *
- */
-public interface ModelParameter<T extends BaseModelBuilderParams>{
-
- void setParameters(T params);
-
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+
+/**
+ *
+ */
+public interface ModelParameter<T extends BaseModelBuilderParams>{
+
+ void setParameters(T params);
+
+
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/Modelable.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/Modelable.java
index 8d2e06c..36a33a9 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/Modelable.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/Modelable.java
@@ -1,45 +1,45 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import java.util.Set;
-
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-import opennlp.tools.namefind.TokenNameFinderModel;
-
-/**
- *
- */
-public interface Modelable extends ModelParameter<BaseModelBuilderParams> {
-
-
- String annotate(String sentence, String namedEntity, String entityType);
-
- void writeAnnotatedSentences();
-
- Set<String> getAnnotatedSentences();
-
- void setAnnotatedSentences(Set<String> annotatedSentences);
-
- void addAnnotatedSentence(String annotatedSentence);
-
- void buildModel( String entityType);
-
- TokenNameFinderModel getModel();
-
- String[] tokenizeSentenceToWords(String sentence);
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import java.util.Set;
+
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+import opennlp.tools.namefind.TokenNameFinderModel;
+
+/**
+ *
+ */
+public interface Modelable extends ModelParameter<BaseModelBuilderParams> {
+
+
+ String annotate(String sentence, String namedEntity, String entityType);
+
+ void writeAnnotatedSentences();
+
+ Set<String> getAnnotatedSentences();
+
+ void setAnnotatedSentences(Set<String> annotatedSentences);
+
+ void addAnnotatedSentence(String annotatedSentence);
+
+ void buildModel( String entityType);
+
+ TokenNameFinderModel getModel();
+
+ String[] tokenizeSentenceToWords(String sentence);
+
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SemiSupervisedModelGenerator.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SemiSupervisedModelGenerator.java
index c97a4c1..3bb303a 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SemiSupervisedModelGenerator.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SemiSupervisedModelGenerator.java
@@ -1,28 +1,28 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-
-/**
- *
-
- */
-public interface SemiSupervisedModelGenerator extends ModelParameter<BaseModelBuilderParams> {
-
- void build(SentenceProvider sentenceProvider, KnownEntityProvider knownEntityProvider,
- ModelGenerationValidator validator, Modelable modelable, int iterations);
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+
+/**
+ *
+
+ */
+public interface SemiSupervisedModelGenerator extends ModelParameter<BaseModelBuilderParams> {
+
+ void build(SentenceProvider sentenceProvider, KnownEntityProvider knownEntityProvider,
+ ModelGenerationValidator validator, Modelable modelable, int iterations);
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SentenceProvider.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SentenceProvider.java
index 5610224..83af594 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SentenceProvider.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SentenceProvider.java
@@ -1,27 +1,27 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import java.util.Set;
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-
-/**
- *
- */
-public interface SentenceProvider extends ModelParameter<BaseModelBuilderParams> {
-
- Set<String> getSentences();
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import java.util.Set;
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+
+/**
+ *
+ */
+public interface SentenceProvider extends ModelParameter<BaseModelBuilderParams> {
+
+ Set<String> getSentences();
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/BaseModelBuilderParams.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/BaseModelBuilderParams.java
index fcb2384..fff456b 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/BaseModelBuilderParams.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/BaseModelBuilderParams.java
@@ -1,90 +1,90 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder.impls;
-
-import java.io.File;
-import java.util.Map;
-
-/**
- *
- * Used to pass params through the processing
- */
-public class BaseModelBuilderParams {
-
- private File modelFile;
- private File sentenceFile;
- private File knownEntitiesFile;
- private File knownEntityBlacklist;
- private File annotatedTrainingDataFile;
- private String entityType;
- private Map<String, String> additionalParams;
-
- public File getModelFile() {
- return modelFile;
- }
-
- public void setModelFile(File modelFile) {
- this.modelFile = modelFile;
- }
-
- public File getSentenceFile() {
- return sentenceFile;
- }
-
- public void setSentenceFile(File sentenceFile) {
- this.sentenceFile = sentenceFile;
- }
-
- public File getKnownEntitiesFile() {
- return knownEntitiesFile;
- }
-
- public void setKnownEntitiesFile(File knownEntitiesFile) {
- this.knownEntitiesFile = knownEntitiesFile;
- }
-
- public File getKnownEntityBlacklist() {
- return knownEntityBlacklist;
- }
-
- public void setKnownEntityBlacklist(File knownEntityBlacklist) {
- this.knownEntityBlacklist = knownEntityBlacklist;
- }
-
- public Map<String, String> getAdditionalParams() {
- return additionalParams;
- }
-
- public void setAdditionalParams(Map<String, String> additionalParams) {
- this.additionalParams = additionalParams;
- }
-
- public String getEntityType() {
- return entityType;
- }
-
- public void setEntityType(String entityType) {
- this.entityType = entityType;
- }
-
- public File getAnnotatedTrainingDataFile() {
- return annotatedTrainingDataFile;
- }
-
- public void setAnnotatedTrainingDataFile(File annotatedTrainingDataFile) {
- this.annotatedTrainingDataFile = annotatedTrainingDataFile;
- }
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder.impls;
+
+import java.io.File;
+import java.util.Map;
+
+/**
+ *
+ * Used to pass params through the processing
+ */
+public class BaseModelBuilderParams {
+
+ private File modelFile;
+ private File sentenceFile;
+ private File knownEntitiesFile;
+ private File knownEntityBlacklist;
+ private File annotatedTrainingDataFile;
+ private String entityType;
+ private Map<String, String> additionalParams;
+
+ public File getModelFile() {
+ return modelFile;
+ }
+
+ public void setModelFile(File modelFile) {
+ this.modelFile = modelFile;
+ }
+
+ public File getSentenceFile() {
+ return sentenceFile;
+ }
+
+ public void setSentenceFile(File sentenceFile) {
+ this.sentenceFile = sentenceFile;
+ }
+
+ public File getKnownEntitiesFile() {
+ return knownEntitiesFile;
+ }
+
+ public void setKnownEntitiesFile(File knownEntitiesFile) {
+ this.knownEntitiesFile = knownEntitiesFile;
+ }
+
+ public File getKnownEntityBlacklist() {
+ return knownEntityBlacklist;
+ }
+
+ public void setKnownEntityBlacklist(File knownEntityBlacklist) {
+ this.knownEntityBlacklist = knownEntityBlacklist;
+ }
+
+ public Map<String, String> getAdditionalParams() {
+ return additionalParams;
+ }
+
+ public void setAdditionalParams(Map<String, String> additionalParams) {
+ this.additionalParams = additionalParams;
+ }
+
+ public String getEntityType() {
+ return entityType;
+ }
+
+ public void setEntityType(String entityType) {
+ this.entityType = entityType;
+ }
+
+ public File getAnnotatedTrainingDataFile() {
+ return annotatedTrainingDataFile;
+ }
+
+ public void setAnnotatedTrainingDataFile(File annotatedTrainingDataFile) {
+ this.annotatedTrainingDataFile = annotatedTrainingDataFile;
+ }
}
\ No newline at end of file
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileKnownEntityProvider.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileKnownEntityProvider.java
index 66c1c0e..7aa7d0c 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileKnownEntityProvider.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileKnownEntityProvider.java
@@ -1,80 +1,80 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder.impls;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import opennlp.addons.modelbuilder.KnownEntityProvider;
-
-/**
- *
- */
-public class FileKnownEntityProvider implements KnownEntityProvider {
-
- Set<String> knownEntities = new HashSet<>();
- BaseModelBuilderParams params;
- @Override
- public Set<String> getKnownEntities() {
- if (knownEntities.isEmpty()) {
- try {
- InputStream fis;
- BufferedReader br;
- String line;
-
- fis = new FileInputStream(params.getKnownEntitiesFile());
- br = new BufferedReader(new InputStreamReader(fis, Charset.forName("UTF-8")));
- while ((line = br.readLine()) != null) {
- knownEntities.add(line);
- }
-
- // Done with the file
- br.close();
- br = null;
- fis = null;
- } catch (IOException ex) {
- Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- return knownEntities;
- }
-
- @Override
- public void addKnownEntity(String unambiguousEntity) {
- knownEntities.add(unambiguousEntity);
- }
-
- @Override
- public String getKnownEntitiesType() {
-
- return params.getEntityType();
- }
-
-
-
- @Override
- public void setParameters(BaseModelBuilderParams params) {
- this.params = params;
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder.impls;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import opennlp.addons.modelbuilder.KnownEntityProvider;
+
+/**
+ *
+ */
+public class FileKnownEntityProvider implements KnownEntityProvider {
+
+ Set<String> knownEntities = new HashSet<>();
+ BaseModelBuilderParams params;
+ @Override
+ public Set<String> getKnownEntities() {
+ if (knownEntities.isEmpty()) {
+ try {
+ InputStream fis;
+ BufferedReader br;
+ String line;
+
+ fis = new FileInputStream(params.getKnownEntitiesFile());
+ br = new BufferedReader(new InputStreamReader(fis, Charset.forName("UTF-8")));
+ while ((line = br.readLine()) != null) {
+ knownEntities.add(line);
+ }
+
+ // Done with the file
+ br.close();
+ br = null;
+ fis = null;
+ } catch (IOException ex) {
+ Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ }
+ return knownEntities;
+ }
+
+ @Override
+ public void addKnownEntity(String unambiguousEntity) {
+ knownEntities.add(unambiguousEntity);
+ }
+
+ @Override
+ public String getKnownEntitiesType() {
+
+ return params.getEntityType();
+ }
+
+
+
+ @Override
+ public void setParameters(BaseModelBuilderParams params) {
+ this.params = params;
+ }
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileModelValidatorImpl.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileModelValidatorImpl.java
index 3951eef..a953c76 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileModelValidatorImpl.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileModelValidatorImpl.java
@@ -1,86 +1,86 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder.impls;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import opennlp.addons.modelbuilder.ModelGenerationValidator;
-
-/**
- *Validates NER results input before inclusion into the model
- */
-public class FileModelValidatorImpl implements ModelGenerationValidator {
-
- private final Set<String> badentities = new HashSet<>();
- BaseModelBuilderParams params;
-
- @Override
- public void setParameters(BaseModelBuilderParams params) {
- this.params = params;
- }
-
- @Override
- public Boolean validSentence(String sentence) {
- //returning true by default, because the sentence provider will return only "valid" sentences in this case
- return true;
- }
-
- @Override
- public Boolean validNamedEntity(String namedEntity) {
-
- if (badentities.isEmpty()) {
- getBlackList();
- }
-//
-// Pattern p = Pattern.compile("[0-9]", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
-// if (p.matcher(namedEntity).find()) {
-// return false;
-// }
- boolean b = true;
- if (badentities.contains(namedEntity.toLowerCase())) {
- b = false;
- }
- return b;
- }
-
- @Override
- public Collection<String> getBlackList() {
- if (params.getKnownEntityBlacklist() == null) {
- return badentities;
- }
- if (!badentities.isEmpty()) {
- try (BufferedReader br = new BufferedReader(new InputStreamReader(
- new FileInputStream(params.getKnownEntityBlacklist()), StandardCharsets.UTF_8))){
- String line;
- while ((line = br.readLine()) != null) {
- badentities.add(line);
- }
- } catch (IOException ex) {
- Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- return badentities;
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder.impls;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import opennlp.addons.modelbuilder.ModelGenerationValidator;
+
+/**
+ *Validates NER results input before inclusion into the model
+ */
+public class FileModelValidatorImpl implements ModelGenerationValidator {
+
+ private final Set<String> badentities = new HashSet<>();
+ BaseModelBuilderParams params;
+
+ @Override
+ public void setParameters(BaseModelBuilderParams params) {
+ this.params = params;
+ }
+
+ @Override
+ public Boolean validSentence(String sentence) {
+ //returning true by default, because the sentence provider will return only "valid" sentences in this case
+ return true;
+ }
+
+ @Override
+ public Boolean validNamedEntity(String namedEntity) {
+
+ if (badentities.isEmpty()) {
+ getBlackList();
+ }
+//
+// Pattern p = Pattern.compile("[0-9]", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
+// if (p.matcher(namedEntity).find()) {
+// return false;
+// }
+ boolean b = true;
+ if (badentities.contains(namedEntity.toLowerCase())) {
+ b = false;
+ }
+ return b;
+ }
+
+ @Override
+ public Collection<String> getBlackList() {
+ if (params.getKnownEntityBlacklist() == null) {
+ return badentities;
+ }
+ if (!badentities.isEmpty()) {
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(
+ new FileInputStream(params.getKnownEntityBlacklist()), StandardCharsets.UTF_8))){
+ String line;
+ while ((line = br.readLine()) != null) {
+ badentities.add(line);
+ }
+ } catch (IOException ex) {
+ Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ }
+ return badentities;
+ }
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileSentenceProvider.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileSentenceProvider.java
index 9f1f5e1..cb6715f 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileSentenceProvider.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileSentenceProvider.java
@@ -1,57 +1,57 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder.impls;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import opennlp.addons.modelbuilder.SentenceProvider;
-
-/**
- * Provides user sentences via a simple text file
- */
-public class FileSentenceProvider implements SentenceProvider {
-
- BaseModelBuilderParams params;
- private final Set<String> sentences = new HashSet<>();
-
- @Override
- public Set<String> getSentences() {
- if (sentences.isEmpty()) {
- try (BufferedReader br = new BufferedReader(new InputStreamReader(
- new FileInputStream(params.getSentenceFile()), StandardCharsets.UTF_8))){
- String line;
- while ((line = br.readLine()) != null) {
- sentences.add(line);
- }
- } catch (IOException ex) {
- Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- return sentences;
- }
-
- public void setParameters(BaseModelBuilderParams params) {
- this.params = params;
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder.impls;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import opennlp.addons.modelbuilder.SentenceProvider;
+
+/**
+ * Provides user sentences via a simple text file
+ */
+public class FileSentenceProvider implements SentenceProvider {
+
+ BaseModelBuilderParams params;
+ private final Set<String> sentences = new HashSet<>();
+
+ @Override
+ public Set<String> getSentences() {
+ if (sentences.isEmpty()) {
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(
+ new FileInputStream(params.getSentenceFile()), StandardCharsets.UTF_8))){
+ String line;
+ while ((line = br.readLine()) != null) {
+ sentences.add(line);
+ }
+ } catch (IOException ex) {
+ Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ }
+ return sentences;
+ }
+
+ public void setParameters(BaseModelBuilderParams params) {
+ this.params = params;
+ }
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelGenerator.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelGenerator.java
index d09a664..126157f 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelGenerator.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelGenerator.java
@@ -1,103 +1,103 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder.impls;
-
-import java.util.HashMap;
-import java.util.Map;
-import opennlp.addons.modelbuilder.KnownEntityProvider;
-import opennlp.addons.modelbuilder.ModelGenerationValidator;
-import opennlp.addons.modelbuilder.Modelable;
-import opennlp.addons.modelbuilder.SemiSupervisedModelGenerator;
-import opennlp.addons.modelbuilder.SentenceProvider;
-import opennlp.tools.namefind.NameFinderME;
-import opennlp.tools.util.Span;
-
-/**
- *
- * Generic impl that handles all processing using the default file implementations
- */
-public class GenericModelGenerator implements SemiSupervisedModelGenerator {
-
- private Map<String, String> params = new HashMap<>();
-
- @Override
- public void setParameters(BaseModelBuilderParams params) {
- this.params = params.getAdditionalParams();
- }
-
- @Override
- public void build(SentenceProvider sentenceProvider, KnownEntityProvider knownEntityProvider,
- ModelGenerationValidator validator, Modelable modelable, int iterations) {
- for (int iteration = 0; iteration < iterations; iteration++) {
- System.out.println("ITERATION: " + iteration);
- System.out.println("\tPerfoming Known Entity Annotation");
- System.out.println("\t\tknowns: " + knownEntityProvider.getKnownEntities().size());
- System.out.println("\t\treading data....: ");
- for (String sentence : sentenceProvider.getSentences()) {
- for (String knownEntity : knownEntityProvider.getKnownEntities()) {
- if (sentence.contains(knownEntity)) {
- //if the same sentence has multiple hits should they be annotated separately?
- modelable.addAnnotatedSentence(modelable.annotate(sentence, knownEntity, knownEntityProvider.getKnownEntitiesType()));
- }
- }
- }
- if (sentenceProvider.getSentences().isEmpty()) {
- System.out.println("No sentences in file");
- return;
- }
- if (knownEntityProvider.getKnownEntities().isEmpty()) {
- System.out.println("No known entities in file");
- return;
- }
- System.out.println("\t\twriting annotated sentences....: ");
- modelable.writeAnnotatedSentences();
- System.out.println("\t\tbuilding model.... ");
- modelable.buildModel(knownEntityProvider.getKnownEntitiesType());
- System.out.println("\t\tmodel building complete.... ");
- NameFinderME nf = new NameFinderME(modelable.getModel());
- System.out.println("\t\tannotated sentences: " + modelable.getAnnotatedSentences().size());
- System.out.println("\tPerforming NER with new model");
- System.out.println("\t\tPrinting NER Results. Add undesired results to the blacklist file and start over");
- for (String sentence : sentenceProvider.getSentences()) {
- if (!validator.validSentence(sentence)) {
- continue;
- }
- String[] tokens = modelable.tokenizeSentenceToWords(sentence);
-
- Span[] find = nf.find(tokens);
- nf.clearAdaptiveData();
-
- String[] namedEntities = Span.spansToStrings(find, tokens);
-
- for (String namedEntity : namedEntities) {
- System.out.println("\t\t" + namedEntity);
- if (validator.validNamedEntity(namedEntity)) {
-
- knownEntityProvider.addKnownEntity(namedEntity);
- modelable.addAnnotatedSentence(modelable.annotate(sentence, namedEntity, knownEntityProvider.getKnownEntitiesType()));
-
- } else {
- System.out.println("\t\t" + namedEntity + "...already blacklisted");
- }
- }
- }
- System.out.println("\t\tannotated sentences: " + modelable.getAnnotatedSentences().size());
- System.out.println("\t\tknowns: " + knownEntityProvider.getKnownEntities().size());
- }
- modelable.writeAnnotatedSentences();
- modelable.buildModel(knownEntityProvider.getKnownEntitiesType());
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder.impls;
+
+import java.util.HashMap;
+import java.util.Map;
+import opennlp.addons.modelbuilder.KnownEntityProvider;
+import opennlp.addons.modelbuilder.ModelGenerationValidator;
+import opennlp.addons.modelbuilder.Modelable;
+import opennlp.addons.modelbuilder.SemiSupervisedModelGenerator;
+import opennlp.addons.modelbuilder.SentenceProvider;
+import opennlp.tools.namefind.NameFinderME;
+import opennlp.tools.util.Span;
+
+/**
+ *
+ * Generic impl that handles all processing using the default file implementations
+ */
+public class GenericModelGenerator implements SemiSupervisedModelGenerator {
+
+ private Map<String, String> params = new HashMap<>();
+
+ @Override
+ public void setParameters(BaseModelBuilderParams params) {
+ this.params = params.getAdditionalParams();
+ }
+
+ @Override
+ public void build(SentenceProvider sentenceProvider, KnownEntityProvider knownEntityProvider,
+ ModelGenerationValidator validator, Modelable modelable, int iterations) {
+ for (int iteration = 0; iteration < iterations; iteration++) {
+ System.out.println("ITERATION: " + iteration);
+ System.out.println("\tPerfoming Known Entity Annotation");
+ System.out.println("\t\tknowns: " + knownEntityProvider.getKnownEntities().size());
+ System.out.println("\t\treading data....: ");
+ for (String sentence : sentenceProvider.getSentences()) {
+ for (String knownEntity : knownEntityProvider.getKnownEntities()) {
+ if (sentence.contains(knownEntity)) {
+ //if the same sentence has multiple hits should they be annotated separately?
+ modelable.addAnnotatedSentence(modelable.annotate(sentence, knownEntity, knownEntityProvider.getKnownEntitiesType()));
+ }
+ }
+ }
+ if (sentenceProvider.getSentences().isEmpty()) {
+ System.out.println("No sentences in file");
+ return;
+ }
+ if (knownEntityProvider.getKnownEntities().isEmpty()) {
+ System.out.println("No known entities in file");
+ return;
+ }
+ System.out.println("\t\twriting annotated sentences....: ");
+ modelable.writeAnnotatedSentences();
+ System.out.println("\t\tbuilding model.... ");
+ modelable.buildModel(knownEntityProvider.getKnownEntitiesType());
+ System.out.println("\t\tmodel building complete.... ");
+ NameFinderME nf = new NameFinderME(modelable.getModel());
+ System.out.println("\t\tannotated sentences: " + modelable.getAnnotatedSentences().size());
+ System.out.println("\tPerforming NER with new model");
+ System.out.println("\t\tPrinting NER Results. Add undesired results to the blacklist file and start over");
+ for (String sentence : sentenceProvider.getSentences()) {
+ if (!validator.validSentence(sentence)) {
+ continue;
+ }
+ String[] tokens = modelable.tokenizeSentenceToWords(sentence);
+
+ Span[] find = nf.find(tokens);
+ nf.clearAdaptiveData();
+
+ String[] namedEntities = Span.spansToStrings(find, tokens);
+
+ for (String namedEntity : namedEntities) {
+ System.out.println("\t\t" + namedEntity);
+ if (validator.validNamedEntity(namedEntity)) {
+
+ knownEntityProvider.addKnownEntity(namedEntity);
+ modelable.addAnnotatedSentence(modelable.annotate(sentence, namedEntity, knownEntityProvider.getKnownEntitiesType()));
+
+ } else {
+ System.out.println("\t\t" + namedEntity + "...already blacklisted");
+ }
+ }
+ }
+ System.out.println("\t\tannotated sentences: " + modelable.getAnnotatedSentences().size());
+ System.out.println("\t\tknowns: " + knownEntityProvider.getKnownEntities().size());
+ }
+ modelable.writeAnnotatedSentences();
+ modelable.buildModel(knownEntityProvider.getKnownEntitiesType());
+ }
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelableImpl.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelableImpl.java
index 8c1f754..2630bc4 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelableImpl.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelableImpl.java
@@ -1,121 +1,121 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder.impls;
-
-import java.io.BufferedOutputStream;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import opennlp.addons.modelbuilder.Modelable;
-import opennlp.tools.namefind.TokenNameFinderFactory;
-import opennlp.tools.util.MarkableFileInputStreamFactory;
-
-import opennlp.tools.namefind.NameFinderME;
-import opennlp.tools.namefind.NameSample;
-import opennlp.tools.namefind.NameSampleDataStream;
-import opennlp.tools.namefind.TokenNameFinderModel;
-
-import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
-import opennlp.tools.util.TrainingParameters;
-
-/**
- * Creates annotations, writes annotations to file, and creates a model and writes to a file
- */
-public class GenericModelableImpl implements Modelable {
-
- private Set<String> annotatedSentences = new HashSet<>();
- BaseModelBuilderParams params;
-
- @Override
- public void setParameters(BaseModelBuilderParams params) {
- this.params = params;
- }
-
- @Override
- public String annotate(String sentence, String namedEntity, String entityType) {
- return sentence.replace(namedEntity, " <START:" + entityType + "> " + namedEntity + " <END> ");
- }
-
- @Override
- public void writeAnnotatedSentences() {
- try (Writer bw = new BufferedWriter(new FileWriter(params.getAnnotatedTrainingDataFile(), false))) {
- for (String s : annotatedSentences) {
- bw.write(s.replace("\n", " ").trim() + "\n");
- }
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- @Override
- public Set<String> getAnnotatedSentences() {
- return annotatedSentences;
- }
-
- @Override
- public void setAnnotatedSentences(Set<String> annotatedSentences) {
- this.annotatedSentences = annotatedSentences;
- }
-
- @Override
- public void addAnnotatedSentence(String annotatedSentence) {
- annotatedSentences.add(annotatedSentence);
- }
-
- @Override
- public void buildModel(String entityType) {
- try (ObjectStream<NameSample> sampleStream = new NameSampleDataStream(new PlainTextByLineStream(
- new MarkableFileInputStreamFactory(params.getAnnotatedTrainingDataFile()), StandardCharsets.UTF_8));
- OutputStream modelOut = new BufferedOutputStream(new FileOutputStream(params.getModelFile()))) {
- System.out.println("\tBuilding Model using " + annotatedSentences.size() + " annotations");
- System.out.println("\t\treading training data...");
- TokenNameFinderModel model;
- model = NameFinderME.train("en", entityType, sampleStream,
- TrainingParameters.defaultParams(), new TokenNameFinderFactory());
- sampleStream.close();
- model.serialize(modelOut);
- System.out.println("\tmodel generated");
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public TokenNameFinderModel getModel() {
- TokenNameFinderModel nerModel = null;
- try {
- nerModel = new TokenNameFinderModel(new FileInputStream(params.getModelFile()));
- } catch (IOException ex) {
- Logger.getLogger(GenericModelableImpl.class.getName()).log(Level.SEVERE, null, ex);
- }
- return nerModel;
- }
-
- @Override
- public String[] tokenizeSentenceToWords(String sentence) {
- return sentence.split("\\s+");
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder.impls;
+
+import java.io.BufferedOutputStream;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import opennlp.addons.modelbuilder.Modelable;
+import opennlp.tools.namefind.TokenNameFinderFactory;
+import opennlp.tools.util.MarkableFileInputStreamFactory;
+
+import opennlp.tools.namefind.NameFinderME;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.namefind.NameSampleDataStream;
+import opennlp.tools.namefind.TokenNameFinderModel;
+
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.PlainTextByLineStream;
+import opennlp.tools.util.TrainingParameters;
+
+/**
+ * Creates annotations, writes annotations to file, and creates a model and writes to a file
+ */
+public class GenericModelableImpl implements Modelable {
+
+ private Set<String> annotatedSentences = new HashSet<>();
+ BaseModelBuilderParams params;
+
+ @Override
+ public void setParameters(BaseModelBuilderParams params) {
+ this.params = params;
+ }
+
+ @Override
+ public String annotate(String sentence, String namedEntity, String entityType) {
+ return sentence.replace(namedEntity, " <START:" + entityType + "> " + namedEntity + " <END> ");
+ }
+
+ @Override
+ public void writeAnnotatedSentences() {
+ try (Writer bw = new BufferedWriter(new FileWriter(params.getAnnotatedTrainingDataFile(), false))) {
+ for (String s : annotatedSentences) {
+ bw.write(s.replace("\n", " ").trim() + "\n");
+ }
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ @Override
+ public Set<String> getAnnotatedSentences() {
+ return annotatedSentences;
+ }
+
+ @Override
+ public void setAnnotatedSentences(Set<String> annotatedSentences) {
+ this.annotatedSentences = annotatedSentences;
+ }
+
+ @Override
+ public void addAnnotatedSentence(String annotatedSentence) {
+ annotatedSentences.add(annotatedSentence);
+ }
+
+ @Override
+ public void buildModel(String entityType) {
+ try (ObjectStream<NameSample> sampleStream = new NameSampleDataStream(new PlainTextByLineStream(
+ new MarkableFileInputStreamFactory(params.getAnnotatedTrainingDataFile()), StandardCharsets.UTF_8));
+ OutputStream modelOut = new BufferedOutputStream(new FileOutputStream(params.getModelFile()))) {
+ System.out.println("\tBuilding Model using " + annotatedSentences.size() + " annotations");
+ System.out.println("\t\treading training data...");
+ TokenNameFinderModel model;
+ model = NameFinderME.train("en", entityType, sampleStream,
+ TrainingParameters.defaultParams(), new TokenNameFinderFactory());
+ sampleStream.close();
+ model.serialize(modelOut);
+ System.out.println("\tmodel generated");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public TokenNameFinderModel getModel() {
+ TokenNameFinderModel nerModel = null;
+ try {
+ nerModel = new TokenNameFinderModel(new FileInputStream(params.getModelFile()));
+ } catch (IOException ex) {
+ Logger.getLogger(GenericModelableImpl.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ return nerModel;
+ }
+
+ @Override
+ public String[] tokenizeSentenceToWords(String sentence) {
+ return sentence.split("\\s+");
+ }
+}
diff --git a/morfologik-addon/pom.xml b/morfologik-addon/pom.xml
index 92b6f72..87928ff 100644
--- a/morfologik-addon/pom.xml
+++ b/morfologik-addon/pom.xml
@@ -1,152 +1,152 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-addons</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>morfologik-addon</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Apache OpenNLP Morfologik Addon</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-tools</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.carrot2</groupId>
- <artifactId>morfologik-stemming</artifactId>
- <version>2.1.9</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.carrot2</groupId>
- <artifactId>morfologik-tools</artifactId>
- <version>2.1.9</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-engine</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-params</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>${maven.compiler.source}</source>
- <target>${maven.compiler.target}</target>
- <compilerArgument>-Xlint</compilerArgument>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>bundle-project-sources</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <descriptors>
- <descriptor>src/main/assembly/bin.xml</descriptor>
- <descriptor>src/main/assembly/src.xml</descriptor>
- </descriptors>
- <!-- Tar package is only compatible with gnu tar,
- many file have more than 100 chars.
- Right now only javadoc files are too long.
- -->
- <tarLongFileMode>gnu</tarLongFileMode>
-
- <finalName>apache-opennlp-morfologik-addon-${project.version}</finalName>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.6</version>
- <executions>
- <execution>
- <id>generate checksums for binary artifacts</id>
- <goals><goal>run</goal></goals>
- <phase>verify</phase>
- <configuration>
- <target>
- <checksum algorithm="sha1" format="MD5SUM">
- <fileset dir="${project.build.directory}">
- <include name="*.zip" />
- <include name="*.gz" />
- </fileset>
- </checksum>
- <checksum algorithm="md5" format="MD5SUM">
- <fileset dir="${project.build.directory}">
- <include name="*.zip" />
- <include name="*.gz" />
- </fileset>
- </checksum>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
... 142 lines suppressed ...