You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2023/05/01 17:11:48 UTC
[opennlp-addons] branch master updated (7490add -> 13e985c)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/opennlp-addons.git
from 7490add Bump junit from 4.12 to 4.13.1 in /japanese-addon (#11)
new 618eb9a adjusts 'opennlp-addons' to be compatible with latest opennlp-tools release
new 13e985c Normalize line endings (#13)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitattributes | 48 ++
.github/CONTRIBUTING.md | 11 +
.github/PULL_REQUEST_TEMPLATE.md | 27 +
.github/workflows/maven.yml | 51 ++
.gitignore | 2 +
NOTICE | 2 +-
geoentitylinker-addon/pom.xml | 187 +++--
.../addons/geoentitylinker/AdminBoundary.java | 276 +++----
.../geoentitylinker/AdminBoundaryContext.java | 306 ++++----
.../AdminBoundaryContextGenerator.java | 834 ++++++++++-----------
.../geoentitylinker/CountryContextEntry.java | 236 +++---
.../addons/geoentitylinker/GazetteerEntry.java | 355 +++++----
.../geoentitylinker/GazetteerSearchCache.java | 98 +--
.../addons/geoentitylinker/GazetteerSearcher.java | 513 +++++++------
.../addons/geoentitylinker/GeoEntityLinker.java | 443 ++++++-----
.../geoentitylinker/indexing/GazetteerIndexer.java | 416 +++++-----
.../indexing/GeonamesFileDownloader.java | 259 +++----
.../indexing/GeonamesProcessor.java | 592 +++++++--------
.../geoentitylinker/indexing/RegionProcessor.java | 230 +++---
.../geoentitylinker/indexing/USGSProcessor.java | 503 ++++++-------
.../scoring/CountryProximityScorer.java | 558 +++++++-------
.../scoring/FuzzyStringMatchScorer.java | 247 +++---
.../scoring/GeoHashBinningScorer.java | 123 ++-
.../scoring/LinkedEntityScorer.java | 80 +-
.../geoentitylinker/scoring/ModelBasedScorer.java | 317 ++++----
.../geoentitylinker/scoring/PlacetypeScorer.java | 164 ++--
.../geoentitylinker/scoring/PointClustering.java | 244 +++---
.../scoring/ProvinceProximityScorer.java | 588 +++++++--------
.../test/java/apache/opennlp/addons/AppTest.java | 38 -
japanese-addon/build.xml | 2 +-
japanese-addon/pom.xml | 89 ++-
.../AuxiliaryInfoNameContextGenerator.java | 2 +-
.../lang/jpn/BrownTokenClassFeatureGenerator.java | 4 +-
.../tools/namefind/AuxiliaryInfoUtilTest.java | 53 +-
...liaryInfoAwareDelegateFeatureGeneratorTest.java | 17 +-
.../lang/jpn/BigramNameFeatureGeneratorTest.java | 36 +-
.../lang/jpn/FeatureGeneratorUtilTest.java | 59 +-
.../lang/jpn/TokenClassFeatureGeneratorTest.java | 21 +-
.../lang/jpn/TokenPatternFeatureGeneratorTest.java | 45 +-
.../lang/jpn/TrigramNameFeatureGeneratorTest.java | 41 +-
jwnl-addon/pom.xml | 128 ++--
.../opennlp/jwnl/lemmatizer/JWNLLemmatizer.java | 35 +-
liblinear-addon/pom.xml | 41 +-
liblinear-addon/src/main/java/LiblinearModel.java | 135 ++--
.../src/main/java/LiblinearTrainer.java | 86 +--
modelbuilder-addon/pom.xml | 107 ++-
.../modelbuilder/DefaultModelBuilderUtil.java | 238 +++---
.../addons/modelbuilder/KnownEntityProvider.java | 93 +--
.../modelbuilder/ModelGenerationValidator.java | 68 +-
.../addons/modelbuilder/ModelParameter.java | 56 +-
.../opennlp/addons/modelbuilder/Modelable.java | 90 +--
.../modelbuilder/SemiSupervisedModelGenerator.java | 56 +-
.../addons/modelbuilder/SentenceProvider.java | 54 +-
.../modelbuilder/impls/BaseModelBuilderParams.java | 178 ++---
.../impls/FileKnownEntityProvider.java | 162 ++--
.../modelbuilder/impls/FileModelValidatorImpl.java | 182 +++--
.../modelbuilder/impls/FileSentenceProvider.java | 127 ++--
.../modelbuilder/impls/GenericModelGenerator.java | 206 ++---
.../modelbuilder/impls/GenericModelableImpl.java | 253 +++----
.../src/test/java/modelbuilder/AppTest.java | 38 -
morfologik-addon/pom.xml | 261 ++++---
.../main/java/opennlp/morfologik/cmdline/CLI.java | 4 +-
.../cmdline/builder/XMLDictionaryToTableTool.java | 4 +-
.../lemmatizer/MorfologikLemmatizer.java | 36 +-
.../tagdict/MorfologikPOSTaggerFactory.java | 36 +-
.../tagdict/MorfologikTagDictionary.java | 26 +-
.../builder/POSDictionayBuilderTest.java | 8 +-
.../lemmatizer/MorfologikLemmatizerTest.java | 36 +-
.../tagdict/MorfologikTagDictionaryTest.java | 33 +-
.../morfologik/tagdict/POSTaggerFactoryTest.java | 57 +-
pom.xml | 532 +++++++++++++
71 files changed, 6079 insertions(+), 5404 deletions(-)
create mode 100644 .gitattributes
create mode 100644 .github/CONTRIBUTING.md
create mode 100644 .github/PULL_REQUEST_TEMPLATE.md
create mode 100644 .github/workflows/maven.yml
delete mode 100644 geoentitylinker-addon/src/test/java/apache/opennlp/addons/AppTest.java
delete mode 100644 modelbuilder-addon/src/test/java/modelbuilder/AppTest.java
create mode 100644 pom.xml
[opennlp-addons] 02/02: Normalize line endings (#13)
Posted by rz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/opennlp-addons.git
commit 13e985c893eb2b8aab66985482f6eeaac452c7df
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Mon May 1 18:40:10 2023 +0200
Normalize line endings (#13)
---
.gitattributes | 48 ++
geoentitylinker-addon/pom.xml | 186 ++---
.../addons/geoentitylinker/AdminBoundary.java | 276 +++----
.../geoentitylinker/AdminBoundaryContext.java | 306 ++++----
.../AdminBoundaryContextGenerator.java | 824 ++++++++++-----------
.../geoentitylinker/CountryContextEntry.java | 236 +++---
.../addons/geoentitylinker/GazetteerEntry.java | 346 ++++-----
.../geoentitylinker/GazetteerSearchCache.java | 98 +--
.../addons/geoentitylinker/GazetteerSearcher.java | 502 ++++++-------
.../addons/geoentitylinker/GeoEntityLinker.java | 440 +++++------
.../geoentitylinker/indexing/GazetteerIndexer.java | 420 +++++------
.../indexing/GeonamesFileDownloader.java | 240 +++---
.../indexing/GeonamesProcessor.java | 592 +++++++--------
.../geoentitylinker/indexing/RegionProcessor.java | 230 +++---
.../geoentitylinker/indexing/USGSProcessor.java | 500 ++++++-------
.../scoring/CountryProximityScorer.java | 554 +++++++-------
.../scoring/FuzzyStringMatchScorer.java | 248 +++----
.../scoring/GeoHashBinningScorer.java | 122 +--
.../scoring/LinkedEntityScorer.java | 80 +-
.../geoentitylinker/scoring/ModelBasedScorer.java | 314 ++++----
.../geoentitylinker/scoring/PlacetypeScorer.java | 164 ++--
.../geoentitylinker/scoring/PointClustering.java | 242 +++---
.../scoring/ProvinceProximityScorer.java | 588 +++++++--------
jwnl-addon/pom.xml | 156 ++--
modelbuilder-addon/pom.xml | 144 ++--
.../modelbuilder/DefaultModelBuilderUtil.java | 242 +++---
.../addons/modelbuilder/KnownEntityProvider.java | 96 +--
.../modelbuilder/ModelGenerationValidator.java | 70 +-
.../addons/modelbuilder/ModelParameter.java | 56 +-
.../opennlp/addons/modelbuilder/Modelable.java | 90 +--
.../modelbuilder/SemiSupervisedModelGenerator.java | 56 +-
.../addons/modelbuilder/SentenceProvider.java | 54 +-
.../modelbuilder/impls/BaseModelBuilderParams.java | 178 ++---
.../impls/FileKnownEntityProvider.java | 160 ++--
.../modelbuilder/impls/FileModelValidatorImpl.java | 172 ++---
.../modelbuilder/impls/FileSentenceProvider.java | 114 +--
.../modelbuilder/impls/GenericModelGenerator.java | 206 +++---
.../modelbuilder/impls/GenericModelableImpl.java | 242 +++---
morfologik-addon/pom.xml | 304 ++++----
39 files changed, 4972 insertions(+), 4924 deletions(-)
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..39bfc13
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,48 @@
+# Handle line endings automatically for files detected as text
+# and leave all files detected as binary untouched.
+* text=auto
+
+#
+# The above will handle all files NOT found below
+#
+# These files are text and should be normalized (Convert crlf => lf)
+*.adoc text eol=lf
+*.html text eol=lf
+*.java text eol=lf
+*.jspf text eol=lf
+*.md text eol=lf
+*.properties text eol=lf
+*.sh text eol=lf
+*.txt text eol=lf
+*.xml text eol=lf
+*.xsd text eol=lf
+*.xsl text eol=lf
+*.yml text eol=lf
+
+LICENSE text eol=lf
+NOTICE text eol=lf
+
+# These files are binary and should be left untouched
+# (binary is a macro for -text -diff)
+*.class binary
+*.dll binary
+*.ear binary
+*.gif binary
+*.ico binary
+*.jar binary
+*.jpg binary
+*.jpeg binary
+*.png binary
+*.ser binary
+*.so binary
+*.war binary
+*.zip binary
+*.exe binary
+*.gz binary
+
+#Windows
+*.bat text eol=crlf
+*.cmd text eol=crlf
+
+#Unix/Linux
+*.sh text eol=lf
\ No newline at end of file
diff --git a/geoentitylinker-addon/pom.xml b/geoentitylinker-addon/pom.xml
index 75be328..6a08a11 100644
--- a/geoentitylinker-addon/pom.xml
+++ b/geoentitylinker-addon/pom.xml
@@ -1,93 +1,93 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-addons</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>geoentitylinker-addon</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Apache OpenNLP GeoentityLinker Addon</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-tools</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>6.0.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers-common</artifactId>
- <version>6.0.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-queryparser</artifactId>
- <version>6.0.0</version>
- </dependency>
- <dependency>
- <groupId>com.spatial4j</groupId>
- <artifactId>spatial4j</artifactId>
- <version>0.4.1</version>
- <type>jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-engine</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-params</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>${maven.compiler.source}</source>
- <target>${maven.compiler.target}</target>
- <compilerArgument>-Xlint</compilerArgument>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>geoentitylinker-addon</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <name>Apache OpenNLP GeoentityLinker Addon</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-tools</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>6.0.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>6.0.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ <version>6.0.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.spatial4j</groupId>
+ <artifactId>spatial4j</artifactId>
+ <version>0.4.1</version>
+ <type>jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerArgument>-Xlint</compilerArgument>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundary.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundary.java
index a01b0bb..15859c1 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundary.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundary.java
@@ -1,138 +1,138 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.util.Objects;
-
-/**
- * Stores an admin boundary down to the US county level. Only US places from the
- * USGS Gazetteer will have county level info
- */
-public class AdminBoundary {
-
- private static final String NO_DATA_FOUND_VALUE = "NO_DATA_FOUND";
- private final String countryCode;
- private final String provinceCode;
- private final String provinceName;
- private final String countryName;
- private final String countyName;
- private final String countyCode;
- private final String countryRegex;
- private final String provinceRegex;
- private final String countyRegex;
-
-
- public AdminBoundary(String countryCode, String countryName, String provinceCode, String provinceName, String countyCode, String countyName,
- String countryRegex, String provinceRegex, String countyRegex) {
- this.countryCode = countryCode;
- this.provinceCode = provinceCode;
- this.provinceName = provinceName;
- this.countryName = countryName;
- this.countyName = countyName.equals("") ? NO_DATA_FOUND_VALUE : countyName;
- this.countyCode = countyCode.equals("") ? NO_DATA_FOUND_VALUE : countyCode;
- this.countryRegex = countryRegex;
- this.provinceRegex = provinceRegex;
- this.countyRegex = countyRegex;
- }
-
- public String getCountryCode() {
- return countryCode;
- }
-
- public String getProvCode() {
- return provinceCode;
- }
-
- public String getProvinceName() {
- return provinceName;
- }
-
- public String getCountryName() {
- return countryName;
- }
-
- public String getCountyName() {
- return countyName;
- }
-
- public String getCountyCode() {
- return countyCode;
- }
-
- @Override
- public String toString() {
- return "AdminBoundary{" + "countryCode=" + countryCode + ", provinceCode=" + provinceCode + ", provinceName=" + provinceName + ", countryName=" + countryName + ", countyName=" + countyName + ", countyCode=" + countyCode + '}';
- }
-
- @Override
- public int hashCode() {
- int hash = 7;
- hash = 11 * hash + Objects.hashCode(this.countryCode);
- hash = 11 * hash + Objects.hashCode(this.provinceCode);
- hash = 11 * hash + Objects.hashCode(this.provinceName);
- hash = 11 * hash + Objects.hashCode(this.countryName);
- hash = 11 * hash + Objects.hashCode(this.countyName);
- hash = 11 * hash + Objects.hashCode(this.countyCode);
- return hash;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- final AdminBoundary other = (AdminBoundary) obj;
- if (!Objects.equals(this.countryCode, other.countryCode)) {
- return false;
- }
- if (!Objects.equals(this.provinceCode, other.provinceCode)) {
- return false;
- }
- if (!Objects.equals(this.provinceName, other.provinceName)) {
- return false;
- }
- if (!Objects.equals(this.countryName, other.countryName)) {
- return false;
- }
- if (!Objects.equals(this.countyName, other.countyName)) {
- return false;
- }
- if (!Objects.equals(this.countyCode, other.countyCode)) {
- return false;
- }
- return true;
- }
-
- public String getProvinceCode() {
- return provinceCode;
- }
-
- public String getCountryRegex() {
- return countryRegex;
- }
-
- public String getProvinceRegex() {
- return provinceRegex;
- }
-
- public String getCountyRegex() {
- return countyRegex;
- }
-
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.util.Objects;
+
+/**
+ * Stores an admin boundary down to the US county level. Only US places from the
+ * USGS Gazetteer will have county level info
+ */
+public class AdminBoundary {
+
+ private static final String NO_DATA_FOUND_VALUE = "NO_DATA_FOUND";
+ private final String countryCode;
+ private final String provinceCode;
+ private final String provinceName;
+ private final String countryName;
+ private final String countyName;
+ private final String countyCode;
+ private final String countryRegex;
+ private final String provinceRegex;
+ private final String countyRegex;
+
+
+ public AdminBoundary(String countryCode, String countryName, String provinceCode, String provinceName, String countyCode, String countyName,
+ String countryRegex, String provinceRegex, String countyRegex) {
+ this.countryCode = countryCode;
+ this.provinceCode = provinceCode;
+ this.provinceName = provinceName;
+ this.countryName = countryName;
+ this.countyName = countyName.equals("") ? NO_DATA_FOUND_VALUE : countyName;
+ this.countyCode = countyCode.equals("") ? NO_DATA_FOUND_VALUE : countyCode;
+ this.countryRegex = countryRegex;
+ this.provinceRegex = provinceRegex;
+ this.countyRegex = countyRegex;
+ }
+
+ public String getCountryCode() {
+ return countryCode;
+ }
+
+ public String getProvCode() {
+ return provinceCode;
+ }
+
+ public String getProvinceName() {
+ return provinceName;
+ }
+
+ public String getCountryName() {
+ return countryName;
+ }
+
+ public String getCountyName() {
+ return countyName;
+ }
+
+ public String getCountyCode() {
+ return countyCode;
+ }
+
+ @Override
+ public String toString() {
+ return "AdminBoundary{" + "countryCode=" + countryCode + ", provinceCode=" + provinceCode + ", provinceName=" + provinceName + ", countryName=" + countryName + ", countyName=" + countyName + ", countyCode=" + countyCode + '}';
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 7;
+ hash = 11 * hash + Objects.hashCode(this.countryCode);
+ hash = 11 * hash + Objects.hashCode(this.provinceCode);
+ hash = 11 * hash + Objects.hashCode(this.provinceName);
+ hash = 11 * hash + Objects.hashCode(this.countryName);
+ hash = 11 * hash + Objects.hashCode(this.countyName);
+ hash = 11 * hash + Objects.hashCode(this.countyCode);
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final AdminBoundary other = (AdminBoundary) obj;
+ if (!Objects.equals(this.countryCode, other.countryCode)) {
+ return false;
+ }
+ if (!Objects.equals(this.provinceCode, other.provinceCode)) {
+ return false;
+ }
+ if (!Objects.equals(this.provinceName, other.provinceName)) {
+ return false;
+ }
+ if (!Objects.equals(this.countryName, other.countryName)) {
+ return false;
+ }
+ if (!Objects.equals(this.countyName, other.countyName)) {
+ return false;
+ }
+ if (!Objects.equals(this.countyCode, other.countyCode)) {
+ return false;
+ }
+ return true;
+ }
+
+ public String getProvinceCode() {
+ return provinceCode;
+ }
+
+ public String getCountryRegex() {
+ return countryRegex;
+ }
+
+ public String getProvinceRegex() {
+ return provinceRegex;
+ }
+
+ public String getCountyRegex() {
+ return countyRegex;
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContext.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContext.java
index beadf4e..85553c3 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContext.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContext.java
@@ -1,153 +1,153 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-public class AdminBoundaryContext {
-
- private final Map<String, Set<Integer>> countryMentions;
- private final Map<String, Set<Integer>> provMentions;
- private final Map<String, Set<Integer>> countyMentions;
- private final Set<String> countryHits;
- private final Set<String> provHits;
- private final Set<String> countyHits;
- private final Map<String, String> countryRefMap;
- private final Map<String, String> countryRegexMap;
- private final Map<String, String> countyRegexMap;
- private final Map<String, String> provinceRegexMap;
- private final Map<String, Map<String, String>> provRefMap;
- private final Map<String, Map<String, String>> countyRefMap;
- private final Set<String> whereClauses;
- private final Map<String, Set<String>> nameCodesMap;
-
- public AdminBoundaryContext(Map<String, Set<Integer>> countryMentions,
- Map<String, Set<Integer>> provMentions,
- Map<String, Set<Integer>> countyMentions,
- Set<String> countryHits,
- Set<String> provHits,
- Set<String> countyHits,
- Map<String, String> countryRefMap,
- Map<String, Map<String, String>> provRefMap,
- Map<String, Map<String, String>> countyRefMap, Map<String, Set<String>> nameCodesMap, Map<String, String> countryRegexMap, Map<String, String> provinceRegexMap,
- Map<String, String> countyRegexMap) {
- this.countryMentions = countryMentions;
- this.provMentions = provMentions;
- this.countyMentions = countyMentions;
- this.countryHits = countryHits;
- this.provHits = provHits;
- this.countyHits = countyHits;
- this.countryRefMap = countryRefMap;
- this.provRefMap = provRefMap;
- this.countyRefMap = countyRefMap;
- this.whereClauses = setWhereClauses();
- this.nameCodesMap = nameCodesMap;
- this.countryRegexMap = countryRegexMap;
- this.provinceRegexMap = provinceRegexMap;
- this.countyRegexMap = countyRegexMap;
- }
-
- public Map<String, Set<String>> getNameCodesMap() {
- return nameCodesMap;
- }
-
- public Map<String, Set<Integer>> getCountryMentions() {
- return countryMentions;
- }
-
- public Map<String, Set<Integer>> getProvMentions() {
- return provMentions;
- }
-
- public Map<String, Set<Integer>> getCountyMentions() {
- return countyMentions;
- }
-
- public Set<String> getCountryHits() {
- return countryHits;
- }
-
- public Set<String> getProvHits() {
- return provHits;
- }
-
- public Set<String> getCountyHits() {
- return countyHits;
- }
-
- public Map<String, String> getCountryRefMap() {
- return countryRefMap;
- }
-
- public Map<String, Map<String, String>> getProvRefMap() {
- return provRefMap;
- }
-
- public Map<String, Map<String, String>> getCountyRefMap() {
- return countyRefMap;
- }
-
- public Set<String> getWhereClauses() {
- return whereClauses;
- }
-
- private Set<String> setWhereClauses() {
- Set<String> clauses = new HashSet<>();
- for (String countryCode : this.getCountryHits()) {
- String gazType = countryCode.equalsIgnoreCase("us") ? " AND gazsource:usgs" : " AND gazsource:geonames";
- if (countryCode.toLowerCase().matches(".*rg[0-9].*")) {
- gazType = " AND gazsource:region";
- }
- Map<String, String> provsForCountry = this.getProvRefMap().get(countryCode);
- if (provsForCountry == null) {
- provsForCountry = new HashMap<>();
- }
- Map<String, String> provs = new HashMap<>();
-
-// if (!provsForCountry.isEmpty()) {
-// for (String pcode : provsForCountry.keySet()) {
-// if (this.getProvHits().contains(pcode)) {
-// provs.put(pcode, provsForCountry.get(pcode));
-//
-// clauses.add(" countrycode:" + countryCode + " AND admincode:" + pcode + gazType);
-//
-// }
-// }
-// }
- if (provs.isEmpty()) {
- //got a country with no mentioned provs
- clauses.add(" countrycode:" + countryCode + gazType);
- }
- }
- return clauses;
- }
-
- public Map<String, String> getCountryRegexMap() {
- return countryRegexMap;
- }
-
- public Map<String, String> getCountyRegexMap() {
- return countyRegexMap;
- }
-
- public Map<String, String> getProvinceRegexMap() {
- return provinceRegexMap;
- }
-
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class AdminBoundaryContext {
+
+ private final Map<String, Set<Integer>> countryMentions;
+ private final Map<String, Set<Integer>> provMentions;
+ private final Map<String, Set<Integer>> countyMentions;
+ private final Set<String> countryHits;
+ private final Set<String> provHits;
+ private final Set<String> countyHits;
+ private final Map<String, String> countryRefMap;
+ private final Map<String, String> countryRegexMap;
+ private final Map<String, String> countyRegexMap;
+ private final Map<String, String> provinceRegexMap;
+ private final Map<String, Map<String, String>> provRefMap;
+ private final Map<String, Map<String, String>> countyRefMap;
+ private final Set<String> whereClauses;
+ private final Map<String, Set<String>> nameCodesMap;
+
+ public AdminBoundaryContext(Map<String, Set<Integer>> countryMentions,
+ Map<String, Set<Integer>> provMentions,
+ Map<String, Set<Integer>> countyMentions,
+ Set<String> countryHits,
+ Set<String> provHits,
+ Set<String> countyHits,
+ Map<String, String> countryRefMap,
+ Map<String, Map<String, String>> provRefMap,
+ Map<String, Map<String, String>> countyRefMap, Map<String, Set<String>> nameCodesMap, Map<String, String> countryRegexMap, Map<String, String> provinceRegexMap,
+ Map<String, String> countyRegexMap) {
+ this.countryMentions = countryMentions;
+ this.provMentions = provMentions;
+ this.countyMentions = countyMentions;
+ this.countryHits = countryHits;
+ this.provHits = provHits;
+ this.countyHits = countyHits;
+ this.countryRefMap = countryRefMap;
+ this.provRefMap = provRefMap;
+ this.countyRefMap = countyRefMap;
+ this.whereClauses = setWhereClauses();
+ this.nameCodesMap = nameCodesMap;
+ this.countryRegexMap = countryRegexMap;
+ this.provinceRegexMap = provinceRegexMap;
+ this.countyRegexMap = countyRegexMap;
+ }
+
+ public Map<String, Set<String>> getNameCodesMap() {
+ return nameCodesMap;
+ }
+
+ public Map<String, Set<Integer>> getCountryMentions() {
+ return countryMentions;
+ }
+
+ public Map<String, Set<Integer>> getProvMentions() {
+ return provMentions;
+ }
+
+ public Map<String, Set<Integer>> getCountyMentions() {
+ return countyMentions;
+ }
+
+ public Set<String> getCountryHits() {
+ return countryHits;
+ }
+
+ public Set<String> getProvHits() {
+ return provHits;
+ }
+
+ public Set<String> getCountyHits() {
+ return countyHits;
+ }
+
+ public Map<String, String> getCountryRefMap() {
+ return countryRefMap;
+ }
+
+ public Map<String, Map<String, String>> getProvRefMap() {
+ return provRefMap;
+ }
+
+ public Map<String, Map<String, String>> getCountyRefMap() {
+ return countyRefMap;
+ }
+
+ public Set<String> getWhereClauses() {
+ return whereClauses;
+ }
+
+ private Set<String> setWhereClauses() {
+ Set<String> clauses = new HashSet<>();
+ for (String countryCode : this.getCountryHits()) {
+ String gazType = countryCode.equalsIgnoreCase("us") ? " AND gazsource:usgs" : " AND gazsource:geonames";
+ if (countryCode.toLowerCase().matches(".*rg[0-9].*")) {
+ gazType = " AND gazsource:region";
+ }
+ Map<String, String> provsForCountry = this.getProvRefMap().get(countryCode);
+ if (provsForCountry == null) {
+ provsForCountry = new HashMap<>();
+ }
+ Map<String, String> provs = new HashMap<>();
+
+// if (!provsForCountry.isEmpty()) {
+// for (String pcode : provsForCountry.keySet()) {
+// if (this.getProvHits().contains(pcode)) {
+// provs.put(pcode, provsForCountry.get(pcode));
+//
+// clauses.add(" countrycode:" + countryCode + " AND admincode:" + pcode + gazType);
+//
+// }
+// }
+// }
+ if (provs.isEmpty()) {
+ //got a country with no mentioned provs
+ clauses.add(" countrycode:" + countryCode + gazType);
+ }
+ }
+ return clauses;
+ }
+
+ public Map<String, String> getCountryRegexMap() {
+ return countryRegexMap;
+ }
+
+ public Map<String, String> getCountyRegexMap() {
+ return countyRegexMap;
+ }
+
+ public Map<String, String> getProvinceRegexMap() {
+ return provinceRegexMap;
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContextGenerator.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContextGenerator.java
index a6741fe..166cfca 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContextGenerator.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContextGenerator.java
@@ -1,412 +1,412 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.logging.Level;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Finds instances of country mentions in a String, typically a document text.
- * Used to boost or degrade scoring of linked geo entities
- *
- */
-public class AdminBoundaryContextGenerator {
-
- private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private List<CountryContextEntry> countrydata;
- private Map<String, Set<String>> nameCodesMap = new HashMap<>();
- private final Map<String, Set<Integer>> countryMentions = new HashMap<>();
-
- Map<String, String> countryRegexMap = new HashMap<>();
- Map<String, String> provinceRegexMap = new HashMap<>();
- Map<String, String> countyRegexMap = new HashMap<>();
-
- private final Set<CountryContextEntry> countryHits = new HashSet<>();
- private final EntityLinkerProperties properties;
- private final List<AdminBoundary> adminBoundaryData= new ArrayList<>();
- private final Set<AdminBoundary> adminBoundaryHits = new HashSet<>();
- private AdminBoundaryContext context;
-
- public AdminBoundaryContext getContext(String text) {
- context = null;
- nameCodesMap.clear();
- context = process(text);
-
- return context;
- }
-
- private final Set<String> countryHitSet = new HashSet<>();
- private final Map<String, String> countryMap = new HashMap<>();
- private final Map<String, Map<String, String>> provMap = new HashMap<>();
- private final Map<String, Map<String, String>> countyMap = new HashMap<>();
-
- private Map<String, Set<Integer>> provMentions = new HashMap<>();
- private Map<String, Set<Integer>> countyMentions = new HashMap<>();
-
- private final Set<String> provHits = new HashSet<>();
- private final Set<String> countyHits = new HashSet<>();
-
- public static void main(String[] args) {
- try {
- AdminBoundaryContextGenerator countryContext
- = new AdminBoundaryContextGenerator(new EntityLinkerProperties(new File("C:\\Temp\\gaz_data\\newCountryContextfile.txt")));
-
- AdminBoundaryContext c = countryContext.process("This artcle is about fairfax county virginia in the north of florida in the united states. It is also about Moscow and atlanta. Hillsborough county florida is a nice place. Eastern Africa people are cool.");
- System.out.println(c);
- } catch (Exception ex) {
- java.util.logging.Logger.getLogger(AdminBoundaryContextGenerator.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
-
- public AdminBoundaryContextGenerator(EntityLinkerProperties properties) throws IOException {
- this.properties = properties;
- if (countrydata == null) {
- String path = this.properties.getProperty("opennlp.geoentitylinker.countrycontext.filepath", "");
- if (path == null || path.trim().isEmpty()) {
- throw new IOException("missing country context data configuration. Property opennlp.geoentitylinker.countrycontext.filepath must have a valid path value in entitylinker properties file");
- }
- File countryContextFile = new File(path);
- if (countryContextFile == null || !countryContextFile.exists()) {
- throw new IOException("missing country context file");
- }
- //countrydata = getCountryContextFromFile(countryContextFile);
- getContextFromFile(countryContextFile);
- if (adminBoundaryData.isEmpty()) {
- throw new IOException("missing country context data");
- }
- }
- }
-
- public Map<String, Set<Integer>> getCountryMentions() {
- return countryMentions;
- }
-
- /**
- * @return returns the last set of hits after calling regexFind
- */
- public Set<CountryContextEntry> getCountryHits() {
- return countryHits;
- }
-
- /**
- * @return returns the last name to codes map after calling regexFind
- */
- public Map<String, Set<String>> getNameCodesMap() {
- return nameCodesMap;
- }
-
- public void setNameCodesMap(Map<String, Set<String>> nameCodesMap) {
- this.nameCodesMap = nameCodesMap;
- }
-
- private void reset() {
- this.nameCodesMap.clear();
- this.countryHitSet.clear();
- this.countryHits.clear();
- this.countryMentions.clear();
- this.provHits.clear();
- this.provMentions.clear();
- this.countyHits.clear();
- this.countyMentions.clear();
- this.adminBoundaryHits.clear();
- }
-
- /**
- * Finds indicators of countries, provinces, and cities, as per the USGS and
- * Geonames gazetteers. The results of this are used to score toponymns
- * downstream. The full text of a document should be passed in here.
- *
- * @param text the full text of the document (block of text).
- */
- private AdminBoundaryContext process(String text) {
- try {
- reset();
- Map<String, Set<Integer>> countryhitMap = regexfind(text, countryMap, countryHitSet, "country");
- if (!countryhitMap.isEmpty()) {
- for (String cc : countryhitMap.keySet()) {
- Map<String, String> provsForCc = provMap.get(cc);
- if (provsForCc != null) {
- provMentions.putAll(regexfind(text, provsForCc, provHits, "province"));
- if (provMentions != null) {
- for (String prov : provMentions.keySet()) {
- Map<String, String> get = countyMap.get(prov);
- if (get != null) {
- countyMentions.putAll(regexfind(text, get, countyHits, "province"));
- }
- }
- }
- }
- }
- } else {
- for (Map<String, String> provsForCc : provMap.values()) {
- if (provsForCc != null) {
- provMentions = regexfind(text, provsForCc, provHits, "province");
- if (provMentions != null) {
- for (String prov : provMentions.keySet()) {
- //fake a country hit based on a province hit... this gets fuzzy
- String cc = prov.split("\\.")[0];
- if (!countryhitMap.containsKey(cc)) {
- countryhitMap.put(cc, provMentions.get(prov));
- countryHitSet.add(cc);
- } else {
- countryhitMap.get(cc).addAll(provMentions.get(prov));
- }
- Map<String, String> get = countyMap.get(prov);
- if (get != null) {
- countyMentions = regexfind(text, get, countyHits, "oounty");
- }
- }
- }
- }
- }
- }
-
- Map<String, String> countryRefMap = new HashMap<>();
-
- for (String c : countryHitSet) {
- String countryName = countryMap.get(c);
- if (countryName != null) {
- countryRefMap.put(c, countryName);
- }
- }
-
- return new AdminBoundaryContext(countryhitMap, provMentions, countyMentions, countryHitSet, provHits, countyHits,
- countryRefMap, provMap, countyMap, nameCodesMap, countryRegexMap, provinceRegexMap, countyRegexMap);
- } catch (Exception e) {
- LOG.error(e.getLocalizedMessage(), e);
- }
- return null;
- }
-
- /**
- * discovers indicators of admin boundary data using regex.
- *
- * @param docText the full text
- * @param lookupMap a map to use to find names. the key=a location code, the
- * value is an actual name.
- * @param hitsRef a reference to a set that stores the hits by id
- */
- private Map<String, Set<Integer>> regexfind(String docText, Map<String, String> lookupMap, Set<String> hitsRef, String locationType) {
- Map<String, Set<Integer>> mentions = new HashMap<>();
- if (lookupMap == null) {
- return mentions;
- }
- try {
-
- for (String entry : lookupMap.keySet()) {
-
- String name = lookupMap.get(entry).toLowerCase();
- if (name == null) {
- continue;
- }
- switch (locationType) {
- case "country":
- if (this.countryRegexMap.containsKey(entry)) {
- name = countryRegexMap.get(entry);
- }
- break;
-
- case "province":
- if (this.provinceRegexMap.containsKey(entry)) {
- name = provinceRegexMap.get(entry);
- }
- break;
- case "county":
- if (this.countyRegexMap.containsKey(entry)) {
- name = countyRegexMap.get(entry);
- }
- break;
- }
- name = "(^|[^\\p{L}\\p{Nd}])" + name.replace(", the", "") + "([^\\p{L}\\p{Nd}]|$)";
- Pattern regex = Pattern.compile(name, Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
- Matcher rs = regex.matcher(docText);
- String code = entry.toLowerCase();
- code = code.trim().replace("", "");
- boolean found = false;
- while (rs.find()) {
- found = true;
- Integer start = rs.start();
- String hit = rs.group().toLowerCase().trim();
- hit = hit.replaceAll("\\.|,|;|\\?|!|\\\\|/|\"|'|=|-|&", "");
- if (mentions.containsKey(code)) {
- mentions.get(code).add(start);
- } else {
- Set<Integer> newset = new HashSet<>();
- newset.add(start);
- mentions.put(code, newset);
- }
- if (!hit.equals("")) {
- if (this.nameCodesMap.containsKey(hit)) {
- nameCodesMap.get(hit).add(code);
- } else {
- HashSet<String> newset = new HashSet<>();
- newset.add(code);
- nameCodesMap.put(hit, newset);
- }
- }
-
- }
- if (found) {
- hitsRef.add(code);
-
- }
- }
-
- } catch (Exception ex) {
- LOG.error(ex.getLocalizedMessage(), ex);
- }
-
- return mentions;
- }
-
- private void getContextFromFile(File countryContextFile) {
- if (this.adminBoundaryData != null && !this.adminBoundaryData.isEmpty()) {
- return;
- }
-
- BufferedReader reader;
- try {
- reader = new BufferedReader(new FileReader(countryContextFile));
- String line;
- int lineNum = 0;
- while ((line = reader.readLine()) != null) {
- String[] values = line.split("\t");
- if (lineNum == 0) {
- lineNum++;
- continue;
- //skip column name headers
- }
- if (values.length == 9) {
- AdminBoundary entry = new AdminBoundary(
- values[0].toLowerCase().trim().replace("", ""),
- values[3].toLowerCase().trim(),
- values[1].toLowerCase().trim(),
- values[4].toLowerCase().trim(),
- values[2].toLowerCase().trim(),
- values[5].toLowerCase().trim(),
- values[6].toLowerCase().trim(),
- values[7].toLowerCase().trim(),
- values[8].toLowerCase().trim());
- this.adminBoundaryData.add(entry);
- } else {
- throw new IllegalArgumentException("Improperly formatted file");
- }
-
- }
- reader.close();
- } catch (IOException ex) {
- LOG.error(ex.getLocalizedMessage(), ex);
- }
-
- loadMaps(this.adminBoundaryData);
-
- }
-
- private void loadMaps(List<AdminBoundary> boundaries) {
- for (AdminBoundary adm : boundaries) {
- if (!adm.getCountryCode().equals("null")) {
- countryMap.put(adm.getCountryCode(), adm.getCountryName());
- if (countryRegexMap.containsKey(adm.getCountryCode())) {
- String currentRegex = countryRegexMap.get(adm.getCountryCode());
- if (currentRegex.length() > adm.getCountryRegex().length()) {
- // the longest one wins if they are not all the same for each entry in the file
- countryRegexMap.put(adm.getCountryCode(), currentRegex);
- }//else do nothing
- } else {
- countryRegexMap.put(adm.getCountryCode(), adm.getCountryRegex());
- }
-
- if (!adm.getProvCode().equals("null")) {
- Map<String, String> provs = provMap.get(adm.getCountryCode());
- if (provs == null) {
- provs = new HashMap<>();
- }
- //if (!provs.containsKey(adm.getProvCode())) {
- provs.put(adm.getCountryCode() + "." + adm.getProvCode(), adm.getProvinceName());
- provMap.put(adm.getCountryCode(), provs);
- // }
-
- if (!adm.getCountyCode().equalsIgnoreCase("no_data_found") && !adm.getCountyName().equalsIgnoreCase("no_data_found")) {
- Map<String, String> counties = countyMap.get(adm.getCountryCode() + "." + adm.getProvCode());
- if (counties == null) {
- counties = new HashMap<>();
- } // if (!counties.containsKey(adm.getCountyCode())) {
- String countyid = adm.getCountryCode() + "." + adm.getProvCode() + "." + adm.getCountyCode();
- counties.put(countyid, adm.getCountyName());
- countyMap.put(adm.getCountryCode() + "." + adm.getProvCode(), counties);
- // }
-
- }
-
- }
- }
- }
- fillProvRegexMap();
- fillCountyRegexMap();
- }
-
- private void fillProvRegexMap() {
- this.provinceRegexMap = new HashMap<>();
- // this.adminBoundaryData
- for (AdminBoundary adm : adminBoundaryData) {
-
- if (provinceRegexMap.containsKey(adm.getProvCode())) {
- String currentRegex = provinceRegexMap.get(adm.getProvCode());
- if (currentRegex.length() > adm.getProvinceRegex().length()) {
- // the longest one wins if they are not all the same for each entry in the file
- provinceRegexMap.put(adm.getProvCode(), currentRegex);
- }//else do nothing
- } else {
- provinceRegexMap.put(adm.getProvCode(), adm.getProvinceRegex());
- }
- }
- }
-
- private void fillCountyRegexMap() {
- this.countyRegexMap = new HashMap<>();
- // this.adminBoundaryData
- for (AdminBoundary adm : adminBoundaryData) {
-
- if (countyRegexMap.containsKey(adm.getCountyCode())) {
- String currentRegex = countyRegexMap.get(adm.getCountyCode());
- if (currentRegex.length() > adm.getCountyRegex().length()) {
- // the longest one wins if they are not all the same for each entry in the file
- countyRegexMap.put(adm.getCountyCode(), currentRegex);
- }//else do nothing
- } else {
- countyRegexMap.put(adm.getCountyCode(), adm.getCountyRegex());
- }
- }
-
- }
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Finds instances of country mentions in a String, typically a document text.
+ * Used to boost or degrade scoring of linked geo entities
+ *
+ */
+public class AdminBoundaryContextGenerator {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private List<CountryContextEntry> countrydata;
+ private Map<String, Set<String>> nameCodesMap = new HashMap<>();
+ private final Map<String, Set<Integer>> countryMentions = new HashMap<>();
+
+ Map<String, String> countryRegexMap = new HashMap<>();
+ Map<String, String> provinceRegexMap = new HashMap<>();
+ Map<String, String> countyRegexMap = new HashMap<>();
+
+ private final Set<CountryContextEntry> countryHits = new HashSet<>();
+ private final EntityLinkerProperties properties;
+ private final List<AdminBoundary> adminBoundaryData= new ArrayList<>();
+ private final Set<AdminBoundary> adminBoundaryHits = new HashSet<>();
+ private AdminBoundaryContext context;
+
+ public AdminBoundaryContext getContext(String text) {
+ context = null;
+ nameCodesMap.clear();
+ context = process(text);
+
+ return context;
+ }
+
+ private final Set<String> countryHitSet = new HashSet<>();
+ private final Map<String, String> countryMap = new HashMap<>();
+ private final Map<String, Map<String, String>> provMap = new HashMap<>();
+ private final Map<String, Map<String, String>> countyMap = new HashMap<>();
+
+ private Map<String, Set<Integer>> provMentions = new HashMap<>();
+ private Map<String, Set<Integer>> countyMentions = new HashMap<>();
+
+ private final Set<String> provHits = new HashSet<>();
+ private final Set<String> countyHits = new HashSet<>();
+
+ public static void main(String[] args) {
+ try {
+ AdminBoundaryContextGenerator countryContext
+ = new AdminBoundaryContextGenerator(new EntityLinkerProperties(new File("C:\\Temp\\gaz_data\\newCountryContextfile.txt")));
+
+ AdminBoundaryContext c = countryContext.process("This artcle is about fairfax county virginia in the north of florida in the united states. It is also about Moscow and atlanta. Hillsborough county florida is a nice place. Eastern Africa people are cool.");
+ System.out.println(c);
+ } catch (Exception ex) {
+ java.util.logging.Logger.getLogger(AdminBoundaryContextGenerator.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ }
+
+ public AdminBoundaryContextGenerator(EntityLinkerProperties properties) throws IOException {
+ this.properties = properties;
+ if (countrydata == null) {
+ String path = this.properties.getProperty("opennlp.geoentitylinker.countrycontext.filepath", "");
+ if (path == null || path.trim().isEmpty()) {
+ throw new IOException("missing country context data configuration. Property opennlp.geoentitylinker.countrycontext.filepath must have a valid path value in entitylinker properties file");
+ }
+ File countryContextFile = new File(path);
+ if (countryContextFile == null || !countryContextFile.exists()) {
+ throw new IOException("missing country context file");
+ }
+ //countrydata = getCountryContextFromFile(countryContextFile);
+ getContextFromFile(countryContextFile);
+ if (adminBoundaryData.isEmpty()) {
+ throw new IOException("missing country context data");
+ }
+ }
+ }
+
+ public Map<String, Set<Integer>> getCountryMentions() {
+ return countryMentions;
+ }
+
+ /**
+ * @return returns the last set of hits after calling regexFind
+ */
+ public Set<CountryContextEntry> getCountryHits() {
+ return countryHits;
+ }
+
+ /**
+ * @return returns the last name to codes map after calling regexFind
+ */
+ public Map<String, Set<String>> getNameCodesMap() {
+ return nameCodesMap;
+ }
+
+ public void setNameCodesMap(Map<String, Set<String>> nameCodesMap) {
+ this.nameCodesMap = nameCodesMap;
+ }
+
+ private void reset() {
+ this.nameCodesMap.clear();
+ this.countryHitSet.clear();
+ this.countryHits.clear();
+ this.countryMentions.clear();
+ this.provHits.clear();
+ this.provMentions.clear();
+ this.countyHits.clear();
+ this.countyMentions.clear();
+ this.adminBoundaryHits.clear();
+ }
+
+ /**
+ * Finds indicators of countries, provinces, and cities, as per the USGS and
+ * Geonames gazetteers. The results of this are used to score toponymns
+ * downstream. The full text of a document should be passed in here.
+ *
+ * @param text the full text of the document (block of text).
+ */
+ private AdminBoundaryContext process(String text) {
+ try {
+ reset();
+ Map<String, Set<Integer>> countryhitMap = regexfind(text, countryMap, countryHitSet, "country");
+ if (!countryhitMap.isEmpty()) {
+ for (String cc : countryhitMap.keySet()) {
+ Map<String, String> provsForCc = provMap.get(cc);
+ if (provsForCc != null) {
+ provMentions.putAll(regexfind(text, provsForCc, provHits, "province"));
+ if (provMentions != null) {
+ for (String prov : provMentions.keySet()) {
+ Map<String, String> get = countyMap.get(prov);
+ if (get != null) {
+ countyMentions.putAll(regexfind(text, get, countyHits, "province"));
+ }
+ }
+ }
+ }
+ }
+ } else {
+ for (Map<String, String> provsForCc : provMap.values()) {
+ if (provsForCc != null) {
+ provMentions = regexfind(text, provsForCc, provHits, "province");
+ if (provMentions != null) {
+ for (String prov : provMentions.keySet()) {
+ //fake a country hit based on a province hit... this gets fuzzy
+ String cc = prov.split("\\.")[0];
+ if (!countryhitMap.containsKey(cc)) {
+ countryhitMap.put(cc, provMentions.get(prov));
+ countryHitSet.add(cc);
+ } else {
+ countryhitMap.get(cc).addAll(provMentions.get(prov));
+ }
+ Map<String, String> get = countyMap.get(prov);
+ if (get != null) {
+ countyMentions = regexfind(text, get, countyHits, "oounty");
+ }
+ }
+ }
+ }
+ }
+ }
+
+ Map<String, String> countryRefMap = new HashMap<>();
+
+ for (String c : countryHitSet) {
+ String countryName = countryMap.get(c);
+ if (countryName != null) {
+ countryRefMap.put(c, countryName);
+ }
+ }
+
+ return new AdminBoundaryContext(countryhitMap, provMentions, countyMentions, countryHitSet, provHits, countyHits,
+ countryRefMap, provMap, countyMap, nameCodesMap, countryRegexMap, provinceRegexMap, countyRegexMap);
+ } catch (Exception e) {
+ LOG.error(e.getLocalizedMessage(), e);
+ }
+ return null;
+ }
+
+ /**
+ * discovers indicators of admin boundary data using regex.
+ *
+ * @param docText the full text
+ * @param lookupMap a map to use to find names. the key=a location code, the
+ * value is an actual name.
+ * @param hitsRef a reference to a set that stores the hits by id
+ */
+ private Map<String, Set<Integer>> regexfind(String docText, Map<String, String> lookupMap, Set<String> hitsRef, String locationType) {
+ Map<String, Set<Integer>> mentions = new HashMap<>();
+ if (lookupMap == null) {
+ return mentions;
+ }
+ try {
+
+ for (String entry : lookupMap.keySet()) {
+
+ String name = lookupMap.get(entry).toLowerCase();
+ if (name == null) {
+ continue;
+ }
+ switch (locationType) {
+ case "country":
+ if (this.countryRegexMap.containsKey(entry)) {
+ name = countryRegexMap.get(entry);
+ }
+ break;
+
+ case "province":
+ if (this.provinceRegexMap.containsKey(entry)) {
+ name = provinceRegexMap.get(entry);
+ }
+ break;
+ case "county":
+ if (this.countyRegexMap.containsKey(entry)) {
+ name = countyRegexMap.get(entry);
+ }
+ break;
+ }
+ name = "(^|[^\\p{L}\\p{Nd}])" + name.replace(", the", "") + "([^\\p{L}\\p{Nd}]|$)";
+ Pattern regex = Pattern.compile(name, Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+ Matcher rs = regex.matcher(docText);
+ String code = entry.toLowerCase();
+ code = code.trim().replace("", "");
+ boolean found = false;
+ while (rs.find()) {
+ found = true;
+ Integer start = rs.start();
+ String hit = rs.group().toLowerCase().trim();
+ hit = hit.replaceAll("\\.|,|;|\\?|!|\\\\|/|\"|'|=|-|&", "");
+ if (mentions.containsKey(code)) {
+ mentions.get(code).add(start);
+ } else {
+ Set<Integer> newset = new HashSet<>();
+ newset.add(start);
+ mentions.put(code, newset);
+ }
+ if (!hit.equals("")) {
+ if (this.nameCodesMap.containsKey(hit)) {
+ nameCodesMap.get(hit).add(code);
+ } else {
+ HashSet<String> newset = new HashSet<>();
+ newset.add(code);
+ nameCodesMap.put(hit, newset);
+ }
+ }
+
+ }
+ if (found) {
+ hitsRef.add(code);
+
+ }
+ }
+
+ } catch (Exception ex) {
+ LOG.error(ex.getLocalizedMessage(), ex);
+ }
+
+ return mentions;
+ }
+
+ private void getContextFromFile(File countryContextFile) {
+ if (this.adminBoundaryData != null && !this.adminBoundaryData.isEmpty()) {
+ return;
+ }
+
+ BufferedReader reader;
+ try {
+ reader = new BufferedReader(new FileReader(countryContextFile));
+ String line;
+ int lineNum = 0;
+ while ((line = reader.readLine()) != null) {
+ String[] values = line.split("\t");
+ if (lineNum == 0) {
+ lineNum++;
+ continue;
+ //skip column name headers
+ }
+ if (values.length == 9) {
+ AdminBoundary entry = new AdminBoundary(
+ values[0].toLowerCase().trim().replace("", ""),
+ values[3].toLowerCase().trim(),
+ values[1].toLowerCase().trim(),
+ values[4].toLowerCase().trim(),
+ values[2].toLowerCase().trim(),
+ values[5].toLowerCase().trim(),
+ values[6].toLowerCase().trim(),
+ values[7].toLowerCase().trim(),
+ values[8].toLowerCase().trim());
+ this.adminBoundaryData.add(entry);
+ } else {
+ throw new IllegalArgumentException("Improperly formatted file");
+ }
+
+ }
+ reader.close();
+ } catch (IOException ex) {
+ LOG.error(ex.getLocalizedMessage(), ex);
+ }
+
+ loadMaps(this.adminBoundaryData);
+
+ }
+
+ private void loadMaps(List<AdminBoundary> boundaries) {
+ for (AdminBoundary adm : boundaries) {
+ if (!adm.getCountryCode().equals("null")) {
+ countryMap.put(adm.getCountryCode(), adm.getCountryName());
+ if (countryRegexMap.containsKey(adm.getCountryCode())) {
+ String currentRegex = countryRegexMap.get(adm.getCountryCode());
+ if (currentRegex.length() > adm.getCountryRegex().length()) {
+ // the longest one wins if they are not all the same for each entry in the file
+ countryRegexMap.put(adm.getCountryCode(), currentRegex);
+ }//else do nothing
+ } else {
+ countryRegexMap.put(adm.getCountryCode(), adm.getCountryRegex());
+ }
+
+ if (!adm.getProvCode().equals("null")) {
+ Map<String, String> provs = provMap.get(adm.getCountryCode());
+ if (provs == null) {
+ provs = new HashMap<>();
+ }
+ //if (!provs.containsKey(adm.getProvCode())) {
+ provs.put(adm.getCountryCode() + "." + adm.getProvCode(), adm.getProvinceName());
+ provMap.put(adm.getCountryCode(), provs);
+ // }
+
+ if (!adm.getCountyCode().equalsIgnoreCase("no_data_found") && !adm.getCountyName().equalsIgnoreCase("no_data_found")) {
+ Map<String, String> counties = countyMap.get(adm.getCountryCode() + "." + adm.getProvCode());
+ if (counties == null) {
+ counties = new HashMap<>();
+ } // if (!counties.containsKey(adm.getCountyCode())) {
+ String countyid = adm.getCountryCode() + "." + adm.getProvCode() + "." + adm.getCountyCode();
+ counties.put(countyid, adm.getCountyName());
+ countyMap.put(adm.getCountryCode() + "." + adm.getProvCode(), counties);
+ // }
+
+ }
+
+ }
+ }
+ }
+ fillProvRegexMap();
+ fillCountyRegexMap();
+ }
+
+ private void fillProvRegexMap() {
+ this.provinceRegexMap = new HashMap<>();
+ // this.adminBoundaryData
+ for (AdminBoundary adm : adminBoundaryData) {
+
+ if (provinceRegexMap.containsKey(adm.getProvCode())) {
+ String currentRegex = provinceRegexMap.get(adm.getProvCode());
+ if (currentRegex.length() > adm.getProvinceRegex().length()) {
+ // the longest one wins if they are not all the same for each entry in the file
+ provinceRegexMap.put(adm.getProvCode(), currentRegex);
+ }//else do nothing
+ } else {
+ provinceRegexMap.put(adm.getProvCode(), adm.getProvinceRegex());
+ }
+ }
+ }
+
+ private void fillCountyRegexMap() {
+ this.countyRegexMap = new HashMap<>();
+ // this.adminBoundaryData
+ for (AdminBoundary adm : adminBoundaryData) {
+
+ if (countyRegexMap.containsKey(adm.getCountyCode())) {
+ String currentRegex = countyRegexMap.get(adm.getCountyCode());
+ if (currentRegex.length() > adm.getCountyRegex().length()) {
+ // the longest one wins if they are not all the same for each entry in the file
+ countyRegexMap.put(adm.getCountyCode(), currentRegex);
+ }//else do nothing
+ } else {
+ countyRegexMap.put(adm.getCountyCode(), adm.getCountyRegex());
+ }
+ }
+
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/CountryContextEntry.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/CountryContextEntry.java
index 324bd0f..04039e9 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/CountryContextEntry.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/CountryContextEntry.java
@@ -1,118 +1,118 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.util.Objects;
-
-/**
- *Stores a tuple from the opennlp.geoentitylinker.countrycontext.txt file, which is used to find country mentions in document text.
- *
- */
-public class CountryContextEntry {
- /*
- * rc,cc1, full_name_nd_ro,dsg
- */
-
- private String rc;
- private String cc1;
- private String full_name_nd_ro;
- private String dsg;
- private String provCode;
- public CountryContextEntry() {
- }
-
- public CountryContextEntry(String rc, String cc1, String full_name_nd_ro, String dsg) {
- this.rc = rc;
- this.cc1 = cc1;
- this.full_name_nd_ro = full_name_nd_ro;
- this.dsg = dsg;
- }
-
- public String getProvCode() {
- return provCode;
- }
-
- public void setProvCode(String provCode) {
- this.provCode = provCode;
- }
-
- public String getRc() {
- return rc;
- }
-
- public void setRc(String rc) {
- this.rc = rc;
- }
-
- public String getCc1() {
- return cc1;
- }
-
- public void setCc1(String cc1) {
- this.cc1 = cc1;
- }
-
- public String getFull_name_nd_ro() {
- return full_name_nd_ro;
- }
-
- public void setFull_name_nd_ro(String full_name_nd_ro) {
- this.full_name_nd_ro = full_name_nd_ro;
- }
-
- public String getDsg() {
- return dsg;
- }
-
- public void setDsg(String dsg) {
- this.dsg = dsg;
- }
-
- @Override
- public int hashCode() {
- int hash = 7;
- hash = 17 * hash + Objects.hashCode(this.rc);
- hash = 17 * hash + Objects.hashCode(this.cc1);
- hash = 17 * hash + Objects.hashCode(this.full_name_nd_ro);
- hash = 17 * hash + Objects.hashCode(this.dsg);
- return hash;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- final CountryContextEntry other = (CountryContextEntry) obj;
- if (!Objects.equals(this.rc, other.rc)) {
- return false;
- }
- if (!Objects.equals(this.cc1, other.cc1)) {
- return false;
- }
- if (!Objects.equals(this.full_name_nd_ro, other.full_name_nd_ro)) {
- return false;
- }
- if (!Objects.equals(this.dsg, other.dsg)) {
- return false;
- }
- return true;
- }
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.util.Objects;
+
+/**
+ *Stores a tuple from the opennlp.geoentitylinker.countrycontext.txt file, which is used to find country mentions in document text.
+ *
+ */
+public class CountryContextEntry {
+ /*
+ * rc,cc1, full_name_nd_ro,dsg
+ */
+
+ private String rc;
+ private String cc1;
+ private String full_name_nd_ro;
+ private String dsg;
+ private String provCode;
+ public CountryContextEntry() {
+ }
+
+ public CountryContextEntry(String rc, String cc1, String full_name_nd_ro, String dsg) {
+ this.rc = rc;
+ this.cc1 = cc1;
+ this.full_name_nd_ro = full_name_nd_ro;
+ this.dsg = dsg;
+ }
+
+ public String getProvCode() {
+ return provCode;
+ }
+
+ public void setProvCode(String provCode) {
+ this.provCode = provCode;
+ }
+
+ public String getRc() {
+ return rc;
+ }
+
+ public void setRc(String rc) {
+ this.rc = rc;
+ }
+
+ public String getCc1() {
+ return cc1;
+ }
+
+ public void setCc1(String cc1) {
+ this.cc1 = cc1;
+ }
+
+ public String getFull_name_nd_ro() {
+ return full_name_nd_ro;
+ }
+
+ public void setFull_name_nd_ro(String full_name_nd_ro) {
+ this.full_name_nd_ro = full_name_nd_ro;
+ }
+
+ public String getDsg() {
+ return dsg;
+ }
+
+ public void setDsg(String dsg) {
+ this.dsg = dsg;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 7;
+ hash = 17 * hash + Objects.hashCode(this.rc);
+ hash = 17 * hash + Objects.hashCode(this.cc1);
+ hash = 17 * hash + Objects.hashCode(this.full_name_nd_ro);
+ hash = 17 * hash + Objects.hashCode(this.dsg);
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final CountryContextEntry other = (CountryContextEntry) obj;
+ if (!Objects.equals(this.rc, other.rc)) {
+ return false;
+ }
+ if (!Objects.equals(this.cc1, other.cc1)) {
+ return false;
+ }
+ if (!Objects.equals(this.full_name_nd_ro, other.full_name_nd_ro)) {
+ return false;
+ }
+ if (!Objects.equals(this.dsg, other.dsg)) {
+ return false;
+ }
+ return true;
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerEntry.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerEntry.java
index 56e6eef..b7ef387 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerEntry.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerEntry.java
@@ -1,173 +1,173 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import opennlp.tools.entitylinker.BaseLink;
-
-/**
- * Stores a minimal amount of information from a geographic placenames gazetteer.
- */
-public class GazetteerEntry extends BaseLink {
-
- private Double latitude;
- private Double longitude;
- private String source;
- private String indexID;
- private Map<String, String> indexData = new HashMap<>();
- private String countryCode;
- private String provinceCode;
- private String hierarchy;
-
- public GazetteerEntry(String parentID, String itemID, String itemName, String itemType) {
- super(parentID, itemID, itemName, itemType);
- }
-
- /**
- * @return returns the id from the lucene document
- */
- public String getIndexID() {
- return indexID;
- }
- /**
- * sets the id from the lucene document
- */
- public void setIndexID(String indexID) {
- this.indexID = indexID;
- }
-
- /**
- * @return Retrieves the latitude from the gazetteer
- */
- public Double getLatitude() {
- return latitude;
- }
-
- /**
- * sets the latitude from the gazetteer
- */
- public void setLatitude(Double latitude) {
- this.latitude = latitude;
- }
-
- /**
- * @return Retrieves the longitude from the gaz
- */
- public Double getLongitude() {
- return longitude;
- }
-
- /**
- * sets the longitude from the gaz
- *
- * @param longitude
- */
- public void setLongitude(Double longitude) {
- this.longitude = longitude;
- }
-
- /**
- * @return Retrieves the source of the gazetteer data
- */
- public String getSource() {
- return source;
- }
-
- /**
- * sets the source (the source of the gazetteer data)
- *
- * @param source
- */
- public void setSource(String source) {
- this.source = source;
- }
-
- /**
- * @return Retrieves all the other fields in the gazetteer in the form of a map
- */
- public Map<String, String> getIndexData() {
- return indexData;
- }
-
- /**
- * sets the other fields in the gazetteer in the form of a map
- *
- * @param indexData stores all fields in the index as fieldname:value
- */
- public void setIndexData(Map<String, String> indexData) {
- this.indexData = indexData;
- }
-
- @Override
- public String toString() {
-
- return super.toString() + "\n\t\tGazateerEntry\n" + "\t\tlatitude=" + latitude + ", \n\t\tlongitude=" + longitude + ", \n\t\tsource=" + source + ", \n\t\tindexID=" + indexID + ",\n\t\tindexData=" + indexData + "\n";
- }
-
- @Override
- public int hashCode() {
- int hash = 5;
- hash = 71 * hash + Objects.hashCode(this.source);
- hash = 71 * hash + Objects.hashCode(this.indexID);
- return hash;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- final GazetteerEntry other = (GazetteerEntry) obj;
- if (!Objects.equals(this.source, other.source)) {
- return false;
- }
- if (!Objects.equals(this.indexID, other.indexID)) {
- return false;
- }
- return true;
- }
-
-
- public String getCountryCode() {
- return countryCode;
- }
-
- public void setCountryCode(String countryCode) {
- this.countryCode = countryCode;
- }
-
- public String getProvinceCode() {
- return provinceCode;
- }
-
- public void setProvinceCode(String provinceCode) {
- this.provinceCode = provinceCode;
- }
-
- public String getHierarchy() {
- return hierarchy;
- }
-
- public void setHierarchy(String hierarchy) {
- this.hierarchy = hierarchy;
- }
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import opennlp.tools.entitylinker.BaseLink;
+
+/**
+ * Stores a minimal amount of information from a geographic placenames gazetteer.
+ */
+public class GazetteerEntry extends BaseLink {
+
+ private Double latitude;
+ private Double longitude;
+ private String source;
+ private String indexID;
+ private Map<String, String> indexData = new HashMap<>();
+ private String countryCode;
+ private String provinceCode;
+ private String hierarchy;
+
+ public GazetteerEntry(String parentID, String itemID, String itemName, String itemType) {
+ super(parentID, itemID, itemName, itemType);
+ }
+
+ /**
+ * @return returns the id from the lucene document
+ */
+ public String getIndexID() {
+ return indexID;
+ }
+ /**
+ * sets the id from the lucene document
+ */
+ public void setIndexID(String indexID) {
+ this.indexID = indexID;
+ }
+
+ /**
+ * @return Retrieves the latitude from the gazetteer
+ */
+ public Double getLatitude() {
+ return latitude;
+ }
+
+ /**
+ * sets the latitude from the gazetteer
+ */
+ public void setLatitude(Double latitude) {
+ this.latitude = latitude;
+ }
+
+ /**
+ * @return Retrieves the longitude from the gaz
+ */
+ public Double getLongitude() {
+ return longitude;
+ }
+
+ /**
+ * sets the longitude from the gaz
+ *
+ * @param longitude
+ */
+ public void setLongitude(Double longitude) {
+ this.longitude = longitude;
+ }
+
+ /**
+ * @return Retrieves the source of the gazetteer data
+ */
+ public String getSource() {
+ return source;
+ }
+
+ /**
+ * sets the source (the source of the gazetteer data)
+ *
+ * @param source
+ */
+ public void setSource(String source) {
+ this.source = source;
+ }
+
+ /**
+ * @return Retrieves all the other fields in the gazetteer in the form of a map
+ */
+ public Map<String, String> getIndexData() {
+ return indexData;
+ }
+
+ /**
+ * sets the other fields in the gazetteer in the form of a map
+ *
+ * @param indexData stores all fields in the index as fieldname:value
+ */
+ public void setIndexData(Map<String, String> indexData) {
+ this.indexData = indexData;
+ }
+
+ @Override
+ public String toString() {
+
+ return super.toString() + "\n\t\tGazateerEntry\n" + "\t\tlatitude=" + latitude + ", \n\t\tlongitude=" + longitude + ", \n\t\tsource=" + source + ", \n\t\tindexID=" + indexID + ",\n\t\tindexData=" + indexData + "\n";
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 5;
+ hash = 71 * hash + Objects.hashCode(this.source);
+ hash = 71 * hash + Objects.hashCode(this.indexID);
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final GazetteerEntry other = (GazetteerEntry) obj;
+ if (!Objects.equals(this.source, other.source)) {
+ return false;
+ }
+ if (!Objects.equals(this.indexID, other.indexID)) {
+ return false;
+ }
+ return true;
+ }
+
+
+ public String getCountryCode() {
+ return countryCode;
+ }
+
+ public void setCountryCode(String countryCode) {
+ this.countryCode = countryCode;
+ }
+
+ public String getProvinceCode() {
+ return provinceCode;
+ }
+
+ public void setProvinceCode(String provinceCode) {
+ this.provinceCode = provinceCode;
+ }
+
+ public String getHierarchy() {
+ return hierarchy;
+ }
+
+ public void setHierarchy(String hierarchy) {
+ this.hierarchy = hierarchy;
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearchCache.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearchCache.java
index ac5b01e..7d13c81 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearchCache.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearchCache.java
@@ -1,49 +1,49 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- *
- * Caches gazateer query results statically. Clears itself if more than 10000 results are cached.
- */
-public class GazetteerSearchCache {
-
- private static Map<String, ArrayList<GazetteerEntry>> gazCache = new HashMap<>();
-
-/**
- * returns the cached entries. Returns null if the query does not exist in the cache
- * @param searchString
- * @return
- */
- public static synchronized ArrayList<GazetteerEntry> get(String searchString) {
- return gazCache.get(searchString);
- }
-
- public static synchronized void put(String searchString, ArrayList<GazetteerEntry> hits) {
- if (gazCache.size() > 10000) {
- gazCache.clear();
- }
- if (!gazCache.containsKey(searchString)) {
- gazCache.put(searchString, hits);
- }
- }
-
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ *
+ * Caches gazateer query results statically. Clears itself if more than 10000 results are cached.
+ */
+public class GazetteerSearchCache {
+
+ private static Map<String, ArrayList<GazetteerEntry>> gazCache = new HashMap<>();
+
+/**
+ * returns the cached entries. Returns null if the query does not exist in the cache
+ * @param searchString
+ * @return
+ */
+ public static synchronized ArrayList<GazetteerEntry> get(String searchString) {
+ return gazCache.get(searchString);
+ }
+
+ public static synchronized void put(String searchString, ArrayList<GazetteerEntry> hits) {
+ if (gazCache.size() > 10000) {
+ gazCache.clear();
+ }
+ if (!gazCache.containsKey(searchString)) {
+ gazCache.put(searchString, hits);
+ }
+ }
+
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearcher.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearcher.java
index 06e0365..4f5ad5d 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearcher.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearcher.java
@@ -1,251 +1,251 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.queryparser.classic.ParseException;
-
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
-import org.apache.lucene.analysis.util.CharArraySet;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- * Searches Gazetteers stored in a MMapDirectory Lucene index. The structure of
- * these indices are based on loading the indexes using the GazetteerIndexer
- *
- */
-public class GazetteerSearcher {
-
- private final String REGEX_CLEAN = "[^\\p{L}\\p{Nd}]";
- private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private double scoreCutoff = .70;
- private final boolean doubleQuoteAllSearchTerms = false;
- private boolean useHierarchyField = false;
-
- private final EntityLinkerProperties properties;
-
- private Directory opennlpIndex;//= new MMapDirectory(new File(indexloc));
- private IndexReader opennlpReader;// = DirectoryReader.open(geonamesIndex);
- private IndexSearcher opennlpSearcher;// = new IndexSearcher(geonamesReader);
- private Analyzer opennlpAnalyzer;
-
- public static void main(String[] args) {
- try {
- boolean b = true;
- new GazetteerSearcher(new EntityLinkerProperties(new File("c:\\temp\\entitylinker.properties"))).find("alabama", 5, " countrycode:us AND gazsource:usgs");
- } catch (IOException ex) {
- LOG.error(ex.getLocalizedMessage(), ex);
- }
- }
-
- public GazetteerSearcher(EntityLinkerProperties properties) throws IOException {
- this.properties = properties;
- init();
- }
-
- /**
- * Searches the single lucene index that includes the location hierarchy.
- *
- * @param searchString the location name to search for
- * @param rowsReturned how many index entries to return (top N...)
- * @param whereClause the conditional statement that defines the index type
- * and the country oode.
- * @return
- */
- public ArrayList<GazetteerEntry> find(String searchString, int rowsReturned, String whereClause) {
- ArrayList<GazetteerEntry> linkedData = new ArrayList<>();
- searchString = cleanInput(searchString);
- if (searchString.isEmpty()) {
- return linkedData;
- }
- try {
- /*
- * build the search string Sometimes no country context is found. In this
- * case the code variables will be empty strings
- */
- String placeNameQueryString = "placename:(" + searchString.toLowerCase() + ") " + "AND " + whereClause;
- if (searchString.trim().contains(" ") && useHierarchyField) {
- placeNameQueryString = "(placename:(" + searchString.toLowerCase() + ") AND hierarchy:(" + formatForHierarchy(searchString) + "))"
- + " AND " + whereClause;
- }
-
- /*
- * check the cache and go no further if the records already exist
- */
- ArrayList<GazetteerEntry> get = GazetteerSearchCache.get(placeNameQueryString);
- if (get != null) {
-
- return get;
- }
- /*
- * search the placename
- */
- QueryParser parser = new QueryParser(placeNameQueryString, opennlpAnalyzer);
- Query q = parser.parse(placeNameQueryString);
- //Filter filter = new QueryWrapperFilter(new QueryParser(Version.LUCENE_48, whereClause, opennlpAnalyzer).parse(whereClause));
-
- TopDocs bestDocs = opennlpSearcher.search(q, rowsReturned);
- Double maxscore = 0d;
- for (int i = 0; i < bestDocs.scoreDocs.length; ++i) {
- int docId = bestDocs.scoreDocs[i].doc;
- double sc = bestDocs.scoreDocs[i].score;
- if (maxscore.compareTo(sc) < 0) {
- maxscore = sc;
- }
- Document d = opennlpSearcher.doc(docId);
- List<IndexableField> fields = d.getFields();
-
- String lat = d.get("latitude");
- String lon = d.get("longitude");
- String placename = d.get("placename");
- String parentid = d.get("countrycode").toLowerCase();
- String provid = d.get("admincode");
- String itemtype = d.get("loctype");
- String source = d.get("gazsource");
- String hier = d.get("hierarchy");
-
- GazetteerEntry ge = new GazetteerEntry(parentid, String.valueOf(docId), placename, itemtype);
- ge.getScoreMap().put("lucene", sc);
- ge.setIndexID(String.valueOf(docId));
- ge.setSource(source);
- ge.setLatitude(Double.valueOf(lat));
- ge.setLongitude(Double.valueOf(lon));
- ge.setProvinceCode(provid);
- ge.setCountryCode(parentid);
- ge.setHierarchy(hier);
- for (IndexableField field : fields) {
- ge.getIndexData().put(field.name(), d.get(field.name()));
- }
-
- /*
- * only want hits above the levenshtein thresh. This should be a low
- * thresh due to the use of the hierarchy field in the index
- */
- // if (normLev > scoreCutoff) {
- if (ge.getItemParentID().equalsIgnoreCase(parentid) || parentid.equalsIgnoreCase("")) {
- //make sure we don't produce a duplicate
- if (!linkedData.contains(ge)) {
- linkedData.add(ge);
- /*
- * add the records to the cache for this query
- */
- GazetteerSearchCache.put(placeNameQueryString, linkedData);
- }
- }
- }
-
- } catch (IOException | ParseException ex) {
- LOG.error(ex.getLocalizedMessage(), ex);
- }
-
- return linkedData;
- }
-
- /**
- * Replaces any noise chars with a space, and depending on configuration adds
- * double quotes to the string
- *
- * @param input
- * @return
- */
- private String cleanInput(String input) {
- String output = input.replaceAll(REGEX_CLEAN, " ").trim();
- output = output.replace(" ", " ");
- if (doubleQuoteAllSearchTerms) {
- return "\"" + output + "\"";
- } else {
- return output;
- }
-
- }
-
- private void init() throws IOException {
-
- if (opennlpIndex == null) {
- String indexloc = properties.getProperty("opennlp.geoentitylinker.gaz", "");
- if (indexloc.equals("")) {
- LOG.error("Opennlp combined Gaz directory location not found!");
- }
-
- opennlpIndex = new MMapDirectory(Paths.get(indexloc));
- opennlpReader = DirectoryReader.open(opennlpIndex);
- opennlpSearcher = new IndexSearcher(opennlpReader);
- opennlpAnalyzer
- = //new StandardAnalyzer(Version.LUCENE_48, new CharArraySet(Version.LUCENE_48, new ArrayList(), true));
- new StandardAnalyzer(new CharArraySet(new ArrayList<>(), true));
- Map<String, Analyzer> analyMap = new HashMap<>();
-
- analyMap.put("countrycode", new KeywordAnalyzer());
- analyMap.put("admincode", new KeywordAnalyzer());
- analyMap.put("loctype", new KeywordAnalyzer());
- analyMap.put("countycode", new KeywordAnalyzer());
- analyMap.put("gazsource", new KeywordAnalyzer());
-
- opennlpAnalyzer
- = new PerFieldAnalyzerWrapper(opennlpAnalyzer, analyMap);
-
- String cutoff = properties.getProperty("opennlp.geoentitylinker.gaz.lucenescore.min", String.valueOf(scoreCutoff));
- String usehierarchy = properties.getProperty("opennlp.geoentitylinker.gaz.hierarchyfield", String.valueOf("0"));
- if (cutoff != null && !cutoff.isEmpty()) {
- scoreCutoff = Double.parseDouble(cutoff);
- }
- if (usehierarchy != null && !usehierarchy.isEmpty()) {
- useHierarchyField = Boolean.parseBoolean(usehierarchy);
- }
- // opennlp.geoentitylinker.gaz.doublequote=false
- //opennlp.geoentitylinker.gaz.hierarchyfield=false
-
- }
- }
-
- private String formatForHierarchy(String searchTerm) {
- String[] parts = cleanInput(searchTerm).split(" ");
- String out = "";
- if (parts.length != 0) {
- for (String string : parts) {
- out += string + " AND ";
- }
- out = out.substring(0, out.lastIndexOf(" AND "));
- } else {
- out = cleanInput(searchTerm);
- }
- return out;
- }
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.queryparser.classic.ParseException;
+
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * Searches Gazetteers stored in a MMapDirectory Lucene index. The structure of
+ * these indices are based on loading the indexes using the GazetteerIndexer
+ *
+ */
+public class GazetteerSearcher {
+
+ private final String REGEX_CLEAN = "[^\\p{L}\\p{Nd}]";
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private double scoreCutoff = .70;
+ private final boolean doubleQuoteAllSearchTerms = false;
+ private boolean useHierarchyField = false;
+
+ private final EntityLinkerProperties properties;
+
+ private Directory opennlpIndex;//= new MMapDirectory(new File(indexloc));
+ private IndexReader opennlpReader;// = DirectoryReader.open(geonamesIndex);
+ private IndexSearcher opennlpSearcher;// = new IndexSearcher(geonamesReader);
+ private Analyzer opennlpAnalyzer;
+
+ public static void main(String[] args) {
+ try {
+ boolean b = true;
+ new GazetteerSearcher(new EntityLinkerProperties(new File("c:\\temp\\entitylinker.properties"))).find("alabama", 5, " countrycode:us AND gazsource:usgs");
+ } catch (IOException ex) {
+ LOG.error(ex.getLocalizedMessage(), ex);
+ }
+ }
+
+ public GazetteerSearcher(EntityLinkerProperties properties) throws IOException {
+ this.properties = properties;
+ init();
+ }
+
+ /**
+ * Searches the single lucene index that includes the location hierarchy.
+ *
+ * @param searchString the location name to search for
+ * @param rowsReturned how many index entries to return (top N...)
+ * @param whereClause the conditional statement that defines the index type
+ * and the country oode.
+ * @return
+ */
+ public ArrayList<GazetteerEntry> find(String searchString, int rowsReturned, String whereClause) {
+ ArrayList<GazetteerEntry> linkedData = new ArrayList<>();
+ searchString = cleanInput(searchString);
+ if (searchString.isEmpty()) {
+ return linkedData;
+ }
+ try {
+ /*
+ * build the search string Sometimes no country context is found. In this
+ * case the code variables will be empty strings
+ */
+ String placeNameQueryString = "placename:(" + searchString.toLowerCase() + ") " + "AND " + whereClause;
+ if (searchString.trim().contains(" ") && useHierarchyField) {
+ placeNameQueryString = "(placename:(" + searchString.toLowerCase() + ") AND hierarchy:(" + formatForHierarchy(searchString) + "))"
+ + " AND " + whereClause;
+ }
+
+ /*
+ * check the cache and go no further if the records already exist
+ */
+ ArrayList<GazetteerEntry> get = GazetteerSearchCache.get(placeNameQueryString);
+ if (get != null) {
+
+ return get;
+ }
+ /*
+ * search the placename
+ */
+ QueryParser parser = new QueryParser(placeNameQueryString, opennlpAnalyzer);
+ Query q = parser.parse(placeNameQueryString);
+ //Filter filter = new QueryWrapperFilter(new QueryParser(Version.LUCENE_48, whereClause, opennlpAnalyzer).parse(whereClause));
+
+ TopDocs bestDocs = opennlpSearcher.search(q, rowsReturned);
+ Double maxscore = 0d;
+ for (int i = 0; i < bestDocs.scoreDocs.length; ++i) {
+ int docId = bestDocs.scoreDocs[i].doc;
+ double sc = bestDocs.scoreDocs[i].score;
+ if (maxscore.compareTo(sc) < 0) {
+ maxscore = sc;
+ }
+ Document d = opennlpSearcher.doc(docId);
+ List<IndexableField> fields = d.getFields();
+
+ String lat = d.get("latitude");
+ String lon = d.get("longitude");
+ String placename = d.get("placename");
+ String parentid = d.get("countrycode").toLowerCase();
+ String provid = d.get("admincode");
+ String itemtype = d.get("loctype");
+ String source = d.get("gazsource");
+ String hier = d.get("hierarchy");
+
+ GazetteerEntry ge = new GazetteerEntry(parentid, String.valueOf(docId), placename, itemtype);
+ ge.getScoreMap().put("lucene", sc);
+ ge.setIndexID(String.valueOf(docId));
+ ge.setSource(source);
+ ge.setLatitude(Double.valueOf(lat));
+ ge.setLongitude(Double.valueOf(lon));
+ ge.setProvinceCode(provid);
+ ge.setCountryCode(parentid);
+ ge.setHierarchy(hier);
+ for (IndexableField field : fields) {
+ ge.getIndexData().put(field.name(), d.get(field.name()));
+ }
+
+ /*
+ * only want hits above the levenshtein thresh. This should be a low
+ * thresh due to the use of the hierarchy field in the index
+ */
+ // if (normLev > scoreCutoff) {
+ if (ge.getItemParentID().equalsIgnoreCase(parentid) || parentid.equalsIgnoreCase("")) {
+ //make sure we don't produce a duplicate
+ if (!linkedData.contains(ge)) {
+ linkedData.add(ge);
+ /*
+ * add the records to the cache for this query
+ */
+ GazetteerSearchCache.put(placeNameQueryString, linkedData);
+ }
+ }
+ }
+
+ } catch (IOException | ParseException ex) {
+ LOG.error(ex.getLocalizedMessage(), ex);
+ }
+
+ return linkedData;
+ }
+
+ /**
+ * Replaces any noise chars with a space, and depending on configuration adds
+ * double quotes to the string
+ *
+ * @param input
+ * @return
+ */
+ private String cleanInput(String input) {
+ String output = input.replaceAll(REGEX_CLEAN, " ").trim();
+ output = output.replace(" ", " ");
+ if (doubleQuoteAllSearchTerms) {
+ return "\"" + output + "\"";
+ } else {
+ return output;
+ }
+
+ }
+
+ private void init() throws IOException {
+
+ if (opennlpIndex == null) {
+ String indexloc = properties.getProperty("opennlp.geoentitylinker.gaz", "");
+ if (indexloc.equals("")) {
+ LOG.error("Opennlp combined Gaz directory location not found!");
+ }
+
+ opennlpIndex = new MMapDirectory(Paths.get(indexloc));
+ opennlpReader = DirectoryReader.open(opennlpIndex);
+ opennlpSearcher = new IndexSearcher(opennlpReader);
+ opennlpAnalyzer
+ = //new StandardAnalyzer(Version.LUCENE_48, new CharArraySet(Version.LUCENE_48, new ArrayList(), true));
+ new StandardAnalyzer(new CharArraySet(new ArrayList<>(), true));
+ Map<String, Analyzer> analyMap = new HashMap<>();
+
+ analyMap.put("countrycode", new KeywordAnalyzer());
+ analyMap.put("admincode", new KeywordAnalyzer());
+ analyMap.put("loctype", new KeywordAnalyzer());
+ analyMap.put("countycode", new KeywordAnalyzer());
+ analyMap.put("gazsource", new KeywordAnalyzer());
+
+ opennlpAnalyzer
+ = new PerFieldAnalyzerWrapper(opennlpAnalyzer, analyMap);
+
+ String cutoff = properties.getProperty("opennlp.geoentitylinker.gaz.lucenescore.min", String.valueOf(scoreCutoff));
+ String usehierarchy = properties.getProperty("opennlp.geoentitylinker.gaz.hierarchyfield", String.valueOf("0"));
+ if (cutoff != null && !cutoff.isEmpty()) {
+ scoreCutoff = Double.parseDouble(cutoff);
+ }
+ if (usehierarchy != null && !usehierarchy.isEmpty()) {
+ useHierarchyField = Boolean.parseBoolean(usehierarchy);
+ }
+ // opennlp.geoentitylinker.gaz.doublequote=false
+ //opennlp.geoentitylinker.gaz.hierarchyfield=false
+
+ }
+ }
+
+ private String formatForHierarchy(String searchTerm) {
+ String[] parts = cleanInput(searchTerm).split(" ");
+ String out = "";
+ if (parts.length != 0) {
+ for (String string : parts) {
+ out += string + " AND ";
+ }
+ out = out.substring(0, out.lastIndexOf(" AND "));
+ } else {
+ out = cleanInput(searchTerm);
+ }
+ return out;
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GeoEntityLinker.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GeoEntityLinker.java
index 36de509..7fa49be 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GeoEntityLinker.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GeoEntityLinker.java
@@ -1,220 +1,220 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker;
-
-import java.io.IOException;
-import opennlp.addons.geoentitylinker.scoring.ModelBasedScorer;
-import opennlp.addons.geoentitylinker.scoring.LinkedEntityScorer;
-import opennlp.addons.geoentitylinker.scoring.CountryProximityScorer;
-import opennlp.addons.geoentitylinker.scoring.GeoHashBinningScorer;
-import opennlp.addons.geoentitylinker.scoring.FuzzyStringMatchScorer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import opennlp.addons.geoentitylinker.scoring.PlacetypeScorer;
-import opennlp.addons.geoentitylinker.scoring.ProvinceProximityScorer;
-import opennlp.tools.entitylinker.BaseLink;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.EntityLinker;
-
-/**
- * Links location entities to the USGS and GeoNames gazetteers, and uses several
- * scoring techniques to enable resolution. The gazetteers are stored in lucene
- * indexes. The indexes can be built using the GeoEntityLinkerSetupUtils class
- * in this same package.
- */
-public class GeoEntityLinker implements EntityLinker<LinkedSpan> {
-
- private static Integer topN = 2;
- private AdminBoundaryContextGenerator countryContext;
- private EntityLinkerProperties linkerProperties;
- private GazetteerSearcher gazateerSearcher;
- private final List<LinkedEntityScorer<AdminBoundaryContext>> scorers = new ArrayList<>();
-
- @Override
- public List<LinkedSpan> find(String doctext, Span[] sentences, Span[][] tokensBySentence, Span[][] namesBySentence) {
- ArrayList<LinkedSpan> spans = new ArrayList<>();
-
- if (linkerProperties == null) {
- throw new IllegalArgumentException("EntityLinkerProperties cannot be null");
- }
- //countryMentions = countryContext.regexfind(doctext);
- AdminBoundaryContext context = countryContext.getContext(doctext);
- for (int s = 0; s < sentences.length; s++) {
- Span[] names = namesBySentence[s];
-
- Span[] tokenSpans = tokensBySentence[s];
- String[] tokens = Span.spansToStrings(tokenSpans, sentences[s].getCoveredText(doctext));
-
- String[] matches = Span.spansToStrings(names, tokens);
-
- for (int i = 0; i < matches.length; i++) {
-
- ArrayList<BaseLink> geoNamesEntries = new ArrayList<>();
- if (!context.getWhereClauses().isEmpty()) {
- for (String whereclause : context.getWhereClauses()) {
- ArrayList<GazetteerEntry> find = gazateerSearcher.find(matches[i], topN, whereclause);
- for (GazetteerEntry gazetteerEntry : find) {
- if (!geoNamesEntries.contains(gazetteerEntry)) {
- geoNamesEntries.add(gazetteerEntry);
- }
- }
-
- }
- } else {//this means there were no where clauses generated so the where clause will default to look at the entire index
- ArrayList<GazetteerEntry> find = gazateerSearcher.find(matches[i], topN, " gaztype:usgs geonames regions ");
- for (GazetteerEntry gazetteerEntry : find) {
- if (!geoNamesEntries.contains(gazetteerEntry)) {
- geoNamesEntries.add(gazetteerEntry);
- }
- }
- }
- if (geoNamesEntries.isEmpty()) {
- continue;
- }
- /*
- * Normalize the returned scores for this name... this will assist the
- * sort
- */
- if (!spans.isEmpty()) {
-
- Double maxscore = 0d;
- for (BaseLink gazetteerEntry : geoNamesEntries) {
- Double deNormScore = gazetteerEntry.getScoreMap().get("lucene");
- if (deNormScore.compareTo(maxscore) > 0) {
- maxscore = deNormScore;
- }
- }
- for (BaseLink gazetteerEntry : geoNamesEntries) {
- Double deNormScore = gazetteerEntry.getScoreMap().get("lucene");
- Double normalize = normalize(deNormScore, 0d, maxscore);
- gazetteerEntry.getScoreMap().put("normlucene", normalize);
- }
- }
- LinkedSpan<BaseLink> newspan = new LinkedSpan<>(geoNamesEntries, names[i], 0);
- newspan.setSearchTerm(matches[i]);
- newspan.setLinkedEntries(geoNamesEntries);
- newspan.setSentenceid(s);
- spans.add(newspan);
- }
-
- }
-
- if (!scorers.isEmpty()) {
- for (LinkedEntityScorer scorer : scorers) {
- scorer.score(spans, doctext, sentences, linkerProperties, context);
- }
- }
- /*
- * sort the data with the best score on top based on the sum of the scores
- * below from the score map for each baselink object
- */
- for (LinkedSpan<BaseLink> s : spans) {
- ArrayList<BaseLink> linkedData = s.getLinkedEntries();
- linkedData.sort(Collections.reverseOrder((o1, o2) -> {
- Map<String, Double> o1scoreMap = o1.getScoreMap();
- Map<String, Double> o2scoreMap = o2.getScoreMap();
- if (o1scoreMap.size() != o2scoreMap.size()) {
- return 0;
- }
- double sumo1 = 0d;
- double sumo2 = 0d;
- for (String object : o1scoreMap.keySet()) {
- if (object.equals("typescore")
- || object.equals("countrycontext")
- || object.equals("placenamedicecoef")
- || object.equals("provincecontext")
- || object.equals("geohashbin")
- || object.equals("normlucene")) {
- sumo1 += o1scoreMap.get(object);
- sumo2 += o2scoreMap.get(object);
- }
- }
-
- return Double.compare(sumo1,
- sumo2);
- }));
- //prune the list to topN
- Iterator<BaseLink> iterator = linkedData.iterator();
- int n = 0;
- while (iterator.hasNext()) {
- if (n >= topN) {
- iterator.remove();
- }
- iterator.next();
- n++;
- }
- }
-
- return spans;
- }
-
- /**
- * transposes a value within one range to a relative value in a different
- * range. Used to normalize distances in this class.
- *
- * @param valueToNormalize the value to place within the new range
- * @param minimum the min of the set to be transposed
- * @param maximum the max of the set to be transposed
- * @return
- */
- private Double normalize(Double valueToNormalize, double minimum, double maximum) {
- double d = ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
- d = Double.isNaN(d) ? 0d : d;
- return d;
- }
-
- private void loadScorers() {
- if (scorers.isEmpty()) {
- scorers.add(new ProvinceProximityScorer());
- scorers.add(new GeoHashBinningScorer());
- scorers.add(new CountryProximityScorer());
- scorers.add(new ModelBasedScorer());
- scorers.add(new FuzzyStringMatchScorer());
- scorers.add(new PlacetypeScorer());
- }
- }
-
- @Override
- public void init(EntityLinkerProperties properties) throws IOException {
-
- this.linkerProperties = properties;
- countryContext = new AdminBoundaryContextGenerator(this.linkerProperties);
- gazateerSearcher = new GazetteerSearcher(this.linkerProperties);
- String rowsRetStr = this.linkerProperties.getProperty("opennlp.geoentitylinker.gaz.rowsreturned", "2");
- int rws;
- try {
- rws = Integer.parseInt(rowsRetStr);
- } catch (NumberFormatException e) {
- rws = 2;
- }
- topN = rws;
- loadScorers();
-
- }
-
- @Override
- public List<LinkedSpan> find(String doctext, Span[] sentences, Span[][] tokensBySentence,
- Span[][] namesBySentence, int sentenceIndex) {
- throw new UnsupportedOperationException("The GeoEntityLinker requires the entire document "
- + "for proper scoring. This method is unsupported");
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker;
+
+import java.io.IOException;
+import opennlp.addons.geoentitylinker.scoring.ModelBasedScorer;
+import opennlp.addons.geoentitylinker.scoring.LinkedEntityScorer;
+import opennlp.addons.geoentitylinker.scoring.CountryProximityScorer;
+import opennlp.addons.geoentitylinker.scoring.GeoHashBinningScorer;
+import opennlp.addons.geoentitylinker.scoring.FuzzyStringMatchScorer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import opennlp.addons.geoentitylinker.scoring.PlacetypeScorer;
+import opennlp.addons.geoentitylinker.scoring.ProvinceProximityScorer;
+import opennlp.tools.entitylinker.BaseLink;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.EntityLinker;
+
+/**
+ * Links location entities to the USGS and GeoNames gazetteers, and uses several
+ * scoring techniques to enable resolution. The gazetteers are stored in lucene
+ * indexes. The indexes can be built using the GeoEntityLinkerSetupUtils class
+ * in this same package.
+ */
+public class GeoEntityLinker implements EntityLinker<LinkedSpan> {
+
+ private static Integer topN = 2;
+ private AdminBoundaryContextGenerator countryContext;
+ private EntityLinkerProperties linkerProperties;
+ private GazetteerSearcher gazateerSearcher;
+ private final List<LinkedEntityScorer<AdminBoundaryContext>> scorers = new ArrayList<>();
+
+ @Override
+ public List<LinkedSpan> find(String doctext, Span[] sentences, Span[][] tokensBySentence, Span[][] namesBySentence) {
+ ArrayList<LinkedSpan> spans = new ArrayList<>();
+
+ if (linkerProperties == null) {
+ throw new IllegalArgumentException("EntityLinkerProperties cannot be null");
+ }
+ //countryMentions = countryContext.regexfind(doctext);
+ AdminBoundaryContext context = countryContext.getContext(doctext);
+ for (int s = 0; s < sentences.length; s++) {
+ Span[] names = namesBySentence[s];
+
+ Span[] tokenSpans = tokensBySentence[s];
+ String[] tokens = Span.spansToStrings(tokenSpans, sentences[s].getCoveredText(doctext));
+
+ String[] matches = Span.spansToStrings(names, tokens);
+
+ for (int i = 0; i < matches.length; i++) {
+
+ ArrayList<BaseLink> geoNamesEntries = new ArrayList<>();
+ if (!context.getWhereClauses().isEmpty()) {
+ for (String whereclause : context.getWhereClauses()) {
+ ArrayList<GazetteerEntry> find = gazateerSearcher.find(matches[i], topN, whereclause);
+ for (GazetteerEntry gazetteerEntry : find) {
+ if (!geoNamesEntries.contains(gazetteerEntry)) {
+ geoNamesEntries.add(gazetteerEntry);
+ }
+ }
+
+ }
+ } else {//this means there were no where clauses generated so the where clause will default to look at the entire index
+ ArrayList<GazetteerEntry> find = gazateerSearcher.find(matches[i], topN, " gaztype:usgs geonames regions ");
+ for (GazetteerEntry gazetteerEntry : find) {
+ if (!geoNamesEntries.contains(gazetteerEntry)) {
+ geoNamesEntries.add(gazetteerEntry);
+ }
+ }
+ }
+ if (geoNamesEntries.isEmpty()) {
+ continue;
+ }
+ /*
+ * Normalize the returned scores for this name... this will assist the
+ * sort
+ */
+ if (!spans.isEmpty()) {
+
+ Double maxscore = 0d;
+ for (BaseLink gazetteerEntry : geoNamesEntries) {
+ Double deNormScore = gazetteerEntry.getScoreMap().get("lucene");
+ if (deNormScore.compareTo(maxscore) > 0) {
+ maxscore = deNormScore;
+ }
+ }
+ for (BaseLink gazetteerEntry : geoNamesEntries) {
+ Double deNormScore = gazetteerEntry.getScoreMap().get("lucene");
+ Double normalize = normalize(deNormScore, 0d, maxscore);
+ gazetteerEntry.getScoreMap().put("normlucene", normalize);
+ }
+ }
+ LinkedSpan<BaseLink> newspan = new LinkedSpan<>(geoNamesEntries, names[i], 0);
+ newspan.setSearchTerm(matches[i]);
+ newspan.setLinkedEntries(geoNamesEntries);
+ newspan.setSentenceid(s);
+ spans.add(newspan);
+ }
+
+ }
+
+ if (!scorers.isEmpty()) {
+ for (LinkedEntityScorer scorer : scorers) {
+ scorer.score(spans, doctext, sentences, linkerProperties, context);
+ }
+ }
+ /*
+ * sort the data with the best score on top based on the sum of the scores
+ * below from the score map for each baselink object
+ */
+ for (LinkedSpan<BaseLink> s : spans) {
+ ArrayList<BaseLink> linkedData = s.getLinkedEntries();
+ linkedData.sort(Collections.reverseOrder((o1, o2) -> {
+ Map<String, Double> o1scoreMap = o1.getScoreMap();
+ Map<String, Double> o2scoreMap = o2.getScoreMap();
+ if (o1scoreMap.size() != o2scoreMap.size()) {
+ return 0;
+ }
+ double sumo1 = 0d;
+ double sumo2 = 0d;
+ for (String object : o1scoreMap.keySet()) {
+ if (object.equals("typescore")
+ || object.equals("countrycontext")
+ || object.equals("placenamedicecoef")
+ || object.equals("provincecontext")
+ || object.equals("geohashbin")
+ || object.equals("normlucene")) {
+ sumo1 += o1scoreMap.get(object);
+ sumo2 += o2scoreMap.get(object);
+ }
+ }
+
+ return Double.compare(sumo1,
+ sumo2);
+ }));
+ //prune the list to topN
+ Iterator<BaseLink> iterator = linkedData.iterator();
+ int n = 0;
+ while (iterator.hasNext()) {
+ if (n >= topN) {
+ iterator.remove();
+ }
+ iterator.next();
+ n++;
+ }
+ }
+
+ return spans;
+ }
+
+ /**
+ * transposes a value within one range to a relative value in a different
+ * range. Used to normalize distances in this class.
+ *
+ * @param valueToNormalize the value to place within the new range
+ * @param minimum the min of the set to be transposed
+ * @param maximum the max of the set to be transposed
+ * @return
+ */
+ private Double normalize(Double valueToNormalize, double minimum, double maximum) {
+ double d = ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
+ d = Double.isNaN(d) ? 0d : d;
+ return d;
+ }
+
+ private void loadScorers() {
+ if (scorers.isEmpty()) {
+ scorers.add(new ProvinceProximityScorer());
+ scorers.add(new GeoHashBinningScorer());
+ scorers.add(new CountryProximityScorer());
+ scorers.add(new ModelBasedScorer());
+ scorers.add(new FuzzyStringMatchScorer());
+ scorers.add(new PlacetypeScorer());
+ }
+ }
+
+ @Override
+ public void init(EntityLinkerProperties properties) throws IOException {
+
+ this.linkerProperties = properties;
+ countryContext = new AdminBoundaryContextGenerator(this.linkerProperties);
+ gazateerSearcher = new GazetteerSearcher(this.linkerProperties);
+ String rowsRetStr = this.linkerProperties.getProperty("opennlp.geoentitylinker.gaz.rowsreturned", "2");
+ int rws;
+ try {
+ rws = Integer.parseInt(rowsRetStr);
+ } catch (NumberFormatException e) {
+ rws = 2;
+ }
+ topN = rws;
+ loadScorers();
+
+ }
+
+ @Override
+ public List<LinkedSpan> find(String doctext, Span[] sentences, Span[][] tokensBySentence,
+ Span[][] namesBySentence, int sentenceIndex) {
+ throw new UnsupportedOperationException("The GeoEntityLinker requires the entire document "
+ + "for proper scoring. This method is unsupported");
+ }
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GazetteerIndexer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GazetteerIndexer.java
index 66e7bfe..d41b6ff 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GazetteerIndexer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GazetteerIndexer.java
@@ -1,210 +1,210 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.indexing;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.util.CharArraySet;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
-
-/**
- * Creates two lucene indexes, geonames and usgs for use in GeoEntityLinker.
- */
-public class GazetteerIndexer {
-
- public static void main(String[] args) {
-
- if (args.length != 8) {
- System.out.println("Usage: GazetteerIndexer geonamesData geoNamesCountryInfo geonamesAdmin1CodesASCII "
- + "usgsDataFile usgsGovUnitsFile outputIndexDir outputCountryContextFile regionsFile");
- System.out.println();
- System.out.println("The GazetteerIndexer.index methods javadoc explains how to retrieve the data files.");
- return;
- }
-
- File geonamesData = new File(args[0]);
- File geoNamesCountryInfo = new File(args[1]);
- File geonamesAdmin1CodesASCII = new File(args[2]);
- File usgsDataFile = new File(args[3]);
- File usgsGovUnitsFile = new File(args[4]);
- File outputIndexDir = new File(args[5]);
- File outputCountryContextFile = new File(args[6]);
- File regionsFile = new File(args[7]);
-
- try {
- GazetteerIndexer i = new GazetteerIndexer();
- i.index(geonamesData,
- geoNamesCountryInfo,
- geonamesAdmin1CodesASCII,
- usgsDataFile,
- usgsGovUnitsFile,
- outputIndexDir,
- outputCountryContextFile,
- regionsFile);
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
-
- public GazetteerIndexer() {
-
- }
-
- public interface Separable {
-
- String getSeparator();
- }
-
- public enum GazType implements Separable {
-
- GEONAMES {
- @Override
- public String toString() {
- return "/opennlp_geoentitylinker_geonames_idx";
- }
-
- @Override
- public String getSeparator() {
- return "\t";
- }
- },
- USGS {
- @Override
- public String toString() {
- return "/opennlp_geoentitylinker_usgsgaz_idx";
- }
-
- @Override
- public String getSeparator() {
- return "\\|";
- }
- }
- }
-
- /**
- *
- * @param geonamesData the actual Geonames gazetteer data downloaded from
- * <a href="https://download.geonames.org/export/dump/">here</a>, then click on this link
- * 'allCountries.zip'
- *
- * @param geoNamesCountryInfo the countryinfo lookup table that can be
- * downloaded from
- * <a href="https://download.geonames.org/export/dump/countryInfo.txt">here</a>. You'll need to
- * copy the page into a file or scrape it
- *
- * @param geonamesAdmin1CodesASCII The lookup data for the province names for
- * each place found
- * <a href="https://download.geonames.org/export/dump/admin1CodesASCII.txt">here</a>, highlight the
- * table view, and copy results into a text file. Make sure the tab delimited
- * format is maintained.
- *
- * @param usgsDataFile the actual USGS gazetteer downloaded from
- * <a href="https://geonames.usgs.gov/domestic/download_data.htm">here</a> click on the
- * national_file####.zip link to get all the most recent features
- *
- * @param usgsGovUnitsFile go to
- * <a href="https://geonames.usgs.gov/domestic/download_data.htm">here</a> in the section titled
- * "Topical Gazetteers -- File Format" click on the dropdown list and select
- * "Government Units". The downloaded file is what you need for this param.
- *
- * @param outputIndexDir where you want the final index. Must be a directory, not an actual file.
- *
- * @param outputCountryContextFile The output countrycontext file. This is a
- * very important file used inside the GeoEntityLinker to assist in toponym
- * resolution.
- *
- * @param regionsFile this file contains a list of regions in the following
- * format: tab delimited text with index 0 as the name of the region, index 1
- * as the longitude, and index 2 as the latitude
- *
- * @throws Exception
- */
- public void index(File geonamesData, File geoNamesCountryInfo, File geonamesAdmin1CodesASCII, File usgsDataFile,
- File usgsGovUnitsFile, File outputIndexDir, File outputCountryContextFile, File regionsFile)
- throws Exception {
- if (!outputIndexDir.isDirectory()) {
- throw new IllegalArgumentException("outputIndexDir must be a directory.");
- }
- if (!geonamesData.exists()) {
- throw new FileNotFoundException("geonames data file does not exist");
- }
- if (!geoNamesCountryInfo.exists()) {
- throw new FileNotFoundException("geoNamesCountryCodes data file does not exist");
- }
- if (!geonamesAdmin1CodesASCII.exists()) {
- throw new FileNotFoundException("geonamesAdmin1CodesASCII data file does not exist");
- }
-
- if (!usgsDataFile.exists()) {
- throw new FileNotFoundException("usgsDataFile data file does not exist");
- }
- if (!usgsGovUnitsFile.exists()) {
- throw new FileNotFoundException("usgsGovUnitsFile data file does not exist");
- }
- if (!outputIndexDir.exists()) {
- throw new FileNotFoundException("outputIndexDir data file does not exist");
- }
- if (!regionsFile.exists()) {
- throw new FileNotFoundException("regionsFile data file does not exist");
- }
-
- String indexloc = outputIndexDir.getPath() + "/opennlp_geoentitylinker_gazetteer";
- Directory index = new MMapDirectory(Paths.get(indexloc));
- Analyzer a = new StandardAnalyzer(new CharArraySet(new ArrayList<>(), true));
- Map<String, Analyzer> analyMap = new HashMap<>();
-
- analyMap.put("countrycode", new KeywordAnalyzer());
- analyMap.put("admincode", new KeywordAnalyzer());
- analyMap.put("loctype", new KeywordAnalyzer());
- analyMap.put("countycode", new KeywordAnalyzer());
- analyMap.put("gazsource", new KeywordAnalyzer());
-
- PerFieldAnalyzerWrapper aWrapper = new PerFieldAnalyzerWrapper(a, analyMap);
- IndexWriterConfig config = new IndexWriterConfig(aWrapper);
- try (IndexWriter w = new IndexWriter(index, config)) {
- //write the column headers for the countryContextFile
- try (FileWriter countryContextFileWriter = new FileWriter(outputCountryContextFile, false)) {
- String colNamesForCountryContextFile = "countrycode\tprovcode\tcountycode\tcountryname\tprovincename\tcountyname\tcountryregex\tprovregex\tcountyregex\n";
- countryContextFileWriter.write(colNamesForCountryContextFile);
- countryContextFileWriter.flush();
- }
-
- USGSProcessor.process(usgsGovUnitsFile, usgsDataFile, outputCountryContextFile, w);
- GeonamesProcessor.process(geoNamesCountryInfo, geonamesAdmin1CodesASCII, geonamesData, outputCountryContextFile, w);
-
- RegionProcessor.process(regionsFile, outputCountryContextFile, w);
- w.commit();
- }
-
- System.out.println("\nIndexing complete. Be sure to add '" + indexloc + "' and context file '" +
- outputCountryContextFile.getPath() + "' to entitylinker.properties file");
- }
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.indexing;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+
+/**
+ * Creates two lucene indexes, geonames and usgs for use in GeoEntityLinker.
+ */
+public class GazetteerIndexer {
+
+ public static void main(String[] args) {
+
+ if (args.length != 8) {
+ System.out.println("Usage: GazetteerIndexer geonamesData geoNamesCountryInfo geonamesAdmin1CodesASCII "
+ + "usgsDataFile usgsGovUnitsFile outputIndexDir outputCountryContextFile regionsFile");
+ System.out.println();
+ System.out.println("The GazetteerIndexer.index methods javadoc explains how to retrieve the data files.");
+ return;
+ }
+
+ File geonamesData = new File(args[0]);
+ File geoNamesCountryInfo = new File(args[1]);
+ File geonamesAdmin1CodesASCII = new File(args[2]);
+ File usgsDataFile = new File(args[3]);
+ File usgsGovUnitsFile = new File(args[4]);
+ File outputIndexDir = new File(args[5]);
+ File outputCountryContextFile = new File(args[6]);
+ File regionsFile = new File(args[7]);
+
+ try {
+ GazetteerIndexer i = new GazetteerIndexer();
+ i.index(geonamesData,
+ geoNamesCountryInfo,
+ geonamesAdmin1CodesASCII,
+ usgsDataFile,
+ usgsGovUnitsFile,
+ outputIndexDir,
+ outputCountryContextFile,
+ regionsFile);
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ public GazetteerIndexer() {
+
+ }
+
+ public interface Separable {
+
+ String getSeparator();
+ }
+
+ public enum GazType implements Separable {
+
+ GEONAMES {
+ @Override
+ public String toString() {
+ return "/opennlp_geoentitylinker_geonames_idx";
+ }
+
+ @Override
+ public String getSeparator() {
+ return "\t";
+ }
+ },
+ USGS {
+ @Override
+ public String toString() {
+ return "/opennlp_geoentitylinker_usgsgaz_idx";
+ }
+
+ @Override
+ public String getSeparator() {
+ return "\\|";
+ }
+ }
+ }
+
+ /**
+ *
+ * @param geonamesData the actual Geonames gazetteer data downloaded from
+ * <a href="https://download.geonames.org/export/dump/">here</a>, then click on this link
+ * 'allCountries.zip'
+ *
+ * @param geoNamesCountryInfo the countryinfo lookup table that can be
+ * downloaded from
+ * <a href="https://download.geonames.org/export/dump/countryInfo.txt">here</a>. You'll need to
+ * copy the page into a file or scrape it
+ *
+ * @param geonamesAdmin1CodesASCII The lookup data for the province names for
+ * each place found
+ * <a href="https://download.geonames.org/export/dump/admin1CodesASCII.txt">here</a>, highlight the
+ * table view, and copy results into a text file. Make sure the tab delimited
+ * format is maintained.
+ *
+ * @param usgsDataFile the actual USGS gazetteer downloaded from
+ * <a href="https://geonames.usgs.gov/domestic/download_data.htm">here</a> click on the
+ * national_file####.zip link to get all the most recent features
+ *
+ * @param usgsGovUnitsFile go to
+ * <a href="https://geonames.usgs.gov/domestic/download_data.htm">here</a> in the section titled
+ * "Topical Gazetteers -- File Format" click on the dropdown list and select
+ * "Government Units". The downloaded file is what you need for this param.
+ *
+ * @param outputIndexDir where you want the final index. Must be a directory, not an actual file.
+ *
+ * @param outputCountryContextFile The output countrycontext file. This is a
+ * very important file used inside the GeoEntityLinker to assist in toponym
+ * resolution.
+ *
+ * @param regionsFile this file contains a list of regions in the following
+ * format: tab delimited text with index 0 as the name of the region, index 1
+ * as the longitude, and index 2 as the latitude
+ *
+ * @throws Exception
+ */
+ public void index(File geonamesData, File geoNamesCountryInfo, File geonamesAdmin1CodesASCII, File usgsDataFile,
+ File usgsGovUnitsFile, File outputIndexDir, File outputCountryContextFile, File regionsFile)
+ throws Exception {
+ if (!outputIndexDir.isDirectory()) {
+ throw new IllegalArgumentException("outputIndexDir must be a directory.");
+ }
+ if (!geonamesData.exists()) {
+ throw new FileNotFoundException("geonames data file does not exist");
+ }
+ if (!geoNamesCountryInfo.exists()) {
+ throw new FileNotFoundException("geoNamesCountryCodes data file does not exist");
+ }
+ if (!geonamesAdmin1CodesASCII.exists()) {
+ throw new FileNotFoundException("geonamesAdmin1CodesASCII data file does not exist");
+ }
+
+ if (!usgsDataFile.exists()) {
+ throw new FileNotFoundException("usgsDataFile data file does not exist");
+ }
+ if (!usgsGovUnitsFile.exists()) {
+ throw new FileNotFoundException("usgsGovUnitsFile data file does not exist");
+ }
+ if (!outputIndexDir.exists()) {
+ throw new FileNotFoundException("outputIndexDir data file does not exist");
+ }
+ if (!regionsFile.exists()) {
+ throw new FileNotFoundException("regionsFile data file does not exist");
+ }
+
+ String indexloc = outputIndexDir.getPath() + "/opennlp_geoentitylinker_gazetteer";
+ Directory index = new MMapDirectory(Paths.get(indexloc));
+ Analyzer a = new StandardAnalyzer(new CharArraySet(new ArrayList<>(), true));
+ Map<String, Analyzer> analyMap = new HashMap<>();
+
+ analyMap.put("countrycode", new KeywordAnalyzer());
+ analyMap.put("admincode", new KeywordAnalyzer());
+ analyMap.put("loctype", new KeywordAnalyzer());
+ analyMap.put("countycode", new KeywordAnalyzer());
+ analyMap.put("gazsource", new KeywordAnalyzer());
+
+ PerFieldAnalyzerWrapper aWrapper = new PerFieldAnalyzerWrapper(a, analyMap);
+ IndexWriterConfig config = new IndexWriterConfig(aWrapper);
+ try (IndexWriter w = new IndexWriter(index, config)) {
+ //write the column headers for the countryContextFile
+ try (FileWriter countryContextFileWriter = new FileWriter(outputCountryContextFile, false)) {
+ String colNamesForCountryContextFile = "countrycode\tprovcode\tcountycode\tcountryname\tprovincename\tcountyname\tcountryregex\tprovregex\tcountyregex\n";
+ countryContextFileWriter.write(colNamesForCountryContextFile);
+ countryContextFileWriter.flush();
+ }
+
+ USGSProcessor.process(usgsGovUnitsFile, usgsDataFile, outputCountryContextFile, w);
+ GeonamesProcessor.process(geoNamesCountryInfo, geonamesAdmin1CodesASCII, geonamesData, outputCountryContextFile, w);
+
+ RegionProcessor.process(regionsFile, outputCountryContextFile, w);
+ w.commit();
+ }
+
+ System.out.println("\nIndexing complete. Be sure to add '" + indexloc + "' and context file '" +
+ outputCountryContextFile.getPath() + "' to entitylinker.properties file");
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesFileDownloader.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesFileDownloader.java
index 7d27e3f..58d680c 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesFileDownloader.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesFileDownloader.java
@@ -1,120 +1,120 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.indexing;
-
-import java.io.BufferedOutputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URL;
-import java.util.Enumeration;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
-
-public class GeonamesFileDownloader {
-
- final static int size = 1024;
- private static final String ALL_COUNTRIES = "http://download.geonames.org/export/dump/ZM.zip";
- private static final String COUNTRY_INFO = "";
- private static final String ADM1_LOOKUP = "";
-
- public static void main(String[] args) {
- downloadGeonamesFiles(COUNTRY_INFO, "c:\\temp\\gazetteers");
- }
-
- public static void downloadGeonamesFiles(String outputFileName, String outputDir) {
- String fileDownload = fileDownload(ALL_COUNTRIES, outputDir);
-
- unzipMyZip(fileDownload, outputDir);
-
- fileDownload(COUNTRY_INFO, outputDir);
- fileDownload(ADM1_LOOKUP, outputDir);
-
- }
-
- public static void writeFile(InputStream in, OutputStream out)
- throws IOException {
- byte[] buffer = new byte[1024];
- int len;
-
- while ((len = in.read(buffer)) != 0) {
- out.write(buffer, 0, len);
- }
-
- in.close();
- out.close();
- }
-
- public static void unzipMyZip(String zipFileName, String directoryToExtractTo) {
- Enumeration<? extends ZipEntry> entriesEnum;
- ZipFile zip;
- try {
- zip = new ZipFile(zipFileName);
- entriesEnum = zip.entries();
- while (entriesEnum.hasMoreElements()) {
- ZipEntry entry = entriesEnum.nextElement();
- InputStream is = zip.getInputStream(entry); // get the input stream
- OutputStream os = new FileOutputStream(zipFileName.replace("\\.zip", ".txt"));
- byte[] buf = new byte[4096];
- int r;
- while ((r = is.read(buf)) != -1) {
- os.write(buf, 0, r);
- }
- os.close();
- is.close();
- }
- } catch (IOException ioe) {
- System.err.println("Some Exception Occurred:");
- ioe.printStackTrace();
- }
- }
-
- public static String fileUrl(String fAddress, String localFileName, String destDir) {
- String filename = destDir + "\\" + localFileName;
- try (InputStream is = new URL(fAddress).openConnection().getInputStream();
- OutputStream outStream = new BufferedOutputStream(new FileOutputStream(destDir + "\\" + localFileName))) {
-
- byte[] buf = new byte[size];
- int byteRead, byteWritten = 0;
- while ((byteRead = is.read(buf)) != -1) {
- outStream.write(buf, 0, byteRead);
- byteWritten += byteRead;
- }
- System.out.println("Downloaded Successfully.");
- System.out.println("File name:\"" + localFileName + "\"\nNo ofbytes :" + byteWritten);
- } catch (Exception e) {
- e.printStackTrace();
- }
- return filename;
- }
-
- public static String fileDownload(String fAddress, String destinationDir) {
- int slashIndex = fAddress.lastIndexOf('/');
- int periodIndex = fAddress.lastIndexOf('.');
-
- String fileName = fAddress.substring(slashIndex + 1);
- String retFileName = "";
- if (periodIndex >= 1 && slashIndex >= 0
- && slashIndex < fAddress.length() - 1) {
- retFileName = fileUrl(fAddress, fileName, destinationDir);
- } else {
- System.err.println("path or file name.");
- }
- return retFileName;
- }
-
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.indexing;
+
+import java.io.BufferedOutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.Enumeration;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+public class GeonamesFileDownloader {
+
+ final static int size = 1024;
+ private static final String ALL_COUNTRIES = "http://download.geonames.org/export/dump/ZM.zip";
+ private static final String COUNTRY_INFO = "";
+ private static final String ADM1_LOOKUP = "";
+
+ public static void main(String[] args) {
+ downloadGeonamesFiles(COUNTRY_INFO, "c:\\temp\\gazetteers");
+ }
+
+ public static void downloadGeonamesFiles(String outputFileName, String outputDir) {
+ String fileDownload = fileDownload(ALL_COUNTRIES, outputDir);
+
+ unzipMyZip(fileDownload, outputDir);
+
+ fileDownload(COUNTRY_INFO, outputDir);
+ fileDownload(ADM1_LOOKUP, outputDir);
+
+ }
+
+ public static void writeFile(InputStream in, OutputStream out)
+ throws IOException {
+ byte[] buffer = new byte[1024];
+ int len;
+
+ while ((len = in.read(buffer)) != 0) {
+ out.write(buffer, 0, len);
+ }
+
+ in.close();
+ out.close();
+ }
+
+ public static void unzipMyZip(String zipFileName, String directoryToExtractTo) {
+ Enumeration<? extends ZipEntry> entriesEnum;
+ ZipFile zip;
+ try {
+ zip = new ZipFile(zipFileName);
+ entriesEnum = zip.entries();
+ while (entriesEnum.hasMoreElements()) {
+ ZipEntry entry = entriesEnum.nextElement();
+ InputStream is = zip.getInputStream(entry); // get the input stream
+ OutputStream os = new FileOutputStream(zipFileName.replace("\\.zip", ".txt"));
+ byte[] buf = new byte[4096];
+ int r;
+ while ((r = is.read(buf)) != -1) {
+ os.write(buf, 0, r);
+ }
+ os.close();
+ is.close();
+ }
+ } catch (IOException ioe) {
+ System.err.println("Some Exception Occurred:");
+ ioe.printStackTrace();
+ }
+ }
+
+ public static String fileUrl(String fAddress, String localFileName, String destDir) {
+ String filename = destDir + "\\" + localFileName;
+ try (InputStream is = new URL(fAddress).openConnection().getInputStream();
+ OutputStream outStream = new BufferedOutputStream(new FileOutputStream(destDir + "\\" + localFileName))) {
+
+ byte[] buf = new byte[size];
+ int byteRead, byteWritten = 0;
+ while ((byteRead = is.read(buf)) != -1) {
+ outStream.write(buf, 0, byteRead);
+ byteWritten += byteRead;
+ }
+ System.out.println("Downloaded Successfully.");
+ System.out.println("File name:\"" + localFileName + "\"\nNo ofbytes :" + byteWritten);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return filename;
+ }
+
+ public static String fileDownload(String fAddress, String destinationDir) {
+ int slashIndex = fAddress.lastIndexOf('/');
+ int periodIndex = fAddress.lastIndexOf('.');
+
+ String fileName = fAddress.substring(slashIndex + 1);
+ String retFileName = "";
+ if (periodIndex >= 1 && slashIndex >= 0
+ && slashIndex < fAddress.length() - 1) {
+ retFileName = fileUrl(fAddress, fileName, destinationDir);
+ } else {
+ System.err.println("path or file name.");
+ }
+ return retFileName;
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesProcessor.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesProcessor.java
index f560d33..c58c92d 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesProcessor.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesProcessor.java
@@ -1,296 +1,296 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.indexing;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import opennlp.addons.geoentitylinker.AdminBoundary;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexWriter;
-
-public class GeonamesProcessor {
-
- public static void process(File countryCodesLookupFile, File adm1CodesLookupFile, File geonamesGazetteerFile, File outputCountryContextFile, IndexWriter w) throws Exception {
- Map<String, String> countryCodes = getCountryCodes(countryCodesLookupFile);
-
- Map<String, AdminBoundary> adm1s = getProvData(adm1CodesLookupFile, countryCodes);
- // List<AdminBoundary> adm2s = getCountryContextFromFile(new File("C:\\temp\\gazetteers\\geonamesdata\\admin2Codes.txt"));
- //admin2Codes.txt
-
- readFile(geonamesGazetteerFile, GazetteerIndexer.GazType.GEONAMES, adm1s, countryCodes, w);
- //now append to the coutnry context file
- writeCountryContextFile(outputCountryContextFile, adm1s);
-
- }
-
- public GeonamesProcessor() {
- }
-
- private static Map<String, AdminBoundary> getProvData(File adm1CodesLookupFile, Map<String, String> ccodes) {
- System.out.println("Attempting to read geonames province data from: " + adm1CodesLookupFile.getPath());
-
- Map<String, AdminBoundary> outmap = new HashMap<>();
- BufferedReader reader;
- Set<String> nullcodes = new HashSet<>();
- try {
-
- reader = new BufferedReader(new FileReader(adm1CodesLookupFile));
- int i = 0;
- String line;
- while ((line = reader.readLine()) != null) {
-
- // String line = reader.readLine();
- String[] values = line.split("\t");
- if (values.length != 4) {
- throw new IOException("improperly formatted province lookup file");
- }
- String ccode = values[0].toLowerCase();
-
- String[] split = ccode.split("\\.");
- String pcode = "";
- if (split.length == 2) {
- //System.out.println(split);
- ccode = split[0];
- pcode = split[1];
- }
-
- String pname = values[2];
-
- if (ccode.matches("[0-9].*")) {
- String code = ccode;
- ccode = pcode;
- pcode = code;
- }
-
- String cname = ccodes.get(ccode);
-
- if (cname == null) {
- nullcodes.add(ccode);
- }
- AdminBoundary data = new AdminBoundary(ccode, cname, pcode, pname, "NO_DATA_FOUND", "NO_DATA_FOUND", cname, pname, "NO_DATA_FOUND");
- // System.out.println(data);
- outmap.put(ccode + "." + pcode, data);
-
- }
- System.out.println("INFO: there were " + nullcodes.size() + " null prov codes. This is due to inconsistencies in reference data.");
- reader.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- System.out.println("Successfully read geonames province data from: " + adm1CodesLookupFile.getPath());
-
- return outmap;
-
- }
-
- private static Map<String, String> getCountryCodes(File countryContextFile) {
- Map<String, String> ccs = new HashMap<>();
- BufferedReader reader;
- try {
-
- reader = new BufferedReader(new FileReader(countryContextFile));
- int i = 0;
- String line;
- boolean start = false;
- while ((line = reader.readLine()) != null) {
- if (!line.toLowerCase().startsWith("#iso\t") && !start) {
-
- continue;
- } else {
- start = true;
- }
- String[] values = line.split("\t");
-
- String ccode = values[0].toLowerCase();//this is the 2 digit ISO code
- String cname = values[4].toLowerCase();
- if (!ccode.equals("")) {
- ccs.put(ccode, cname);
- }
-
- }
- reader.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- ccs.put("SS", "South Sudan");
- ccs.put("CS", "Kosovo");
- return ccs;
-
- }
-
- public static void writeCountryContextFile(File outfile, Map<String, AdminBoundary> adms) {
- // FileWriter writer = null;
- try (FileWriter writer = new FileWriter(outfile, true)) {
- BufferedWriter bw = new BufferedWriter(writer);
-
- for (String admKey : adms.keySet()) {
- AdminBoundary adm = adms.get(admKey);
- if (adm == null) {
- continue;
- }
- String province = adm.getProvinceName();
- String country = adm.getCountryName();
-
- String line = adm.getCountryCode() + "\t" + adm.getProvCode() + "\t" + "\t" + country + "\t" + province + "\t" + "\t" + "(" + country + ")" + "\t"
- + adm.getProvinceName() + "\t" + adm.getCountyName() + "\n";
- bw.write(line);
- // System.out.println(line);
-
- }
-
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- System.out.println("successfully wrote Geonames entries to country oontext file");
- }
-
- /**
- *
- * @param gazateerInputData the Geonames allCounties.txt file
- * @param type the types of gaz entry, usgs, geonames, or regions
- * @param adms the province info
- * @param countrycodes the country code info
- * @param w the lucene index writer
- * @throws Exception
- */
- public static void readFile(File gazateerInputData, GazetteerIndexer.GazType type, Map<String, AdminBoundary> adms, Map<String, String> countrycodes, IndexWriter w) throws Exception {
-
- BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData));
- String[] boosts = "ADM1 ADM1H ADM2 ADM2H ADM3 ADM3H ADM4 ADM4H ADM5 ADMD ADMDH PCLD PCLH PCLI PCLIX TERR PCLIX PPL PPLA PPLA2 PPLA3 PPLA4 PPLC PPLCH PPLF PPLG PPLH PPLL PPLQ PPLR PPLS PPLX STLMT".split(" ");
- Map<String, Float> boostMap = new HashMap<>();
- for (String boost : boosts) {
- boostMap.put(boost.toLowerCase(), 10f);
- }
- String[] fieldStrings = new String[]{
- "geonameid",
- "name",
- "asciiname",
- "alternatenames",
- "latitude",
- "longitude",
- "feature_class",
- "feature_code",
- "country code",
- "cc2",
- "admin1_code",
- "admin2_code",
- "admin3_code",
- "admin4_code",
- "population",
- "elevation",
- "dem ",
- "timezone",
- "modification_date"};
-
- List<String> fields = Arrays.asList(fieldStrings);
- int counter = 0;
- System.out.println("reading gazetteer data from file...........");
- String line;
- while ((line = reader.readLine()) != null) {
- String[] values = line.split(type.getSeparator());
-
- Document doc = new Document();
- String admincode = values[10].toLowerCase();
- String ccode = values[8].toLowerCase();
- if (ccode.contains(",")) {
- String[] codes = ccode.split(",");
- if (codes.length > 0) {
- ccode = codes[0];
- }
- }
- AdminBoundary adm = adms.get(ccode + "." + admincode);
-
- String placeName = values[2];
- String lat = values[4];
- String lon = values[5];
- String dsg = values[7].toLowerCase();
-
- String id = values[0];
- String concatIndexEntry;
- String countryname;
- if (adm != null) {
- concatIndexEntry = adm.getCountryName() + ", " + adm.getProvinceName() + ", " + placeName;
- countryname = adm.getCountryName();
- } else {
- //there is no admin info, but we can still use the countrycode to concat the country name
- String n = countrycodes.get(ccode);
- countryname = n;
- if (n != null) {
- concatIndexEntry = n + ", " + placeName;
- } else {
- ///don't want a single token hierarchy entry.
- concatIndexEntry = "";
- }
- }
- if (ccode == null) {
- System.out.println("naughty country code");
- }
- for (int i = 0; i < fields.size() - 1; i++) {
- doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
-
- }
- if (dsg.equals("pcli")) {
- System.out.println("placename: " + placeName + " RESET TO: " + countryname);
- placeName = countryname;
- }
- /*
- * add standard fields to the index
- */
- doc.add(new TextField("hierarchy", concatIndexEntry, Field.Store.YES));
- doc.add(new TextField("placename", placeName, Field.Store.YES));
- // doc.add(new TextField("countryname", countryname, Field.Store.YES));
- //System.out.println(placeName);
-
- doc.add(new TextField("latitude", lat, Field.Store.YES));
- doc.add(new TextField("longitude", lon, Field.Store.YES));
- doc.add(new StringField("loctype", dsg, Field.Store.YES));
- doc.add(new StringField("admincode", (ccode + "." + admincode).toLowerCase(), Field.Store.YES));
- doc.add(new StringField("countrycode", ccode.toLowerCase(), Field.Store.YES));
- doc.add(new StringField("countycode", "", Field.Store.YES));
- doc.add(new StringField("locid", id, Field.Store.YES));
- placeName = placeName.replace("republic of", "").replace("federative", "");
- if (id.equals("3175395")) {
- System.out.println(placeName);
- }
- doc.add(new StringField("gazsource", "geonames", Field.Store.YES));
-
- w.addDocument(doc);
-
- counter++;
- if (counter % 100000 == 0) {
- w.commit();
- System.out.println(counter + " .........Geonames entries committed to index..............");
- }
-
- }
-
- System.out.println("Completed indexing geonames gaz! index name is: " + type.toString());
- }
-
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.indexing;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import opennlp.addons.geoentitylinker.AdminBoundary;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+
+public class GeonamesProcessor {
+
+ public static void process(File countryCodesLookupFile, File adm1CodesLookupFile, File geonamesGazetteerFile, File outputCountryContextFile, IndexWriter w) throws Exception {
+ Map<String, String> countryCodes = getCountryCodes(countryCodesLookupFile);
+
+ Map<String, AdminBoundary> adm1s = getProvData(adm1CodesLookupFile, countryCodes);
+ // List<AdminBoundary> adm2s = getCountryContextFromFile(new File("C:\\temp\\gazetteers\\geonamesdata\\admin2Codes.txt"));
+ //admin2Codes.txt
+
+ readFile(geonamesGazetteerFile, GazetteerIndexer.GazType.GEONAMES, adm1s, countryCodes, w);
+ //now append to the coutnry context file
+ writeCountryContextFile(outputCountryContextFile, adm1s);
+
+ }
+
+ public GeonamesProcessor() {
+ }
+
+ private static Map<String, AdminBoundary> getProvData(File adm1CodesLookupFile, Map<String, String> ccodes) {
+ System.out.println("Attempting to read geonames province data from: " + adm1CodesLookupFile.getPath());
+
+ Map<String, AdminBoundary> outmap = new HashMap<>();
+ BufferedReader reader;
+ Set<String> nullcodes = new HashSet<>();
+ try {
+
+ reader = new BufferedReader(new FileReader(adm1CodesLookupFile));
+ int i = 0;
+ String line;
+ while ((line = reader.readLine()) != null) {
+
+ // String line = reader.readLine();
+ String[] values = line.split("\t");
+ if (values.length != 4) {
+ throw new IOException("improperly formatted province lookup file");
+ }
+ String ccode = values[0].toLowerCase();
+
+ String[] split = ccode.split("\\.");
+ String pcode = "";
+ if (split.length == 2) {
+ //System.out.println(split);
+ ccode = split[0];
+ pcode = split[1];
+ }
+
+ String pname = values[2];
+
+ if (ccode.matches("[0-9].*")) {
+ String code = ccode;
+ ccode = pcode;
+ pcode = code;
+ }
+
+ String cname = ccodes.get(ccode);
+
+ if (cname == null) {
+ nullcodes.add(ccode);
+ }
+ AdminBoundary data = new AdminBoundary(ccode, cname, pcode, pname, "NO_DATA_FOUND", "NO_DATA_FOUND", cname, pname, "NO_DATA_FOUND");
+ // System.out.println(data);
+ outmap.put(ccode + "." + pcode, data);
+
+ }
+ System.out.println("INFO: there were " + nullcodes.size() + " null prov codes. This is due to inconsistencies in reference data.");
+ reader.close();
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ System.out.println("Successfully read geonames province data from: " + adm1CodesLookupFile.getPath());
+
+ return outmap;
+
+ }
+
+ private static Map<String, String> getCountryCodes(File countryContextFile) {
+ Map<String, String> ccs = new HashMap<>();
+ BufferedReader reader;
+ try {
+
+ reader = new BufferedReader(new FileReader(countryContextFile));
+ int i = 0;
+ String line;
+ boolean start = false;
+ while ((line = reader.readLine()) != null) {
+ if (!line.toLowerCase().startsWith("#iso\t") && !start) {
+
+ continue;
+ } else {
+ start = true;
+ }
+ String[] values = line.split("\t");
+
+ String ccode = values[0].toLowerCase();//this is the 2 digit ISO code
+ String cname = values[4].toLowerCase();
+ if (!ccode.equals("")) {
+ ccs.put(ccode, cname);
+ }
+
+ }
+ reader.close();
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ ccs.put("SS", "South Sudan");
+ ccs.put("CS", "Kosovo");
+ return ccs;
+
+ }
+
+ public static void writeCountryContextFile(File outfile, Map<String, AdminBoundary> adms) {
+ // FileWriter writer = null;
+ try (FileWriter writer = new FileWriter(outfile, true)) {
+ BufferedWriter bw = new BufferedWriter(writer);
+
+ for (String admKey : adms.keySet()) {
+ AdminBoundary adm = adms.get(admKey);
+ if (adm == null) {
+ continue;
+ }
+ String province = adm.getProvinceName();
+ String country = adm.getCountryName();
+
+ String line = adm.getCountryCode() + "\t" + adm.getProvCode() + "\t" + "\t" + country + "\t" + province + "\t" + "\t" + "(" + country + ")" + "\t"
+ + adm.getProvinceName() + "\t" + adm.getCountyName() + "\n";
+ bw.write(line);
+ // System.out.println(line);
+
+ }
+
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ System.out.println("successfully wrote Geonames entries to country oontext file");
+ }
+
+ /**
+ *
+ * @param gazateerInputData the Geonames allCounties.txt file
+ * @param type the types of gaz entry, usgs, geonames, or regions
+ * @param adms the province info
+ * @param countrycodes the country code info
+ * @param w the lucene index writer
+ * @throws Exception
+ */
+ public static void readFile(File gazateerInputData, GazetteerIndexer.GazType type, Map<String, AdminBoundary> adms, Map<String, String> countrycodes, IndexWriter w) throws Exception {
+
+ BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData));
+ String[] boosts = "ADM1 ADM1H ADM2 ADM2H ADM3 ADM3H ADM4 ADM4H ADM5 ADMD ADMDH PCLD PCLH PCLI PCLIX TERR PCLIX PPL PPLA PPLA2 PPLA3 PPLA4 PPLC PPLCH PPLF PPLG PPLH PPLL PPLQ PPLR PPLS PPLX STLMT".split(" ");
+ Map<String, Float> boostMap = new HashMap<>();
+ for (String boost : boosts) {
+ boostMap.put(boost.toLowerCase(), 10f);
+ }
+ String[] fieldStrings = new String[]{
+ "geonameid",
+ "name",
+ "asciiname",
+ "alternatenames",
+ "latitude",
+ "longitude",
+ "feature_class",
+ "feature_code",
+ "country code",
+ "cc2",
+ "admin1_code",
+ "admin2_code",
+ "admin3_code",
+ "admin4_code",
+ "population",
+ "elevation",
+ "dem ",
+ "timezone",
+ "modification_date"};
+
+ List<String> fields = Arrays.asList(fieldStrings);
+ int counter = 0;
+ System.out.println("reading gazetteer data from file...........");
+ String line;
+ while ((line = reader.readLine()) != null) {
+ String[] values = line.split(type.getSeparator());
+
+ Document doc = new Document();
+ String admincode = values[10].toLowerCase();
+ String ccode = values[8].toLowerCase();
+ if (ccode.contains(",")) {
+ String[] codes = ccode.split(",");
+ if (codes.length > 0) {
+ ccode = codes[0];
+ }
+ }
+ AdminBoundary adm = adms.get(ccode + "." + admincode);
+
+ String placeName = values[2];
+ String lat = values[4];
+ String lon = values[5];
+ String dsg = values[7].toLowerCase();
+
+ String id = values[0];
+ String concatIndexEntry;
+ String countryname;
+ if (adm != null) {
+ concatIndexEntry = adm.getCountryName() + ", " + adm.getProvinceName() + ", " + placeName;
+ countryname = adm.getCountryName();
+ } else {
+ //there is no admin info, but we can still use the countrycode to concat the country name
+ String n = countrycodes.get(ccode);
+ countryname = n;
+ if (n != null) {
+ concatIndexEntry = n + ", " + placeName;
+ } else {
+ ///don't want a single token hierarchy entry.
+ concatIndexEntry = "";
+ }
+ }
+ if (ccode == null) {
+ System.out.println("naughty country code");
+ }
+ for (int i = 0; i < fields.size() - 1; i++) {
+ doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
+
+ }
+ if (dsg.equals("pcli")) {
+ System.out.println("placename: " + placeName + " RESET TO: " + countryname);
+ placeName = countryname;
+ }
+ /*
+ * add standard fields to the index
+ */
+ doc.add(new TextField("hierarchy", concatIndexEntry, Field.Store.YES));
+ doc.add(new TextField("placename", placeName, Field.Store.YES));
+ // doc.add(new TextField("countryname", countryname, Field.Store.YES));
+ //System.out.println(placeName);
+
+ doc.add(new TextField("latitude", lat, Field.Store.YES));
+ doc.add(new TextField("longitude", lon, Field.Store.YES));
+ doc.add(new StringField("loctype", dsg, Field.Store.YES));
+ doc.add(new StringField("admincode", (ccode + "." + admincode).toLowerCase(), Field.Store.YES));
+ doc.add(new StringField("countrycode", ccode.toLowerCase(), Field.Store.YES));
+ doc.add(new StringField("countycode", "", Field.Store.YES));
+ doc.add(new StringField("locid", id, Field.Store.YES));
+ placeName = placeName.replace("republic of", "").replace("federative", "");
+ if (id.equals("3175395")) {
+ System.out.println(placeName);
+ }
+ doc.add(new StringField("gazsource", "geonames", Field.Store.YES));
+
+ w.addDocument(doc);
+
+ counter++;
+ if (counter % 100000 == 0) {
+ w.commit();
+ System.out.println(counter + " .........Geonames entries committed to index..............");
+ }
+
+ }
+
+ System.out.println("Completed indexing geonames gaz! index name is: " + type.toString());
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/RegionProcessor.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/RegionProcessor.java
index f1d5d9f..c6b2d0c 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/RegionProcessor.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/RegionProcessor.java
@@ -1,115 +1,115 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.indexing;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexWriter;
-
-public class RegionProcessor {
-
- public static void main(String[] args) {
- RegionProcessor.process(new File("C:\\temp\\gazetteers\\regions.txt"), new File("C:\\temp\\gazetteers\\testRegionContext.txt"), null);
- }
-
- /**
- *
- * @param regionsFile the file that stores Region references. the format of
- * this file is tab delimitted text with index 0 as the name of the region,
- * index 1 as the longitude, and index 2 as the latitude
- * @param outputCountryContextfile this is the country context files shared by
- * all indexing processors
- * @param w
- */
- public static void process(File regionsFile, File outputCountryContextfile, IndexWriter w) {
- try {
- readFile(regionsFile, outputCountryContextfile, w);
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
-
- public static void readFile(File gazateerInputData, File outputCountryContextfile, IndexWriter w) throws Exception {
- List<String> ccfileentries = new ArrayList<>();
- BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData));
- List<String> fields = new ArrayList<>();
- int counter = 0;
- System.out.println("reading gazetteer data from Regions file...........");
- String line;
- while ((line = reader.readLine()) != null) {
-
- String[] values = line.split("\t");
- if (counter == 0) {
-
- } else {
- Document doc = new Document();
- for (int i = 0; i < fields.size() - 1; i++) {
- doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
- }
- String placeName = values[0];
- String lat = values[2];
- String lon = values[1];
- String dsg = "region";
- String id = "rg" + counter;
-
- String hierarchy = placeName;
-
- doc.add(new TextField("hierarchy", hierarchy, Field.Store.YES));
- doc.add(new TextField("placename", placeName, Field.Store.YES));
- doc.add(new StringField("latitude", lat, Field.Store.YES));
- doc.add(new StringField("longitude", lon, Field.Store.YES));
- doc.add(new StringField("loctype", dsg, Field.Store.YES));
- doc.add(new StringField("admincode", "", Field.Store.YES));
- doc.add(new StringField("countrycode", id, Field.Store.YES));
- doc.add(new StringField("countycode", "", Field.Store.YES));
-
- doc.add(new StringField("locid", id, Field.Store.YES));
- doc.add(new StringField("gazsource", "region", Field.Store.YES));
- //countrycontext file format
- // US KY 131 United States Kentucky Leslie
-
- ccfileentries.add(id + "\t" + id + "\t" + id + "\t" + placeName + "\t" + "NO_DATA_FOUND" + "\t" + "NO_DATA_FOUND" + "\t" + "("
- + placeName + ")" + "\t" + "NO_DATA_FOUND" + "\t" + "NO_DATA_FOUND" + "\n");
- if (w != null) {
- w.addDocument(doc);
- }
- }
- counter++;
-
- }
- if (w != null) {
- w.commit();
- }
- FileWriter writer = new FileWriter(outputCountryContextfile, true);
- BufferedWriter bw = new BufferedWriter(writer);
- for (String string : ccfileentries) {
- bw.write(string);
- }
- System.out.println("successfully wrote Region entries to country oontext file");
- bw.close();
- System.out.println("Completed indexing regions!");
- }
-
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.indexing;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+
+public class RegionProcessor {
+
+ public static void main(String[] args) {
+ RegionProcessor.process(new File("C:\\temp\\gazetteers\\regions.txt"), new File("C:\\temp\\gazetteers\\testRegionContext.txt"), null);
+ }
+
+ /**
+ *
+ * @param regionsFile the file that stores Region references. the format of
+ * this file is tab delimitted text with index 0 as the name of the region,
+ * index 1 as the longitude, and index 2 as the latitude
+ * @param outputCountryContextfile this is the country context files shared by
+ * all indexing processors
+ * @param w
+ */
+ public static void process(File regionsFile, File outputCountryContextfile, IndexWriter w) {
+ try {
+ readFile(regionsFile, outputCountryContextfile, w);
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ public static void readFile(File gazateerInputData, File outputCountryContextfile, IndexWriter w) throws Exception {
+ List<String> ccfileentries = new ArrayList<>();
+ BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData));
+ List<String> fields = new ArrayList<>();
+ int counter = 0;
+ System.out.println("reading gazetteer data from Regions file...........");
+ String line;
+ while ((line = reader.readLine()) != null) {
+
+ String[] values = line.split("\t");
+ if (counter == 0) {
+
+ } else {
+ Document doc = new Document();
+ for (int i = 0; i < fields.size() - 1; i++) {
+ doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
+ }
+ String placeName = values[0];
+ String lat = values[2];
+ String lon = values[1];
+ String dsg = "region";
+ String id = "rg" + counter;
+
+ String hierarchy = placeName;
+
+ doc.add(new TextField("hierarchy", hierarchy, Field.Store.YES));
+ doc.add(new TextField("placename", placeName, Field.Store.YES));
+ doc.add(new StringField("latitude", lat, Field.Store.YES));
+ doc.add(new StringField("longitude", lon, Field.Store.YES));
+ doc.add(new StringField("loctype", dsg, Field.Store.YES));
+ doc.add(new StringField("admincode", "", Field.Store.YES));
+ doc.add(new StringField("countrycode", id, Field.Store.YES));
+ doc.add(new StringField("countycode", "", Field.Store.YES));
+
+ doc.add(new StringField("locid", id, Field.Store.YES));
+ doc.add(new StringField("gazsource", "region", Field.Store.YES));
+ //countrycontext file format
+ // US KY 131 United States Kentucky Leslie
+
+ ccfileentries.add(id + "\t" + id + "\t" + id + "\t" + placeName + "\t" + "NO_DATA_FOUND" + "\t" + "NO_DATA_FOUND" + "\t" + "("
+ + placeName + ")" + "\t" + "NO_DATA_FOUND" + "\t" + "NO_DATA_FOUND" + "\n");
+ if (w != null) {
+ w.addDocument(doc);
+ }
+ }
+ counter++;
+
+ }
+ if (w != null) {
+ w.commit();
+ }
+ FileWriter writer = new FileWriter(outputCountryContextfile, true);
+ BufferedWriter bw = new BufferedWriter(writer);
+ for (String string : ccfileentries) {
+ bw.write(string);
+ }
+ System.out.println("successfully wrote Region entries to country oontext file");
+ bw.close();
+ System.out.println("Completed indexing regions!");
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/USGSProcessor.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/USGSProcessor.java
index 588befb..46e6200 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/USGSProcessor.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/USGSProcessor.java
@@ -1,250 +1,250 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.indexing;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import opennlp.addons.geoentitylinker.AdminBoundary;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-
-import org.apache.lucene.index.IndexWriter;
-
-public class USGSProcessor {
-
- public static void main(String[] args) {
- try {
- Map<String, AdminBoundary> provData = getProvData(new File("C:\\temp\\gazetteers\\usgsdata\\GOVT_UNITS_20140601.txt"), GazetteerIndexer.GazType.USGS);
- process(new File("C:\\temp\\gazetteers\\usgsdata\\GOVT_UNITS_20140601.txt"), new File("C:\\temp\\gazetteers\\usgsdata\\NationalFile_20140601.txt"), null, null);
- } catch (Exception ex) {
- Logger.getLogger(USGSProcessor.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
-
- public static void process(File lookupData, File usgsGazDataFile, File outputCountryContextfile, IndexWriter w) throws Exception {
- Map<String, AdminBoundary> provData = getProvData(lookupData, GazetteerIndexer.GazType.USGS);
- readFile(usgsGazDataFile, w, GazetteerIndexer.GazType.USGS, provData);
- writeCountryContextFile(outputCountryContextfile, provData);
- }
-
- public static void readFile(File gazateerInputData, IndexWriter w, GazetteerIndexer.GazType type, Map<String, AdminBoundary> lookupMap) throws Exception {
-
- Map<String, StateCentroid> states = new HashMap<>();
- try (BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData))) {
-
- List<String> fields = new ArrayList<>();
- int counter = 0;
- System.out.println("reading gazetteer data from USGS file...........");
- String line;
- while ((line = reader.readLine()) != null) {
-
- String[] values = line.split(type.getSeparator());
- if (counter == 0) {
- for (String columnName : values) {
- fields.add(columnName.replace("»¿", "").trim());
- }
-
- } else {
- Document doc = new Document();
- for (int i = 0; i < fields.size() - 1; i++) {
- doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
- }
- String placeName = values[1];
- String lat = values[9];
- String lon = values[10];
- String dsg = values[2];
- String id = values[0];
-
- String ccode = values[6];
- String admincode = values[3];
- AdminBoundary get = lookupMap.get(admincode + "." + ccode);
- String countyname = "";
- if (get == null) {
- System.out.println("null...continuing to index" + " ccode: " + ccode + " , admincode: " + admincode + " , placename: " + placeName);
- continue;
-
- }
- String countyCode = get.getCountyCode();
-
- if (!get.getCountyName().equals("NO_DATA_FOUND_VALUE")) {
- countyname = get.getCountyName();
- }
- if (!get.getCountyCode().equals("NO_DATA_FOUND_VALUE")) {
- countyCode = get.getCountyCode();
- }
- String hierarchy = get.getCountryName() + ", " + get.getProvinceName() + ", " + countyname + ", " + placeName;
-
- if (states.containsKey(get.getProvinceName())) {
- StateCentroid entry = states.get(get.getProvinceName());
- entry.count++;
- entry.latSum += Double.parseDouble(lat);
- entry.longSum += Double.parseDouble(lon);
- } else {
- StateCentroid centroid = new StateCentroid();
- centroid.statecode = get.getProvCode();
- centroid.count = 1;
- centroid.latSum = Double.parseDouble(lat);
- centroid.longSum = Double.parseDouble(lon);
- states.put(get.getProvinceName(), centroid);
- }
-
- doc.add(new TextField("hierarchy", hierarchy, Field.Store.YES));
- doc.add(new TextField("placename", placeName, Field.Store.YES));
- doc.add(new TextField("latitude", lat, Field.Store.YES));
- doc.add(new TextField("longitude", lon, Field.Store.YES));
- doc.add(new StringField("loctype", dsg, Field.Store.YES));
- doc.add(new StringField("admincode", (get.getCountryCode() + "." + get.getProvCode()).toLowerCase(), Field.Store.YES));
- doc.add(new StringField("countrycode", get.getCountryCode().toLowerCase(), Field.Store.YES));
- doc.add(new StringField("countycode", (get.getCountryCode() + "." + get.getProvCode() + "." + countyCode).toLowerCase(), Field.Store.YES));
-
- doc.add(new StringField("locid", id, Field.Store.YES));
- doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
- w.addDocument(doc);
- }
- counter++;
- if (counter % 100000 == 0) {
- w.commit();
- System.out.println(counter + " .........USGS entries committed to index..............");
- }
-
- }
- }
-
-
- for (String state : states.keySet()) {
- StateCentroid get = states.get(state);
- Document doc = new Document();
- doc.add(new TextField("hierarchy", "united states, " + state, Field.Store.YES));
- doc.add(new TextField("placename", state, Field.Store.YES));
- //calculate a centroid for all the points that were in the state
- doc.add(new TextField("latitude", (get.latSum / get.count) + "", Field.Store.YES));
- doc.add(new TextField("longitude", (get.longSum / get.count) + "", Field.Store.YES));
- doc.add(new StringField("loctype", "adm1", Field.Store.YES));
- doc.add(new StringField("admincode", get.statecode, Field.Store.YES));
- doc.add(new StringField("countrycode", "us", Field.Store.YES));
- doc.add(new StringField("countycode", "", Field.Store.YES));
-
- doc.add(new StringField("locid", "us_state:" + state, Field.Store.YES));
- doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
- w.addDocument(doc);
-
- // System.out.println(get.statecode + "," + (get.latSum / get.count) + "," + (get.longSum / get.count));
- }
- Document doc = new Document();
- doc.add(new TextField("hierarchy", "united states", Field.Store.YES));
- doc.add(new TextField("placename", "united states", Field.Store.YES));
- //calculate a centroid for all the points that were in the state
- doc.add(new TextField("latitude", 39.0 + "", Field.Store.YES));
- doc.add(new TextField("longitude", -103.0 + "", Field.Store.YES));
- doc.add(new StringField("loctype", "pcli", Field.Store.YES));
- doc.add(new StringField("admincode", "", Field.Store.YES));
- doc.add(new StringField("countrycode", "us", Field.Store.YES));
- doc.add(new StringField("countycode", "", Field.Store.YES));
-
- doc.add(new StringField("locid", "us_centroid" + "unitedstates", Field.Store.YES));
- doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
- //System.out.println("uscentroid," + (sumofLatSums / sumOfCounts) + "," + (sumofLonSums / sumOfCounts));
-
- w.addDocument(doc);
- w.commit();
-
- System.out.println("Completed indexing USGS gaz!");
- }
-
- private static class StateCentroid {
-
- double latSum;
- double longSum;
- String statecode;
- int count;
- }
-
- private static Map<String, AdminBoundary> getProvData(File govUnitsFile, GazetteerIndexer.GazType type) {
- System.out.println("Attempting to read USGS province (State) data from: " + govUnitsFile.getPath());
- Map<String, AdminBoundary> outmap = new HashMap<>();
-
- try (BufferedReader reader = new BufferedReader(new FileReader(govUnitsFile))) {
- int i = 0;
- String line;
- String[] fields = null;
- while ((line = reader.readLine()) != null) {
-
- String[] values = line.split(type.getSeparator());
- if (i == 0) {
- fields = values;
- i++;
- continue;
- }
- i++;
- // System.out.println(i);
- String countyCode = values[2];
- String countyName = values[3];
- String stateCode = values[5];
- String stateName = values[6];
- String countryCode = values[7];
- String countryName = values[8];
- AdminBoundary adminBoundary = new AdminBoundary(countryCode, countryName, stateCode, stateName, countyCode,
- countyName, null, null, null);
- outmap.put(stateCode + "." + countyCode, adminBoundary);
- // System.out.println(adminBoundary);
-
- }
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- System.out.println("Successfully read USGS province (State) data from: " + govUnitsFile.getPath());
-
- return outmap;
-
- }
-
- public static void writeCountryContextFile(File outfile, Map<String, AdminBoundary> adms) {
-
- try (BufferedWriter bw = new BufferedWriter(new FileWriter(outfile, true))) {
- for (String admkey : adms.keySet()) {
- AdminBoundary adm = adms.get(admkey);
- if (adm == null) {
- continue;
- }
- String province = adm.getProvinceName();
- String country = adm.getCountryName();
- /*
- * this is the standard format of the country context file... Geonames
- * data will have an empty string for the county
- */
- String line = adm.getCountryCode() + "\t" + adm.getProvCode() + "\t" + adm.getCountyCode() + "\t" + country + "\t" + province + "\t" + adm.getCountyName() + "\t"
- + "(U\\.S\\.[ $]|U\\.S\\.A\\.[ $]|United States|the US[ $]|a us[ $])" + "\t" + adm.getProvinceName() + "\t" + adm.getCountyName() + "\n";
- bw.write(line);
- /// System.out.println(line);
- }
- } catch (IOException ex) {
- Logger.getLogger(GeonamesProcessor.class.getName()).log(Level.SEVERE, null, ex);
- }
- System.out.println("successfully wrote USGS entries to country context file");
- }
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.indexing;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import opennlp.addons.geoentitylinker.AdminBoundary;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+
+import org.apache.lucene.index.IndexWriter;
+
+public class USGSProcessor {
+
+ public static void main(String[] args) {
+ try {
+ Map<String, AdminBoundary> provData = getProvData(new File("C:\\temp\\gazetteers\\usgsdata\\GOVT_UNITS_20140601.txt"), GazetteerIndexer.GazType.USGS);
+ process(new File("C:\\temp\\gazetteers\\usgsdata\\GOVT_UNITS_20140601.txt"), new File("C:\\temp\\gazetteers\\usgsdata\\NationalFile_20140601.txt"), null, null);
+ } catch (Exception ex) {
+ Logger.getLogger(USGSProcessor.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ }
+
+ public static void process(File lookupData, File usgsGazDataFile, File outputCountryContextfile, IndexWriter w) throws Exception {
+ Map<String, AdminBoundary> provData = getProvData(lookupData, GazetteerIndexer.GazType.USGS);
+ readFile(usgsGazDataFile, w, GazetteerIndexer.GazType.USGS, provData);
+ writeCountryContextFile(outputCountryContextfile, provData);
+ }
+
+ public static void readFile(File gazateerInputData, IndexWriter w, GazetteerIndexer.GazType type, Map<String, AdminBoundary> lookupMap) throws Exception {
+
+ Map<String, StateCentroid> states = new HashMap<>();
+ try (BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData))) {
+
+ List<String> fields = new ArrayList<>();
+ int counter = 0;
+ System.out.println("reading gazetteer data from USGS file...........");
+ String line;
+ while ((line = reader.readLine()) != null) {
+
+ String[] values = line.split(type.getSeparator());
+ if (counter == 0) {
+ for (String columnName : values) {
+ fields.add(columnName.replace("»¿", "").trim());
+ }
+
+ } else {
+ Document doc = new Document();
+ for (int i = 0; i < fields.size() - 1; i++) {
+ doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
+ }
+ String placeName = values[1];
+ String lat = values[9];
+ String lon = values[10];
+ String dsg = values[2];
+ String id = values[0];
+
+ String ccode = values[6];
+ String admincode = values[3];
+ AdminBoundary get = lookupMap.get(admincode + "." + ccode);
+ String countyname = "";
+ if (get == null) {
+ System.out.println("null...continuing to index" + " ccode: " + ccode + " , admincode: " + admincode + " , placename: " + placeName);
+ continue;
+
+ }
+ String countyCode = get.getCountyCode();
+
+ if (!get.getCountyName().equals("NO_DATA_FOUND_VALUE")) {
+ countyname = get.getCountyName();
+ }
+ if (!get.getCountyCode().equals("NO_DATA_FOUND_VALUE")) {
+ countyCode = get.getCountyCode();
+ }
+ String hierarchy = get.getCountryName() + ", " + get.getProvinceName() + ", " + countyname + ", " + placeName;
+
+ if (states.containsKey(get.getProvinceName())) {
+ StateCentroid entry = states.get(get.getProvinceName());
+ entry.count++;
+ entry.latSum += Double.parseDouble(lat);
+ entry.longSum += Double.parseDouble(lon);
+ } else {
+ StateCentroid centroid = new StateCentroid();
+ centroid.statecode = get.getProvCode();
+ centroid.count = 1;
+ centroid.latSum = Double.parseDouble(lat);
+ centroid.longSum = Double.parseDouble(lon);
+ states.put(get.getProvinceName(), centroid);
+ }
+
+ doc.add(new TextField("hierarchy", hierarchy, Field.Store.YES));
+ doc.add(new TextField("placename", placeName, Field.Store.YES));
+ doc.add(new TextField("latitude", lat, Field.Store.YES));
+ doc.add(new TextField("longitude", lon, Field.Store.YES));
+ doc.add(new StringField("loctype", dsg, Field.Store.YES));
+ doc.add(new StringField("admincode", (get.getCountryCode() + "." + get.getProvCode()).toLowerCase(), Field.Store.YES));
+ doc.add(new StringField("countrycode", get.getCountryCode().toLowerCase(), Field.Store.YES));
+ doc.add(new StringField("countycode", (get.getCountryCode() + "." + get.getProvCode() + "." + countyCode).toLowerCase(), Field.Store.YES));
+
+ doc.add(new StringField("locid", id, Field.Store.YES));
+ doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
+ w.addDocument(doc);
+ }
+ counter++;
+ if (counter % 100000 == 0) {
+ w.commit();
+ System.out.println(counter + " .........USGS entries committed to index..............");
+ }
+
+ }
+ }
+
+
+ for (String state : states.keySet()) {
+ StateCentroid get = states.get(state);
+ Document doc = new Document();
+ doc.add(new TextField("hierarchy", "united states, " + state, Field.Store.YES));
+ doc.add(new TextField("placename", state, Field.Store.YES));
+ //calculate a centroid for all the points that were in the state
+ doc.add(new TextField("latitude", (get.latSum / get.count) + "", Field.Store.YES));
+ doc.add(new TextField("longitude", (get.longSum / get.count) + "", Field.Store.YES));
+ doc.add(new StringField("loctype", "adm1", Field.Store.YES));
+ doc.add(new StringField("admincode", get.statecode, Field.Store.YES));
+ doc.add(new StringField("countrycode", "us", Field.Store.YES));
+ doc.add(new StringField("countycode", "", Field.Store.YES));
+
+ doc.add(new StringField("locid", "us_state:" + state, Field.Store.YES));
+ doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
+ w.addDocument(doc);
+
+ // System.out.println(get.statecode + "," + (get.latSum / get.count) + "," + (get.longSum / get.count));
+ }
+ Document doc = new Document();
+ doc.add(new TextField("hierarchy", "united states", Field.Store.YES));
+ doc.add(new TextField("placename", "united states", Field.Store.YES));
+ //calculate a centroid for all the points that were in the state
+ doc.add(new TextField("latitude", 39.0 + "", Field.Store.YES));
+ doc.add(new TextField("longitude", -103.0 + "", Field.Store.YES));
+ doc.add(new StringField("loctype", "pcli", Field.Store.YES));
+ doc.add(new StringField("admincode", "", Field.Store.YES));
+ doc.add(new StringField("countrycode", "us", Field.Store.YES));
+ doc.add(new StringField("countycode", "", Field.Store.YES));
+
+ doc.add(new StringField("locid", "us_centroid" + "unitedstates", Field.Store.YES));
+ doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
+ //System.out.println("uscentroid," + (sumofLatSums / sumOfCounts) + "," + (sumofLonSums / sumOfCounts));
+
+ w.addDocument(doc);
+ w.commit();
+
+ System.out.println("Completed indexing USGS gaz!");
+ }
+
+ private static class StateCentroid {
+
+ double latSum;
+ double longSum;
+ String statecode;
+ int count;
+ }
+
+ private static Map<String, AdminBoundary> getProvData(File govUnitsFile, GazetteerIndexer.GazType type) {
+ System.out.println("Attempting to read USGS province (State) data from: " + govUnitsFile.getPath());
+ Map<String, AdminBoundary> outmap = new HashMap<>();
+
+ try (BufferedReader reader = new BufferedReader(new FileReader(govUnitsFile))) {
+ int i = 0;
+ String line;
+ String[] fields = null;
+ while ((line = reader.readLine()) != null) {
+
+ String[] values = line.split(type.getSeparator());
+ if (i == 0) {
+ fields = values;
+ i++;
+ continue;
+ }
+ i++;
+ // System.out.println(i);
+ String countyCode = values[2];
+ String countyName = values[3];
+ String stateCode = values[5];
+ String stateName = values[6];
+ String countryCode = values[7];
+ String countryName = values[8];
+ AdminBoundary adminBoundary = new AdminBoundary(countryCode, countryName, stateCode, stateName, countyCode,
+ countyName, null, null, null);
+ outmap.put(stateCode + "." + countyCode, adminBoundary);
+ // System.out.println(adminBoundary);
+
+ }
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ System.out.println("Successfully read USGS province (State) data from: " + govUnitsFile.getPath());
+
+ return outmap;
+
+ }
+
+ public static void writeCountryContextFile(File outfile, Map<String, AdminBoundary> adms) {
+
+ try (BufferedWriter bw = new BufferedWriter(new FileWriter(outfile, true))) {
+ for (String admkey : adms.keySet()) {
+ AdminBoundary adm = adms.get(admkey);
+ if (adm == null) {
+ continue;
+ }
+ String province = adm.getProvinceName();
+ String country = adm.getCountryName();
+ /*
+ * this is the standard format of the country context file... Geonames
+ * data will have an empty string for the county
+ */
+ String line = adm.getCountryCode() + "\t" + adm.getProvCode() + "\t" + adm.getCountyCode() + "\t" + country + "\t" + province + "\t" + adm.getCountyName() + "\t"
+ + "(U\\.S\\.[ $]|U\\.S\\.A\\.[ $]|United States|the US[ $]|a us[ $])" + "\t" + adm.getProvinceName() + "\t" + adm.getCountyName() + "\n";
+ bw.write(line);
+ /// System.out.println(line);
+ }
+ } catch (IOException ex) {
+ Logger.getLogger(GeonamesProcessor.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ System.out.println("successfully wrote USGS entries to country context file");
+ }
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/CountryProximityScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/CountryProximityScorer.java
index df69fb7..b706c5f 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/CountryProximityScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/CountryProximityScorer.java
@@ -1,277 +1,277 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.regex.Pattern;
-import opennlp.addons.geoentitylinker.AdminBoundaryContext;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.BaseLink;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-
-/**
- * Scores toponyms based on their proximity to a country mention. Based on the
- * heuristic that toponymn mentions are more likely close to their parent
- * country mentions. For instance, if the toponym Berlin is mentioned near an
- * indicator of Germany, it is more likely to be Berlin Germany than Berlin
- * Connecticut (if Connecticut is mentioned further down in the article).
- */
-public class CountryProximityScorer implements LinkedEntityScorer<AdminBoundaryContext> {
-
- private Map<String, Set<String>> nameCodesMap;
- String dominantCode = "";
- private Map<String, String> regexMap = new HashMap<>();
-
- @Override
- public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
-
- regexMap = additionalContext.getCountryRegexMap();
- score(linkedSpans, additionalContext.getCountryMentions(), additionalContext.getNameCodesMap(), docText, sentenceSpans, 1000);
-
- }
-
- /**
- * Assigns a score to each BaseLink in each linkedSpan's set of N best
- * matches. Currently, the scoring indicates the probability that the toponym
- * is correct based on the country context in the document
- *
- * @param linkedData the linked spans, holds the Namefinder results, and the
- * list of BaseLink for each
- * @param countryHits all the country mentions in the document
- * @param nameCodesMap maps a country indicator name to a country code. Used
- * to determine if the namefinder found the same exact toponym the country
- * context did. If so the score is boosted due to the high probability that
- * the NameFinder actually "rediscovered" a country
- * @param docText the full text of the document...not used in this default
- * implementation
- * @param sentences the sentences that correspond to the doc text.
- * @param maxAllowedDist a constant that is used to determine which country
- * mentions, based on proximity within the text, should be used to score the
- * Named Entity.
- * @return
- */
- public List<LinkedSpan> score(List<LinkedSpan> linkedData, Map<String, Set<Integer>> countryHits, Map<String, Set<String>> nameCodesMap, String docText, Span[] sentences, Integer maxAllowedDist) {
- this.nameCodesMap = nameCodesMap;
- setDominantCode(countryHits);
- for (LinkedSpan<BaseLink> linkedspan : linkedData) {
-
- linkedspan = simpleProximityAnalysis(sentences, countryHits, linkedspan, maxAllowedDist);
- }
- return linkedData;
- }
-
- /**
- * sets class level variable to a code based on the number of mentions
- *
- * @param countryHits
- */
- private void setDominantCode(Map<String, Set<Integer>> countryHits) {
- int hits = -1;
- for (String code : countryHits.keySet()) {
- if (countryHits.get(code).size() > hits) {
- hits = countryHits.get(code).size();
- dominantCode = code;
- }
- }
- }
-
- /**
- * Generates distances from each country mention to the span's location in the
- * doc text. Ultimately an attempt to ensure that ambiguously named toponyms
- * are resolved to the correct country and coordinate.
- *
- * @param sentences
- * @param countryHits
- * @param span
- * @return
- */
- private LinkedSpan<BaseLink> simpleProximityAnalysis(Span[] sentences, Map<String, Set<Integer>> countryHits, LinkedSpan<BaseLink> span, Integer maxAllowedDistance) {
- Double score = 0.0;
- /*
- * get the index of the actual span, beginning of sentence //should generate
- * tokens from sentence and create a char offset... //could have large
- * sentences due to poor sentence detection or wonky doc text
- */
- int sentenceIdx = span.getSentenceid();
- int sentIndexInDoc = sentences[sentenceIdx].getStart();
- /*
- * create a map of all the span's proximal country mentions in the document
- * Map< countrycode, set of <distances from this NamedEntity>>
- */
- Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<>();
- //map = Map<countrycode, Set <of distances this span is from all the mentions of the code>>
- for (String cCode : countryHits.keySet()) {
- // iterate over all the regex start values and calculate an offset
- for (Integer cHit : countryHits.get(cCode)) {
- Integer absDist = Math.abs(sentIndexInDoc - cHit);
- //only include near mentions based on a heuristic
- //TODO make this a property
- // if (absDist < maxAllowedDistance) {
- if (distancesFromCodeMap.containsKey(cCode)) {
- distancesFromCodeMap.get(cCode).add(absDist);
- } else {
- HashSet<Integer> newset = new HashSet<>();
- newset.add(absDist);
- distancesFromCodeMap.put(cCode, newset);
- }
- }
- }
- //we now know how far this named entity is from every country mention in the document
-
- /*
- * the gaz matches that have a country code that have mentions in the doc
- * that are closest to the Named Entity should return the best score.
- * Analyzemap generates a likelihood score that the toponym from the gaz is
- * referring to one of the countries, i.e, Map<countrycode, prob that this
- * span is referring to the toponym form this code key>
- */
- Map<String, Double> scoreMap = analyzeMap(distancesFromCodeMap, sentences, span);
- for (BaseLink link : span.getLinkedEntries()) {
- //getItemParentId is the country code
- String spanCountryCode = link.getItemParentID();
- if (scoreMap.containsKey(spanCountryCode)) {
-
- score = scoreMap.get(spanCountryCode);
- ///does the name extracted match a country name?
- if (nameCodesMap.containsKey(link.getItemName().toLowerCase()) || regexMatch(link.getItemName(), link.getItemParentID())) {
- //if so, is it the correct country code for that name?
- if (nameCodesMap.get(link.getItemName().toLowerCase()).contains(link.getItemParentID())) {
- //boost the score because it is likely that this is the location in the text, so add 50% to the score or set to 1
- score = (score + .75) > 1.0 ? 1d : (score + .75);
-
- if (link.getItemParentID().equals(dominantCode)) {
- score = (score + .25) > 1.0 ? 1d : (score + .25);
- }
- }
- }
- }
-
- link.getScoreMap().put("countrycontext", score);
- }
- return span;
- }
-
- /**
- * takes a map of distances from the toponym to each country mention and
- * generates a map of scores for each country code. The map is then correlated
- * to the code of the BaseLink parentid for retrieval. Then the score is added
- * to the overall list.
- *
- * @param distanceMap
- * @param sentences
- * @param span
- * @return
- */
- private Map<String, Double> analyzeMap(Map<String, Set<Integer>> distanceMap, Span[] sentences, LinkedSpan<BaseLink> span) {
-
- Map<String, Double> scoreMap = new HashMap<>();
- if (distanceMap.isEmpty()) {
- return scoreMap;
- }
- TreeSet<Integer> all = new TreeSet<>();
- for (String key : distanceMap.keySet()) {
- all.addAll(distanceMap.get(key));
- }
-
- // get min max for normalization, this could be more efficient
- int min = all.first();
- int max = all.last();
- if (min == max) {
- min = 0;
- }
-
- for (String key : distanceMap.keySet()) {
- TreeSet<Double> normalizedDistances = new TreeSet<>();
- for (Integer i : distanceMap.get(key)) {
- Double norm = normalize(i, min, max);
- //reverse the normed distance so low numbers (closer) are better
- //this could be improved with a "decaying " function using an imcreaseing negative exponent
- Double reverse = Math.abs(norm - 1);
- normalizedDistances.add(reverse);
- }
-
- List<Double> doubles = new ArrayList<>(normalizedDistances);
- scoreMap.put(key, slidingDistanceAverage(doubles));
- }
- return scoreMap;
- }
-
- private boolean regexMatch(String placeName, String countryCode) {
- if (regexMap.containsKey(countryCode)) {
- String regexForCountry = regexMap.get(countryCode);
-
- Pattern p = Pattern.compile(regexForCountry,Pattern.DOTALL|Pattern.CASE_INSENSITIVE);
- return p.matcher(placeName.trim()).matches();
- }
- return false;
- }
-
- /**
- * this method is an attempt to make closer clusters of mentions group
- * together to smooth out the average, so one distant outlier does not kill
- * the score for an obviously good hit. More elegant solution is possible
- * using Math.pow, and making the score decay with distance by using an
- * increasing negative exponent (I think)
- *
- * @param normDis the normalized and sorted set of distances as a list
- * @return
- */
- private Double slidingDistanceAverage(List<Double> normDis) {
- List<Double> windowOfAverages = new ArrayList<>();
-
- if (normDis.size() < 3) {
- windowOfAverages.addAll(normDis);
- } else {
-
- for (int i = 0; i < normDis.size() - 1; i++) {
- double a = normDis.get(i);
- double b = normDis.get(i + 1);
- windowOfAverages.add((a + b) / 2);
-
- }
- }
- double sum = 0d;
- for (double d : windowOfAverages) {
- sum += d;
- }
- double result = sum / windowOfAverages.size();
- //TODO: ++ prob when large amounts of mentions for a code
- //System.out.println("avg of window:" + result);
- return result;
- }
-
- /**
- * transposes a value within one range to a relative value in a different
- * range. Used to normalize distances in this class.
- *
- * @param valueToNormalize the value to place within the new range
- * @param minimum the min of the set to be transposed
- * @param maximum the max of the set to be transposed
- * @return
- */
- private Double normalize(int valueToNormalize, int minimum, int maximum) {
- Double d = (double) ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
- d = d == null ? 0d : d;
- return d;
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Pattern;
+import opennlp.addons.geoentitylinker.AdminBoundaryContext;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.BaseLink;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ * Scores toponyms based on their proximity to a country mention. Based on the
+ * heuristic that toponymn mentions are more likely close to their parent
+ * country mentions. For instance, if the toponym Berlin is mentioned near an
+ * indicator of Germany, it is more likely to be Berlin Germany than Berlin
+ * Connecticut (if Connecticut is mentioned further down in the article).
+ */
+public class CountryProximityScorer implements LinkedEntityScorer<AdminBoundaryContext> {
+
+ private Map<String, Set<String>> nameCodesMap;
+ String dominantCode = "";
+ private Map<String, String> regexMap = new HashMap<>();
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+
+ regexMap = additionalContext.getCountryRegexMap();
+ score(linkedSpans, additionalContext.getCountryMentions(), additionalContext.getNameCodesMap(), docText, sentenceSpans, 1000);
+
+ }
+
+ /**
+ * Assigns a score to each BaseLink in each linkedSpan's set of N best
+ * matches. Currently, the scoring indicates the probability that the toponym
+ * is correct based on the country context in the document
+ *
+ * @param linkedData the linked spans, holds the Namefinder results, and the
+ * list of BaseLink for each
+ * @param countryHits all the country mentions in the document
+ * @param nameCodesMap maps a country indicator name to a country code. Used
+ * to determine if the namefinder found the same exact toponym the country
+ * context did. If so the score is boosted due to the high probability that
+ * the NameFinder actually "rediscovered" a country
+ * @param docText the full text of the document...not used in this default
+ * implementation
+ * @param sentences the sentences that correspond to the doc text.
+ * @param maxAllowedDist a constant that is used to determine which country
+ * mentions, based on proximity within the text, should be used to score the
+ * Named Entity.
+ * @return
+ */
+ public List<LinkedSpan> score(List<LinkedSpan> linkedData, Map<String, Set<Integer>> countryHits, Map<String, Set<String>> nameCodesMap, String docText, Span[] sentences, Integer maxAllowedDist) {
+ this.nameCodesMap = nameCodesMap;
+ setDominantCode(countryHits);
+ for (LinkedSpan<BaseLink> linkedspan : linkedData) {
+
+ linkedspan = simpleProximityAnalysis(sentences, countryHits, linkedspan, maxAllowedDist);
+ }
+ return linkedData;
+ }
+
+ /**
+ * sets class level variable to a code based on the number of mentions
+ *
+ * @param countryHits
+ */
+ private void setDominantCode(Map<String, Set<Integer>> countryHits) {
+ int hits = -1;
+ for (String code : countryHits.keySet()) {
+ if (countryHits.get(code).size() > hits) {
+ hits = countryHits.get(code).size();
+ dominantCode = code;
+ }
+ }
+ }
+
+ /**
+ * Generates distances from each country mention to the span's location in the
+ * doc text. Ultimately an attempt to ensure that ambiguously named toponyms
+ * are resolved to the correct country and coordinate.
+ *
+ * @param sentences
+ * @param countryHits
+ * @param span
+ * @return
+ */
+ private LinkedSpan<BaseLink> simpleProximityAnalysis(Span[] sentences, Map<String, Set<Integer>> countryHits, LinkedSpan<BaseLink> span, Integer maxAllowedDistance) {
+ Double score = 0.0;
+ /*
+ * get the index of the actual span, beginning of sentence //should generate
+ * tokens from sentence and create a char offset... //could have large
+ * sentences due to poor sentence detection or wonky doc text
+ */
+ int sentenceIdx = span.getSentenceid();
+ int sentIndexInDoc = sentences[sentenceIdx].getStart();
+ /*
+ * create a map of all the span's proximal country mentions in the document
+ * Map< countrycode, set of <distances from this NamedEntity>>
+ */
+ Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<>();
+ //map = Map<countrycode, Set <of distances this span is from all the mentions of the code>>
+ for (String cCode : countryHits.keySet()) {
+ // iterate over all the regex start values and calculate an offset
+ for (Integer cHit : countryHits.get(cCode)) {
+ Integer absDist = Math.abs(sentIndexInDoc - cHit);
+ //only include near mentions based on a heuristic
+ //TODO make this a property
+ // if (absDist < maxAllowedDistance) {
+ if (distancesFromCodeMap.containsKey(cCode)) {
+ distancesFromCodeMap.get(cCode).add(absDist);
+ } else {
+ HashSet<Integer> newset = new HashSet<>();
+ newset.add(absDist);
+ distancesFromCodeMap.put(cCode, newset);
+ }
+ }
+ }
+ //we now know how far this named entity is from every country mention in the document
+
+ /*
+ * the gaz matches that have a country code that have mentions in the doc
+ * that are closest to the Named Entity should return the best score.
+ * Analyzemap generates a likelihood score that the toponym from the gaz is
+ * referring to one of the countries, i.e, Map<countrycode, prob that this
+ * span is referring to the toponym form this code key>
+ */
+ Map<String, Double> scoreMap = analyzeMap(distancesFromCodeMap, sentences, span);
+ for (BaseLink link : span.getLinkedEntries()) {
+ //getItemParentId is the country code
+ String spanCountryCode = link.getItemParentID();
+ if (scoreMap.containsKey(spanCountryCode)) {
+
+ score = scoreMap.get(spanCountryCode);
+ ///does the name extracted match a country name?
+ if (nameCodesMap.containsKey(link.getItemName().toLowerCase()) || regexMatch(link.getItemName(), link.getItemParentID())) {
+ //if so, is it the correct country code for that name?
+ if (nameCodesMap.get(link.getItemName().toLowerCase()).contains(link.getItemParentID())) {
+ //boost the score because it is likely that this is the location in the text, so add 50% to the score or set to 1
+ score = (score + .75) > 1.0 ? 1d : (score + .75);
+
+ if (link.getItemParentID().equals(dominantCode)) {
+ score = (score + .25) > 1.0 ? 1d : (score + .25);
+ }
+ }
+ }
+ }
+
+ link.getScoreMap().put("countrycontext", score);
+ }
+ return span;
+ }
+
+ /**
+ * takes a map of distances from the toponym to each country mention and
+ * generates a map of scores for each country code. The map is then correlated
+ * to the code of the BaseLink parentid for retrieval. Then the score is added
+ * to the overall list.
+ *
+ * @param distanceMap
+ * @param sentences
+ * @param span
+ * @return
+ */
+ private Map<String, Double> analyzeMap(Map<String, Set<Integer>> distanceMap, Span[] sentences, LinkedSpan<BaseLink> span) {
+
+ Map<String, Double> scoreMap = new HashMap<>();
+ if (distanceMap.isEmpty()) {
+ return scoreMap;
+ }
+ TreeSet<Integer> all = new TreeSet<>();
+ for (String key : distanceMap.keySet()) {
+ all.addAll(distanceMap.get(key));
+ }
+
+ // get min max for normalization, this could be more efficient
+ int min = all.first();
+ int max = all.last();
+ if (min == max) {
+ min = 0;
+ }
+
+ for (String key : distanceMap.keySet()) {
+ TreeSet<Double> normalizedDistances = new TreeSet<>();
+ for (Integer i : distanceMap.get(key)) {
+ Double norm = normalize(i, min, max);
+ //reverse the normed distance so low numbers (closer) are better
+ //this could be improved with a "decaying " function using an imcreaseing negative exponent
+ Double reverse = Math.abs(norm - 1);
+ normalizedDistances.add(reverse);
+ }
+
+ List<Double> doubles = new ArrayList<>(normalizedDistances);
+ scoreMap.put(key, slidingDistanceAverage(doubles));
+ }
+ return scoreMap;
+ }
+
+ private boolean regexMatch(String placeName, String countryCode) {
+ if (regexMap.containsKey(countryCode)) {
+ String regexForCountry = regexMap.get(countryCode);
+
+ Pattern p = Pattern.compile(regexForCountry,Pattern.DOTALL|Pattern.CASE_INSENSITIVE);
+ return p.matcher(placeName.trim()).matches();
+ }
+ return false;
+ }
+
+ /**
+ * this method is an attempt to make closer clusters of mentions group
+ * together to smooth out the average, so one distant outlier does not kill
+ * the score for an obviously good hit. More elegant solution is possible
+ * using Math.pow, and making the score decay with distance by using an
+ * increasing negative exponent (I think)
+ *
+ * @param normDis the normalized and sorted set of distances as a list
+ * @return
+ */
+ private Double slidingDistanceAverage(List<Double> normDis) {
+ List<Double> windowOfAverages = new ArrayList<>();
+
+ if (normDis.size() < 3) {
+ windowOfAverages.addAll(normDis);
+ } else {
+
+ for (int i = 0; i < normDis.size() - 1; i++) {
+ double a = normDis.get(i);
+ double b = normDis.get(i + 1);
+ windowOfAverages.add((a + b) / 2);
+
+ }
+ }
+ double sum = 0d;
+ for (double d : windowOfAverages) {
+ sum += d;
+ }
+ double result = sum / windowOfAverages.size();
+ //TODO: ++ prob when large amounts of mentions for a code
+ //System.out.println("avg of window:" + result);
+ return result;
+ }
+
+ /**
+ * transposes a value within one range to a relative value in a different
+ * range. Used to normalize distances in this class.
+ *
+ * @param valueToNormalize the value to place within the new range
+ * @param minimum the min of the set to be transposed
+ * @param maximum the max of the set to be transposed
+ * @return
+ */
+ private Double normalize(int valueToNormalize, int minimum, int maximum) {
+ Double d = (double) ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
+ d = d == null ? 0d : d;
+ return d;
+ }
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/FuzzyStringMatchScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/FuzzyStringMatchScorer.java
index 64bd788..5367122 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/FuzzyStringMatchScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/FuzzyStringMatchScorer.java
@@ -1,124 +1,124 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import opennlp.addons.geoentitylinker.AdminBoundaryContext;
-import opennlp.addons.geoentitylinker.GazetteerEntry;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.BaseLink;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-
-/**
- *
- * Generates scores based on string comparisons levenstein and dice
- */
-public class FuzzyStringMatchScorer implements LinkedEntityScorer<AdminBoundaryContext> {
-
- @Override
- public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
-
- for (LinkedSpan<BaseLink> linkedSpan : linkedSpans) {
- for (BaseLink link : linkedSpan.getLinkedEntries()) {
- if (link instanceof GazetteerEntry) {
- GazetteerEntry entry = (GazetteerEntry) link;
- String hierarchy = entry.getHierarchy();
- if (hierarchy != null) {
- Double dice = getDiceCoefficient(linkedSpan.getSearchTerm().toLowerCase(), hierarchy.toLowerCase(), 2);
- link.getScoreMap().put("hierarchydicecoef", dice);
- Double ld = (double) getLevenshteinDistance(linkedSpan.getSearchTerm().toLowerCase(), hierarchy.toLowerCase());
- link.getScoreMap().put("hierarchylevenshtein", ld);
- }
- String placename = entry.getItemName().toLowerCase();
- if (placename != null) {
- Double dice = getDiceCoefficient(linkedSpan.getSearchTerm().toLowerCase(), placename, 2);
- link.getScoreMap().put("placenamedicecoef", dice);
-
- }
- }
- }
- }
-
- }
-
- /**
- * Generates a score based on an overlap of nGrams between two strings using
- * the DiceCoefficient technique.
- *
- * @param s1 first string
- * @param s2 second string
- * @param nGrams number of chars in each gram
- * @return
- */
- public double getDiceCoefficient(String s1, String s2, int nGrams) {
- if (s1.isEmpty() || s2.isEmpty()) {
- return 0d;
- }
- List<String> s1Grams = new ArrayList<>();
- List<String> s2Grams = new ArrayList<>();
- String[] split1 = s1.split("[ ,]");
- for (String token : split1) {
- if (token.trim().equals("")) {
- continue;
- }
- s1Grams.add(token);
- }
- String[] split2 = s2.split("[ ,]");
- for (String token : split2) {
- if (token.trim().equals("")) {
- continue;
- }
- s2Grams.add(token);
- }
-
- Set<String> overlap = new HashSet<>(s1Grams);
- overlap.retainAll(s2Grams);
- double totcombigrams = overlap.size();
-
- return (2 * totcombigrams) / (s1Grams.size() + s2Grams.size());
- }
-
- private int minimum(int a, int b, int c) {
- return Math.min(Math.min(a, b), c);
- }
-
- public int getLevenshteinDistance(CharSequence str1,
- CharSequence str2) {
- int[][] distance = new int[str1.length() + 1][str2.length() + 1];
-
- for (int i = 0; i <= str1.length(); i++) {
- distance[i][0] = i;
- }
- for (int j = 1; j <= str2.length(); j++) {
- distance[0][j] = j;
- }
-
- for (int i = 1; i <= str1.length(); i++) {
- for (int j = 1; j <= str2.length(); j++) {
- distance[i][j] = minimum(
- distance[i - 1][j] + 1,
- distance[i][j - 1] + 1,
- distance[i - 1][j - 1] + ((str1.charAt(i - 1) == str2.charAt(j - 1)) ? 0 : 1));
- }
- }
-
- return distance[str1.length()][str2.length()];
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import opennlp.addons.geoentitylinker.AdminBoundaryContext;
+import opennlp.addons.geoentitylinker.GazetteerEntry;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.BaseLink;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ *
+ * Generates scores based on string comparisons levenstein and dice
+ */
+public class FuzzyStringMatchScorer implements LinkedEntityScorer<AdminBoundaryContext> {
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+
+ for (LinkedSpan<BaseLink> linkedSpan : linkedSpans) {
+ for (BaseLink link : linkedSpan.getLinkedEntries()) {
+ if (link instanceof GazetteerEntry) {
+ GazetteerEntry entry = (GazetteerEntry) link;
+ String hierarchy = entry.getHierarchy();
+ if (hierarchy != null) {
+ Double dice = getDiceCoefficient(linkedSpan.getSearchTerm().toLowerCase(), hierarchy.toLowerCase(), 2);
+ link.getScoreMap().put("hierarchydicecoef", dice);
+ Double ld = (double) getLevenshteinDistance(linkedSpan.getSearchTerm().toLowerCase(), hierarchy.toLowerCase());
+ link.getScoreMap().put("hierarchylevenshtein", ld);
+ }
+ String placename = entry.getItemName().toLowerCase();
+ if (placename != null) {
+ Double dice = getDiceCoefficient(linkedSpan.getSearchTerm().toLowerCase(), placename, 2);
+ link.getScoreMap().put("placenamedicecoef", dice);
+
+ }
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Generates a score based on an overlap of nGrams between two strings using
+ * the DiceCoefficient technique.
+ *
+ * @param s1 first string
+ * @param s2 second string
+ * @param nGrams number of chars in each gram
+ * @return
+ */
+ public double getDiceCoefficient(String s1, String s2, int nGrams) {
+ if (s1.isEmpty() || s2.isEmpty()) {
+ return 0d;
+ }
+ List<String> s1Grams = new ArrayList<>();
+ List<String> s2Grams = new ArrayList<>();
+ String[] split1 = s1.split("[ ,]");
+ for (String token : split1) {
+ if (token.trim().equals("")) {
+ continue;
+ }
+ s1Grams.add(token);
+ }
+ String[] split2 = s2.split("[ ,]");
+ for (String token : split2) {
+ if (token.trim().equals("")) {
+ continue;
+ }
+ s2Grams.add(token);
+ }
+
+ Set<String> overlap = new HashSet<>(s1Grams);
+ overlap.retainAll(s2Grams);
+ double totcombigrams = overlap.size();
+
+ return (2 * totcombigrams) / (s1Grams.size() + s2Grams.size());
+ }
+
+ private int minimum(int a, int b, int c) {
+ return Math.min(Math.min(a, b), c);
+ }
+
+ public int getLevenshteinDistance(CharSequence str1,
+ CharSequence str2) {
+ int[][] distance = new int[str1.length() + 1][str2.length() + 1];
+
+ for (int i = 0; i <= str1.length(); i++) {
+ distance[i][0] = i;
+ }
+ for (int j = 1; j <= str2.length(); j++) {
+ distance[0][j] = j;
+ }
+
+ for (int i = 1; i <= str1.length(); i++) {
+ for (int j = 1; j <= str2.length(); j++) {
+ distance[i][j] = minimum(
+ distance[i - 1][j] + 1,
+ distance[i][j - 1] + 1,
+ distance[i - 1][j - 1] + ((str1.charAt(i - 1) == str2.charAt(j - 1)) ? 0 : 1));
+ }
+ }
+
+ return distance[str1.length()][str2.length()];
+ }
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/GeoHashBinningScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/GeoHashBinningScorer.java
index adf48fd..6b245d6 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/GeoHashBinningScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/GeoHashBinningScorer.java
@@ -1,61 +1,61 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import opennlp.addons.geoentitylinker.AdminBoundaryContext;
-import opennlp.addons.geoentitylinker.GazetteerEntry;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.BaseLink;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-
-/**
- * Scores toponymns based on geographic point binning. Based on the heuristic
- * that docs are generally about a small amount of locations, so one can detect
- * outliers by finding those points that are not near the majority
- *
- */
-public class GeoHashBinningScorer implements LinkedEntityScorer<AdminBoundaryContext> {
-
- private final PointClustering CLUSTERER = new PointClustering();
- private int PRECISION = 3;
-
- @Override
- public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
- List<GazetteerEntry> allGazEntries = new ArrayList<>();
-
- /*
- * collect all the gaz entry references
- */
- for (LinkedSpan<BaseLink> ls : linkedSpans) {
- for (BaseLink bl : ls.getLinkedEntries()) {
- if (bl instanceof GazetteerEntry) {
- allGazEntries.add((GazetteerEntry) bl);
- }
- }
- }
- /*
- * use the point clustering to score each hit
- */
- Map<String, List<GazetteerEntry>> cluster = CLUSTERER.cluster(allGazEntries, PRECISION);
- CLUSTERER.scoreClusters(cluster);
-
- }
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import opennlp.addons.geoentitylinker.AdminBoundaryContext;
+import opennlp.addons.geoentitylinker.GazetteerEntry;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.BaseLink;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ * Scores toponymns based on geographic point binning. Based on the heuristic
+ * that docs are generally about a small amount of locations, so one can detect
+ * outliers by finding those points that are not near the majority
+ *
+ */
+public class GeoHashBinningScorer implements LinkedEntityScorer<AdminBoundaryContext> {
+
+ private final PointClustering CLUSTERER = new PointClustering();
+ private int PRECISION = 3;
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+ List<GazetteerEntry> allGazEntries = new ArrayList<>();
+
+ /*
+ * collect all the gaz entry references
+ */
+ for (LinkedSpan<BaseLink> ls : linkedSpans) {
+ for (BaseLink bl : ls.getLinkedEntries()) {
+ if (bl instanceof GazetteerEntry) {
+ allGazEntries.add((GazetteerEntry) bl);
+ }
+ }
+ }
+ /*
+ * use the point clustering to score each hit
+ */
+ Map<String, List<GazetteerEntry>> cluster = CLUSTERER.cluster(allGazEntries, PRECISION);
+ CLUSTERER.scoreClusters(cluster);
+
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/LinkedEntityScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/LinkedEntityScorer.java
index 5741af0..8054d6f 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/LinkedEntityScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/LinkedEntityScorer.java
@@ -1,40 +1,40 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.util.List;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-
-/**
- * Structure for scoring linked entities. The Map logically represents a pair :
- * "Score type" to the "actual Score."
- * @param <T> a generic for providing additional context
- */
-public interface LinkedEntityScorer<T> {
-
-/**
- * Scores a collection of linked entities. Implementations should populate the scoreMap in the list of BaseLink for each linkedSpan
- * this method internally affects the reference to linkedSpans that was passed in
- * @param linkedSpans the spans that have been linked to some external source and have all the data they need to be scored
- * @param docText the full text of the document.
- * @param sentenceSpans the sentence spans the correspond to the document text
- * @param properties the entitylinker properties config file
- * @param additionalContext any additional data required to perform the scoring operation
- */
- void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, T additionalContext);
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.util.List;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ * Structure for scoring linked entities. The Map logically represents a pair :
+ * "Score type" to the "actual Score."
+ * @param <T> a generic for providing additional context
+ */
+public interface LinkedEntityScorer<T> {
+
+/**
+ * Scores a collection of linked entities. Implementations should populate the scoreMap in the list of BaseLink for each linkedSpan
+ * this method internally affects the reference to linkedSpans that was passed in
+ * @param linkedSpans the spans that have been linked to some external source and have all the data they need to be scored
+ * @param docText the full text of the document.
+ * @param sentenceSpans the sentence spans the correspond to the document text
+ * @param properties the entitylinker properties config file
+ * @param additionalContext any additional data required to perform the scoring operation
+ */
+ void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, T additionalContext);
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ModelBasedScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ModelBasedScorer.java
index 8d473d0..1c39855 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ModelBasedScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ModelBasedScorer.java
@@ -1,157 +1,157 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.io.File;
-import java.lang.invoke.MethodHandles;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import opennlp.addons.geoentitylinker.AdminBoundaryContext;
-import opennlp.tools.doccat.DoccatModel;
-import opennlp.tools.doccat.DocumentCategorizerME;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.BaseLink;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- * Utilizes a doccat model to score toponyms based on surrounding context
- */
-public class ModelBasedScorer implements LinkedEntityScorer<AdminBoundaryContext> {
-
- private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- DocumentCategorizerME documentCategorizerME;
- DoccatModel doccatModel;
- public static final int RADIUS = 200;
- boolean modelexists = false;
-
- @Override
- public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
- try {
- if (doccatModel == null) {
- String path = properties.getProperty("opennlp.geoentitylinker.modelbasedscorer.modelpath", "");
- if (path.equals("")) {
- return;
- }
- modelexists = true;
- doccatModel = new DoccatModel(new File(path));
- documentCategorizerME = new DocumentCategorizerME(doccatModel);
- }
- Map<Integer, String> proximalFeatures = generateProximalFeatures(linkedSpans, sentenceSpans, docText, RADIUS);
- for (Map.Entry<Integer, String> entry : proximalFeatures.entrySet()) {
- Map<String, Double> scores = this.getScore(entry.getValue());
- for (BaseLink link : (List<BaseLink>) linkedSpans.get(entry.getKey()).getLinkedEntries()) {
- double score = 0d;
- if (scores.containsKey(link.getItemParentID())) {
- score = scores.get(link.getItemParentID());
- }
- link.getScoreMap().put("countrymodel", score);
- }
- }
-
- } catch (Exception ex) {
- LOG.error(ex.getLocalizedMessage(), ex);
- }
- }
-
- /**
- * generates features using a BagOfWordsfeatureGenerator that are within the
- * radius of a mention within the doctext
- *
- * @param linkedSpans
- * @param sentenceSpans
- * @param docText
- * @param radius
- * @return a map of the index of the linked span to the string of surrounding
- * text: Map<indexofspan,surrounding text>
- */
- public Map<Integer, String> generateProximalFeatures(List<LinkedSpan> linkedSpans, Span[] sentenceSpans, String docText, int radius) {
- Map<Integer, String> featureBags = new HashMap<>();
- Map<Integer, Integer> nameMentionMap = new HashMap<>();
- /*
- * iterator over the map that contains a mapping of every country code to
- * all of its mentions in the document
- */
- for (int i = 0; i < linkedSpans.size(); i++) {
- LinkedSpan<?> span = linkedSpans.get(i);
- if (span.getLinkedEntries().isEmpty()) {
- //don't care about spans that did not get linked to anything at all; nothing to work with
- continue;
- }
- /*
- * get the sentence the name span was found in, the beginning of the
- * sentence will suffice as a centroid for feature generation around the
- * named entity
- */
- Integer mentionIdx = sentenceSpans[span.getSentenceid()].getStart();
- nameMentionMap.put(i, mentionIdx);
- }
- /*
- * now associate each span to a string that will be used for categorization
- * against the model.
- */
- for (Map.Entry<Integer, Integer> entry : nameMentionMap.entrySet()) {
- featureBags.put(entry.getKey(), getTextChunk(entry.getValue(), docText, radius));
- }
-
- return featureBags;
- }
-
- public String getTextChunk(int mentionIdx, String docText, int radius) {
- int docSize = docText.length();
- int left = 0, right = 0;
- left = (mentionIdx - radius < 0) ? 0 : mentionIdx - radius;
- right = (mentionIdx + radius > docSize) ? docSize : mentionIdx + radius;
- String chunk = "";
- if (right <= left) {
- chunk = "";
- } else {
- /*
- * don't want to chop any words in half, so take fron the first space to
- * the last space in the chunk string
- */
- chunk = docText.substring(left, right);
- if (left != 0) {
- left = chunk.indexOf(" ");
- }
- right = chunk.lastIndexOf(" ");
- /*
- * now get the substring again with only whole words
- */
- if (left < right) {
- chunk = chunk.substring(left, right);
- }
- }
-
- return chunk;
- }
-
- private Map<String, Double> getScore(String text) throws Exception {
- Map<String, Double> scoreMap = new HashMap<>();
- double[] categorize = documentCategorizerME.categorize(List.of(text).toArray(new String[0]));
- int catSize = documentCategorizerME.getNumberOfCategories();
- for (int i = 0; i < catSize; i++) {
- String category = documentCategorizerME.getCategory(i);
- scoreMap.put(category, categorize[documentCategorizerME.getIndex(category)]);
- }
- return scoreMap;
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.io.File;
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import opennlp.addons.geoentitylinker.AdminBoundaryContext;
+import opennlp.tools.doccat.DoccatModel;
+import opennlp.tools.doccat.DocumentCategorizerME;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.BaseLink;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * Utilizes a doccat model to score toponyms based on surrounding context
+ */
+public class ModelBasedScorer implements LinkedEntityScorer<AdminBoundaryContext> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ DocumentCategorizerME documentCategorizerME;
+ DoccatModel doccatModel;
+ public static final int RADIUS = 200;
+ boolean modelexists = false;
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+ try {
+ if (doccatModel == null) {
+ String path = properties.getProperty("opennlp.geoentitylinker.modelbasedscorer.modelpath", "");
+ if (path.equals("")) {
+ return;
+ }
+ modelexists = true;
+ doccatModel = new DoccatModel(new File(path));
+ documentCategorizerME = new DocumentCategorizerME(doccatModel);
+ }
+ Map<Integer, String> proximalFeatures = generateProximalFeatures(linkedSpans, sentenceSpans, docText, RADIUS);
+ for (Map.Entry<Integer, String> entry : proximalFeatures.entrySet()) {
+ Map<String, Double> scores = this.getScore(entry.getValue());
+ for (BaseLink link : (List<BaseLink>) linkedSpans.get(entry.getKey()).getLinkedEntries()) {
+ double score = 0d;
+ if (scores.containsKey(link.getItemParentID())) {
+ score = scores.get(link.getItemParentID());
+ }
+ link.getScoreMap().put("countrymodel", score);
+ }
+ }
+
+ } catch (Exception ex) {
+ LOG.error(ex.getLocalizedMessage(), ex);
+ }
+ }
+
+ /**
+ * generates features using a BagOfWordsfeatureGenerator that are within the
+ * radius of a mention within the doctext
+ *
+ * @param linkedSpans
+ * @param sentenceSpans
+ * @param docText
+ * @param radius
+ * @return a map of the index of the linked span to the string of surrounding
+ * text: Map<indexofspan,surrounding text>
+ */
+ public Map<Integer, String> generateProximalFeatures(List<LinkedSpan> linkedSpans, Span[] sentenceSpans, String docText, int radius) {
+ Map<Integer, String> featureBags = new HashMap<>();
+ Map<Integer, Integer> nameMentionMap = new HashMap<>();
+ /*
+ * iterator over the map that contains a mapping of every country code to
+ * all of its mentions in the document
+ */
+ for (int i = 0; i < linkedSpans.size(); i++) {
+ LinkedSpan<?> span = linkedSpans.get(i);
+ if (span.getLinkedEntries().isEmpty()) {
+ //don't care about spans that did not get linked to anything at all; nothing to work with
+ continue;
+ }
+ /*
+ * get the sentence the name span was found in, the beginning of the
+ * sentence will suffice as a centroid for feature generation around the
+ * named entity
+ */
+ Integer mentionIdx = sentenceSpans[span.getSentenceid()].getStart();
+ nameMentionMap.put(i, mentionIdx);
+ }
+ /*
+ * now associate each span to a string that will be used for categorization
+ * against the model.
+ */
+ for (Map.Entry<Integer, Integer> entry : nameMentionMap.entrySet()) {
+ featureBags.put(entry.getKey(), getTextChunk(entry.getValue(), docText, radius));
+ }
+
+ return featureBags;
+ }
+
+ public String getTextChunk(int mentionIdx, String docText, int radius) {
+ int docSize = docText.length();
+ int left = 0, right = 0;
+ left = (mentionIdx - radius < 0) ? 0 : mentionIdx - radius;
+ right = (mentionIdx + radius > docSize) ? docSize : mentionIdx + radius;
+ String chunk = "";
+ if (right <= left) {
+ chunk = "";
+ } else {
+ /*
+ * don't want to chop any words in half, so take fron the first space to
+ * the last space in the chunk string
+ */
+ chunk = docText.substring(left, right);
+ if (left != 0) {
+ left = chunk.indexOf(" ");
+ }
+ right = chunk.lastIndexOf(" ");
+ /*
+ * now get the substring again with only whole words
+ */
+ if (left < right) {
+ chunk = chunk.substring(left, right);
+ }
+ }
+
+ return chunk;
+ }
+
+ private Map<String, Double> getScore(String text) throws Exception {
+ Map<String, Double> scoreMap = new HashMap<>();
+ double[] categorize = documentCategorizerME.categorize(List.of(text).toArray(new String[0]));
+ int catSize = documentCategorizerME.getNumberOfCategories();
+ for (int i = 0; i < catSize; i++) {
+ String category = documentCategorizerME.getCategory(i);
+ scoreMap.put(category, categorize[documentCategorizerME.getIndex(category)]);
+ }
+ return scoreMap;
+ }
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PlacetypeScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PlacetypeScorer.java
index 4c8e151..9a3e2d1 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PlacetypeScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PlacetypeScorer.java
@@ -1,82 +1,82 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import opennlp.addons.geoentitylinker.AdminBoundaryContext;
-import opennlp.addons.geoentitylinker.GazetteerEntry;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-
-/**
- *
- * @author mgiaconia
- */
-public class PlacetypeScorer implements LinkedEntityScorer<AdminBoundaryContext> {
-
- private static final String[] boosts = "ADM1 ADM1H ADM2 ADM2H ADM3 ADM3H ADM4 ADM4H ADM5 ADMD ADMDH PCLD PCLH PCLI PCLIX TERR PCLIX PPL PPLA PPLA2 PPLA3 PPLA4 PPLC PPLCH PPLF PPLG PPLH PPLL PPLQ PPLR PPLS PPLX STLMT civil Populated_Place".split(" ");
- private Map<String, Double> boosetedTypes = new HashMap<>();
-
- public PlacetypeScorer() {
- fillMap();
- }
-
- @Override
- public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
- for (LinkedSpan<GazetteerEntry> geospan : linkedSpans) {
- ArrayList<GazetteerEntry> linkedEntries = geospan.getLinkedEntries();
- for (GazetteerEntry gazetteerEntry : linkedEntries) {
- String type = gazetteerEntry.getItemType().toLowerCase();
- Double score = getScore(type);
- if (score == null) {
- score = 0d;
- }
- gazetteerEntry.getScoreMap().put("typescore", score);
- }
- }
- }
-
- private Double getScore(String type) {
- Double ret = boosetedTypes.get(type.toLowerCase());
- return ret == null ? 0d : ret;
- }
-
- private void fillMap() {
- if (boosetedTypes.isEmpty()) {
- for (String type : boosts) {
- if (type.equals("PCLI")) {
- boosetedTypes.put(type.toLowerCase(), 1d);
- } else if ((type.startsWith("PC")|| type.startsWith("PP")) && !type.equals("PCLI")) {
- boosetedTypes.put(type.toLowerCase(), .5d);
- } else if (type.startsWith("ADM")) {
- boosetedTypes.put(type.toLowerCase(), .75d);
- }else if (type.equalsIgnoreCase("civil")){
- boosetedTypes.put(type.toLowerCase(), .9d);
- }
- if(type.equalsIgnoreCase("populated_place")){
- boosetedTypes.put("Populated Place", .75d);
- }
-
- }
- boosetedTypes.put("pplc", .9);
- }
- }
-
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import opennlp.addons.geoentitylinker.AdminBoundaryContext;
+import opennlp.addons.geoentitylinker.GazetteerEntry;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ *
+ * @author mgiaconia
+ */
+public class PlacetypeScorer implements LinkedEntityScorer<AdminBoundaryContext> {
+
+ private static final String[] boosts = "ADM1 ADM1H ADM2 ADM2H ADM3 ADM3H ADM4 ADM4H ADM5 ADMD ADMDH PCLD PCLH PCLI PCLIX TERR PCLIX PPL PPLA PPLA2 PPLA3 PPLA4 PPLC PPLCH PPLF PPLG PPLH PPLL PPLQ PPLR PPLS PPLX STLMT civil Populated_Place".split(" ");
+ private Map<String, Double> boosetedTypes = new HashMap<>();
+
+ public PlacetypeScorer() {
+ fillMap();
+ }
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+ for (LinkedSpan<GazetteerEntry> geospan : linkedSpans) {
+ ArrayList<GazetteerEntry> linkedEntries = geospan.getLinkedEntries();
+ for (GazetteerEntry gazetteerEntry : linkedEntries) {
+ String type = gazetteerEntry.getItemType().toLowerCase();
+ Double score = getScore(type);
+ if (score == null) {
+ score = 0d;
+ }
+ gazetteerEntry.getScoreMap().put("typescore", score);
+ }
+ }
+ }
+
+ private Double getScore(String type) {
+ Double ret = boosetedTypes.get(type.toLowerCase());
+ return ret == null ? 0d : ret;
+ }
+
+ private void fillMap() {
+ if (boosetedTypes.isEmpty()) {
+ for (String type : boosts) {
+ if (type.equals("PCLI")) {
+ boosetedTypes.put(type.toLowerCase(), 1d);
+ } else if ((type.startsWith("PC")|| type.startsWith("PP")) && !type.equals("PCLI")) {
+ boosetedTypes.put(type.toLowerCase(), .5d);
+ } else if (type.startsWith("ADM")) {
+ boosetedTypes.put(type.toLowerCase(), .75d);
+ }else if (type.equalsIgnoreCase("civil")){
+ boosetedTypes.put(type.toLowerCase(), .9d);
+ }
+ if(type.equalsIgnoreCase("populated_place")){
+ boosetedTypes.put("Populated Place", .75d);
+ }
+
+ }
+ boosetedTypes.put("pplc", .9);
+ }
+ }
+
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PointClustering.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PointClustering.java
index 8cd1440..a8677e4 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PointClustering.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PointClustering.java
@@ -1,121 +1,121 @@
-/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.io.GeohashUtils;
-import com.spatial4j.core.shape.Point;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import opennlp.addons.geoentitylinker.GazetteerEntry;
-
-/**
- *
- * Clusters a list of lat long points using a simple geohashing/boxing approach
- */
-public class PointClustering {
-
- /**
- * Clusters a set of points from the gazateers. The idea is that locations
- * that matched a name that are closer to each other, the more likely the
- * toponym is to be accurate
- *
- * @param entries
- * @param precision
- * @return
- */
- public Map<String, List<GazetteerEntry>> cluster(List<GazetteerEntry> entries, int precision) {
- Map<String, List<GazetteerEntry>> map = new HashMap<>();
- for (GazetteerEntry entry : entries) {
- Double latw = entry.getLatitude();
- Double lonw = entry.getLongitude();
-
- String key = geoHash(latw, lonw).substring(0, precision);
- if (map.containsKey(key)) {
- map.get(key).add(entry);
- } else {
- List<GazetteerEntry> newlist = new ArrayList<>();
- newlist.add(entry);
- map.put(key, newlist);
- }
- }
- return map;
- }
-
- public void scoreClusters(Map<String, List<GazetteerEntry>> clusters) {
- Double min = 0d;
- Double max = -1d;
- for (String key : clusters.keySet()) {
- int size = clusters.get(key).size();
- if (size > max) {
- max = (double) size;
- }
- }
- for (String key : clusters.keySet()) {
- int size = clusters.get(key).size();
- Double score = normalize((double) size, min, max);
- for (GazetteerEntry entry : clusters.get(key)) {
- entry.getScoreMap().put("geohashbin", score);
- }
- }
-
- }
-
- /**
- * Returns a geohash based on Lucene Spatial
- *
- * @param lat the input latitude Y
- * @param lon the input longitude X
- * @return
- */
- public String geoHash(Double lat, Double lon) {
- String encodeLatLon = GeohashUtils.encodeLatLon(lat, lon);
- return encodeLatLon;
- }
-
- /**
- * Returns the X and Y point for the geohash. Element 0 is the X (longitude)
- * element 1 is the Y (latitude)
- *
- * @param geohash
- * @return
- */
- public double[] geoHashToPoint(String geohash) {
- Point decode = GeohashUtils.decode(geohash, SpatialContext.GEO);
- double[] coords = new double[]{decode.getX(), decode.getY()};
- return coords;
- }
-
- /**
- * Returns the X and Y point for the geohash. Element 0 is the X (longitude)
- * element 1 is the Y (latitude)
- *
- * @param geohash
- * @return
- */
- public String geoHashToPointStr(String geohash) {
- Point decode = GeohashUtils.decode(geohash, SpatialContext.GEO);
- String point = decode.getX() + "," + decode.getY();
- return point;
- }
-
-
- private Double normalize(Double valueToNormalize, double minimum, double maximum) {
- return ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
- }
-}
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.io.GeohashUtils;
+import com.spatial4j.core.shape.Point;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import opennlp.addons.geoentitylinker.GazetteerEntry;
+
+/**
+ *
+ * Clusters a list of lat long points using a simple geohashing/boxing approach
+ */
+public class PointClustering {
+
+ /**
+ * Clusters a set of points from the gazateers. The idea is that locations
+ * that matched a name that are closer to each other, the more likely the
+ * toponym is to be accurate
+ *
+ * @param entries
+ * @param precision
+ * @return
+ */
+ public Map<String, List<GazetteerEntry>> cluster(List<GazetteerEntry> entries, int precision) {
+ Map<String, List<GazetteerEntry>> map = new HashMap<>();
+ for (GazetteerEntry entry : entries) {
+ Double latw = entry.getLatitude();
+ Double lonw = entry.getLongitude();
+
+ String key = geoHash(latw, lonw).substring(0, precision);
+ if (map.containsKey(key)) {
+ map.get(key).add(entry);
+ } else {
+ List<GazetteerEntry> newlist = new ArrayList<>();
+ newlist.add(entry);
+ map.put(key, newlist);
+ }
+ }
+ return map;
+ }
+
+ public void scoreClusters(Map<String, List<GazetteerEntry>> clusters) {
+ Double min = 0d;
+ Double max = -1d;
+ for (String key : clusters.keySet()) {
+ int size = clusters.get(key).size();
+ if (size > max) {
+ max = (double) size;
+ }
+ }
+ for (String key : clusters.keySet()) {
+ int size = clusters.get(key).size();
+ Double score = normalize((double) size, min, max);
+ for (GazetteerEntry entry : clusters.get(key)) {
+ entry.getScoreMap().put("geohashbin", score);
+ }
+ }
+
+ }
+
+ /**
+ * Returns a geohash based on Lucene Spatial
+ *
+ * @param lat the input latitude Y
+ * @param lon the input longitude X
+ * @return
+ */
+ public String geoHash(Double lat, Double lon) {
+ String encodeLatLon = GeohashUtils.encodeLatLon(lat, lon);
+ return encodeLatLon;
+ }
+
+ /**
+ * Returns the X and Y point for the geohash. Element 0 is the X (longitude)
+ * element 1 is the Y (latitude)
+ *
+ * @param geohash
+ * @return
+ */
+ public double[] geoHashToPoint(String geohash) {
+ Point decode = GeohashUtils.decode(geohash, SpatialContext.GEO);
+ double[] coords = new double[]{decode.getX(), decode.getY()};
+ return coords;
+ }
+
+ /**
+ * Returns the X and Y point for the geohash. Element 0 is the X (longitude)
+ * element 1 is the Y (latitude)
+ *
+ * @param geohash
+ * @return
+ */
+ public String geoHashToPointStr(String geohash) {
+ Point decode = GeohashUtils.decode(geohash, SpatialContext.GEO);
+ String point = decode.getX() + "," + decode.getY();
+ return point;
+ }
+
+
+ private Double normalize(Double valueToNormalize, double minimum, double maximum) {
+ return ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
+ }
+}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ProvinceProximityScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ProvinceProximityScorer.java
index a21af45..27e9354 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ProvinceProximityScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ProvinceProximityScorer.java
@@ -1,294 +1,294 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.geoentitylinker.scoring;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.regex.Pattern;
-import opennlp.addons.geoentitylinker.AdminBoundaryContext;
-import opennlp.addons.geoentitylinker.GazetteerEntry;
-import opennlp.tools.entitylinker.BaseLink;
-import opennlp.tools.entitylinker.EntityLinkerProperties;
-import opennlp.tools.entitylinker.LinkedSpan;
-import opennlp.tools.util.Span;
-
-/**
- * Scores toponyms based on their proximity to a province mention. Based on the
- * heuristic that toponymn mentions are more likely close to their parent
- * province mentions. For instance, if the toponym Berlin is mentioned near an
- * indicator of Connecticut, it is more likely to be Berlin Connecticut than
- * Berlin Germany (if Germany did not exist in, or is mentioned further away in
- * the article).
- *
- *
- */
-public class ProvinceProximityScorer implements LinkedEntityScorer<AdminBoundaryContext> {
-
- private Map<String, Set<String>> nameCodesMap;
- String dominantCode = "";
- private Map<String, String> regexMap = new HashMap<>();
-
- @Override
- public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
- if (!additionalContext.getProvHits().isEmpty()) {
- regexMap = additionalContext.getProvinceRegexMap();
- score(linkedSpans, additionalContext.getProvMentions(), additionalContext.getNameCodesMap(), docText, sentenceSpans, 1000);
- } else {
- for (LinkedSpan<BaseLink> span : linkedSpans) {
- for (BaseLink link : span.getLinkedEntries()) {
- link.getScoreMap().put("provincecontext", 0d);
- }
- }
- }
-
- }
-
- /**
- * Assigns a score to each BaseLink in each linkedSpan's set of N best
- * matches. Currently the scoring indicates the probability that the toponym
- * is correct based on the country context in the document
- *
- * @param linkedData the linked spans, holds the Namefinder results, and the
- * list of BaseLink for each
- * @param countryHits all the country mentions in the document
- * @param nameCodesMap maps a province indicator name to a province code. Used
- * to determine if the namefinder found the same exact toponym the country
- * context did. If so the score is boosted due to the high probability that
- * the NameFinder actually "rediscovered" a country
- * @param docText the full text of the document...not used in this default
- * implementation
- * @param sentences the sentences that correspond to the doc text.
- * @param maxAllowedDist a constant that is used to determine which country
- * mentions, based on proximity within the text, should be used to score the
- * Named Entity.
- * @return
- */
- public List<LinkedSpan> score(List<LinkedSpan> linkedData, Map<String, Set<Integer>> countryHits, Map<String, Set<String>> nameCodesMap, String docText, Span[] sentences, Integer maxAllowedDist) {
- this.nameCodesMap = nameCodesMap;
- setDominantCode(countryHits);
- for (LinkedSpan<BaseLink> linkedspan : linkedData) {
-
- linkedspan = simpleProximityAnalysis(sentences, countryHits, linkedspan, maxAllowedDist);
- }
- return linkedData;
- }
-
- /**
- * sets class level variable to a code based on the number of mentions
- *
- * @param countryHits
- */
- private void setDominantCode(Map<String, Set<Integer>> countryHits) {
- int hits = -1;
- for (String code : countryHits.keySet()) {
- if (countryHits.get(code).size() > hits) {
- hits = countryHits.get(code).size();
- dominantCode = code;
- }
- }
- }
-
- /**
- * Generates distances from each country mention to the span's location in the
- * doc text. Ultimately an attempt to ensure that ambiguously named toponyms
- * are resolved to the correct country and coordinate.
- *
- * @param sentences
- * @param countryHits
- * @param span
- * @return
- */
- private LinkedSpan<BaseLink> simpleProximityAnalysis(Span[] sentences, Map<String, Set<Integer>> countryHits, LinkedSpan<BaseLink> span, Integer maxAllowedDistance) {
- Double score = 0.0;
- /*
- * get the index of the actual span, begining of sentence //should generate
- * tokens from sentence and create a char offset... //could have large
- * sentences due to poor sentence detection or wonky doc text
- */
- int sentenceIdx = span.getSentenceid();
- int sentIndexInDoc = sentences[sentenceIdx].getStart();
- /**
- * create a map of all the span's proximal country mentions in the document
- * Map< countrycode, set of <distances from this NamedEntity>>
- */
- Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<>();
- //map = Map<countrycode, Set <of distances this span is from all the mentions of the code>>
- for (String cCode : countryHits.keySet()) {
-//iterate over all the regex start values and calculate an offset
- for (Integer cHit : countryHits.get(cCode)) {
- Integer absDist = Math.abs(sentIndexInDoc - cHit);
- //only include near mentions based on a heuristic
- //TODO make this a property
- // if (absDist < maxAllowedDistance) {
- if (distancesFromCodeMap.containsKey(cCode)) {
- distancesFromCodeMap.get(cCode).add(absDist);
- } else {
- HashSet<Integer> newset = new HashSet<>();
- newset.add(absDist);
- distancesFromCodeMap.put(cCode, newset);
- }
- }
-
- //}
- }
- //we now know how far this named entity is from every country mention in the document
-
- /**
- * the gaz matches that have a country code that have mentions in the doc
- * that are closest to the Named Entity should return the best score.
- * Analyzemap generates a likelihood score that the toponym from the gaz is
- * referring to one of the countries, i.e, Map<countrycode, prob that this
- * span is referring to the toponym form this code key>
- */
- Map<String, Double> scoreMap = analyzeMap(distancesFromCodeMap, sentences, span);
- if (scoreMap.isEmpty()) {
- return span;
- }
- for (BaseLink link : span.getLinkedEntries()) {
- //getItemParentId is the country code
- GazetteerEntry entry = (GazetteerEntry) link;
- String spanCountryCode = entry.getProvinceCode();
- if (scoreMap.containsKey(spanCountryCode)) {
-
- score = scoreMap.get(spanCountryCode);
- ///does the name extracted match a province name?
- if (nameCodesMap.containsKey(link.getItemName().toLowerCase()) || regexMatch(link.getItemName(), link.getItemParentID())) {
- //if so, is it the correct country code for that name?
- if (nameCodesMap.get(entry.getItemName().toLowerCase()).contains(entry.getProvinceCode())) {
- //boost the score becuase it is likely that this is the location in the text, so add 50% to the score or set to 1
- //TODO: make this smarter
- score = (score + .75) > 1.0 ? 1d : (score + .75);
-
- if (entry.getProvinceCode().equals(dominantCode)) {
- score = (score + .25) > 1.0 ? 1d : (score + .25);
- }
- }
- }
- }
- link.getScoreMap().put("provincecontext", score);
- }
- return span;
- }
-
- private boolean regexMatch(String placeName, String countryCode) {
- if (regexMap.containsKey(countryCode)) {
- String regexForCountry = regexMap.get(countryCode);
-
- Pattern p = Pattern.compile(regexForCountry, Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
- return p.matcher(placeName.trim()).matches();
- }
- return false;
- }
-
- /**
- * takes a map of distances from the toponym to each province mention and
- * generates a map of scores for each province code. The map is then
- * correlated to the code of the BaseLink parentid for retrieval. Then the
- * score is added to the overall list.
- *
- * @param distanceMap
- * @param sentences
- * @param span
- * @return
- */
- private Map<String, Double> analyzeMap(Map<String, Set<Integer>> distanceMap, Span[] sentences, LinkedSpan<BaseLink> span) {
-
- Map<String, Double> scoreMap = new HashMap<>();
- if (distanceMap.isEmpty()) {
- return scoreMap;
- }
- TreeSet<Integer> all = new TreeSet<>();
- for (String key : distanceMap.keySet()) {
- all.addAll(distanceMap.get(key));
- }
- //get min max for normalization, this could be more efficient
-
- int min = all.first();
- int max = all.last();
- if (min == max) {
- min = 0;
- }
- for (String key : distanceMap.keySet()) {
-
- TreeSet<Double> normalizedDistances = new TreeSet<>();
- for (Integer i : distanceMap.get(key)) {
- Double norm = normalize(i, min, max);
- //reverse the normed distance so low numbers (closer) are better
- //this could be improved with a "decaying " function using an imcreaseing negative exponent
- Double reverse = Math.abs(norm - 1);
- normalizedDistances.add(reverse);
- }
-
- List<Double> doubles = new ArrayList<>(normalizedDistances);
- scoreMap.put(key, slidingDistanceAverage(doubles));
- }
- return scoreMap;
- }
-
- /**
- * this method is an attempt to make closer clusters of mentions group
- * together to smooth out the average, so one distant outlier does not kill
- * the score for an obviously good hit. More elegant solution is possible
- * using Math.pow, and making the score decay with distance by using an
- * increasing negative exponent
- *
- * @param normDis the normalized and sorted set of distances as a list
- * @return
- */
- private Double slidingDistanceAverage(List<Double> normDis) {
- List<Double> windowOfAverages = new ArrayList<>();
-
- if (normDis.size() < 3) {
- windowOfAverages.addAll(normDis);
- } else {
-
- for (int i = 0; i < normDis.size() - 1; i++) {
- double a = normDis.get(i);
- double b = normDis.get(i + 1);
- windowOfAverages.add((a + b) / 2);
-
- }
- }
- double sum = 0d;
- for (double d : windowOfAverages) {
- sum += d;
- }
- double result = sum / windowOfAverages.size();
- //TODO: ++ prob when large amounts of mentions for a code
- //System.out.println("avg of window:" + result);
- return result;
- }
-
- /**
- * transposes a value within one range to a relative value in a different
- * range. Used to normalize distances in this class.
- *
- * @param valueToNormalize the value to place within the new range
- * @param minimum the min of the set to be transposed
- * @param maximum the max of the set to be transposed
- * @return
- */
- private Double normalize(int valueToNormalize, int minimum, int maximum) {
- Double d = (double) ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
- d = d == null ? 0d : d;
- return d;
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.geoentitylinker.scoring;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Pattern;
+import opennlp.addons.geoentitylinker.AdminBoundaryContext;
+import opennlp.addons.geoentitylinker.GazetteerEntry;
+import opennlp.tools.entitylinker.BaseLink;
+import opennlp.tools.entitylinker.EntityLinkerProperties;
+import opennlp.tools.entitylinker.LinkedSpan;
+import opennlp.tools.util.Span;
+
+/**
+ * Scores toponyms based on their proximity to a province mention. Based on the
+ * heuristic that toponymn mentions are more likely close to their parent
+ * province mentions. For instance, if the toponym Berlin is mentioned near an
+ * indicator of Connecticut, it is more likely to be Berlin Connecticut than
+ * Berlin Germany (if Germany did not exist in, or is mentioned further away in
+ * the article).
+ *
+ *
+ */
+public class ProvinceProximityScorer implements LinkedEntityScorer<AdminBoundaryContext> {
+
+ private Map<String, Set<String>> nameCodesMap;
+ String dominantCode = "";
+ private Map<String, String> regexMap = new HashMap<>();
+
+ @Override
+ public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+ if (!additionalContext.getProvHits().isEmpty()) {
+ regexMap = additionalContext.getProvinceRegexMap();
+ score(linkedSpans, additionalContext.getProvMentions(), additionalContext.getNameCodesMap(), docText, sentenceSpans, 1000);
+ } else {
+ for (LinkedSpan<BaseLink> span : linkedSpans) {
+ for (BaseLink link : span.getLinkedEntries()) {
+ link.getScoreMap().put("provincecontext", 0d);
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Assigns a score to each BaseLink in each linkedSpan's set of N best
+ * matches. Currently the scoring indicates the probability that the toponym
+ * is correct based on the country context in the document
+ *
+ * @param linkedData the linked spans, holds the Namefinder results, and the
+ * list of BaseLink for each
+ * @param countryHits all the country mentions in the document
+ * @param nameCodesMap maps a province indicator name to a province code. Used
+ * to determine if the namefinder found the same exact toponym the country
+ * context did. If so the score is boosted due to the high probability that
+ * the NameFinder actually "rediscovered" a country
+ * @param docText the full text of the document...not used in this default
+ * implementation
+ * @param sentences the sentences that correspond to the doc text.
+ * @param maxAllowedDist a constant that is used to determine which country
+ * mentions, based on proximity within the text, should be used to score the
+ * Named Entity.
+ * @return
+ */
+ public List<LinkedSpan> score(List<LinkedSpan> linkedData, Map<String, Set<Integer>> countryHits, Map<String, Set<String>> nameCodesMap, String docText, Span[] sentences, Integer maxAllowedDist) {
+ this.nameCodesMap = nameCodesMap;
+ setDominantCode(countryHits);
+ for (LinkedSpan<BaseLink> linkedspan : linkedData) {
+
+ linkedspan = simpleProximityAnalysis(sentences, countryHits, linkedspan, maxAllowedDist);
+ }
+ return linkedData;
+ }
+
+ /**
+ * sets class level variable to a code based on the number of mentions
+ *
+ * @param countryHits
+ */
+ private void setDominantCode(Map<String, Set<Integer>> countryHits) {
+ int hits = -1;
+ for (String code : countryHits.keySet()) {
+ if (countryHits.get(code).size() > hits) {
+ hits = countryHits.get(code).size();
+ dominantCode = code;
+ }
+ }
+ }
+
+ /**
+ * Generates distances from each country mention to the span's location in the
+ * doc text. Ultimately an attempt to ensure that ambiguously named toponyms
+ * are resolved to the correct country and coordinate.
+ *
+ * @param sentences
+ * @param countryHits
+ * @param span
+ * @return
+ */
+ private LinkedSpan<BaseLink> simpleProximityAnalysis(Span[] sentences, Map<String, Set<Integer>> countryHits, LinkedSpan<BaseLink> span, Integer maxAllowedDistance) {
+ Double score = 0.0;
+ /*
+ * get the index of the actual span, begining of sentence //should generate
+ * tokens from sentence and create a char offset... //could have large
+ * sentences due to poor sentence detection or wonky doc text
+ */
+ int sentenceIdx = span.getSentenceid();
+ int sentIndexInDoc = sentences[sentenceIdx].getStart();
+ /**
+ * create a map of all the span's proximal country mentions in the document
+ * Map< countrycode, set of <distances from this NamedEntity>>
+ */
+ Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<>();
+ //map = Map<countrycode, Set <of distances this span is from all the mentions of the code>>
+ for (String cCode : countryHits.keySet()) {
+//iterate over all the regex start values and calculate an offset
+ for (Integer cHit : countryHits.get(cCode)) {
+ Integer absDist = Math.abs(sentIndexInDoc - cHit);
+ //only include near mentions based on a heuristic
+ //TODO make this a property
+ // if (absDist < maxAllowedDistance) {
+ if (distancesFromCodeMap.containsKey(cCode)) {
+ distancesFromCodeMap.get(cCode).add(absDist);
+ } else {
+ HashSet<Integer> newset = new HashSet<>();
+ newset.add(absDist);
+ distancesFromCodeMap.put(cCode, newset);
+ }
+ }
+
+ //}
+ }
+ //we now know how far this named entity is from every country mention in the document
+
+ /**
+ * the gaz matches that have a country code that have mentions in the doc
+ * that are closest to the Named Entity should return the best score.
+ * Analyzemap generates a likelihood score that the toponym from the gaz is
+ * referring to one of the countries, i.e, Map<countrycode, prob that this
+ * span is referring to the toponym form this code key>
+ */
+ Map<String, Double> scoreMap = analyzeMap(distancesFromCodeMap, sentences, span);
+ if (scoreMap.isEmpty()) {
+ return span;
+ }
+ for (BaseLink link : span.getLinkedEntries()) {
+ //getItemParentId is the country code
+ GazetteerEntry entry = (GazetteerEntry) link;
+ String spanCountryCode = entry.getProvinceCode();
+ if (scoreMap.containsKey(spanCountryCode)) {
+
+ score = scoreMap.get(spanCountryCode);
+ ///does the name extracted match a province name?
+ if (nameCodesMap.containsKey(link.getItemName().toLowerCase()) || regexMatch(link.getItemName(), link.getItemParentID())) {
+ //if so, is it the correct country code for that name?
+ if (nameCodesMap.get(entry.getItemName().toLowerCase()).contains(entry.getProvinceCode())) {
+ //boost the score becuase it is likely that this is the location in the text, so add 50% to the score or set to 1
+ //TODO: make this smarter
+ score = (score + .75) > 1.0 ? 1d : (score + .75);
+
+ if (entry.getProvinceCode().equals(dominantCode)) {
+ score = (score + .25) > 1.0 ? 1d : (score + .25);
+ }
+ }
+ }
+ }
+ link.getScoreMap().put("provincecontext", score);
+ }
+ return span;
+ }
+
+ private boolean regexMatch(String placeName, String countryCode) {
+ if (regexMap.containsKey(countryCode)) {
+ String regexForCountry = regexMap.get(countryCode);
+
+ Pattern p = Pattern.compile(regexForCountry, Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
+ return p.matcher(placeName.trim()).matches();
+ }
+ return false;
+ }
+
+ /**
+ * takes a map of distances from the toponym to each province mention and
+ * generates a map of scores for each province code. The map is then
+ * correlated to the code of the BaseLink parentid for retrieval. Then the
+ * score is added to the overall list.
+ *
+ * @param distanceMap
+ * @param sentences
+ * @param span
+ * @return
+ */
+ private Map<String, Double> analyzeMap(Map<String, Set<Integer>> distanceMap, Span[] sentences, LinkedSpan<BaseLink> span) {
+
+ Map<String, Double> scoreMap = new HashMap<>();
+ if (distanceMap.isEmpty()) {
+ return scoreMap;
+ }
+ TreeSet<Integer> all = new TreeSet<>();
+ for (String key : distanceMap.keySet()) {
+ all.addAll(distanceMap.get(key));
+ }
+ //get min max for normalization, this could be more efficient
+
+ int min = all.first();
+ int max = all.last();
+ if (min == max) {
+ min = 0;
+ }
+ for (String key : distanceMap.keySet()) {
+
+ TreeSet<Double> normalizedDistances = new TreeSet<>();
+ for (Integer i : distanceMap.get(key)) {
+ Double norm = normalize(i, min, max);
+ //reverse the normed distance so low numbers (closer) are better
+ //this could be improved with a "decaying " function using an imcreaseing negative exponent
+ Double reverse = Math.abs(norm - 1);
+ normalizedDistances.add(reverse);
+ }
+
+ List<Double> doubles = new ArrayList<>(normalizedDistances);
+ scoreMap.put(key, slidingDistanceAverage(doubles));
+ }
+ return scoreMap;
+ }
+
+ /**
+ * this method is an attempt to make closer clusters of mentions group
+ * together to smooth out the average, so one distant outlier does not kill
+ * the score for an obviously good hit. More elegant solution is possible
+ * using Math.pow, and making the score decay with distance by using an
+ * increasing negative exponent
+ *
+ * @param normDis the normalized and sorted set of distances as a list
+ * @return
+ */
+ private Double slidingDistanceAverage(List<Double> normDis) {
+ List<Double> windowOfAverages = new ArrayList<>();
+
+ if (normDis.size() < 3) {
+ windowOfAverages.addAll(normDis);
+ } else {
+
+ for (int i = 0; i < normDis.size() - 1; i++) {
+ double a = normDis.get(i);
+ double b = normDis.get(i + 1);
+ windowOfAverages.add((a + b) / 2);
+
+ }
+ }
+ double sum = 0d;
+ for (double d : windowOfAverages) {
+ sum += d;
+ }
+ double result = sum / windowOfAverages.size();
+ //TODO: ++ prob when large amounts of mentions for a code
+ //System.out.println("avg of window:" + result);
+ return result;
+ }
+
+ /**
+ * transposes a value within one range to a relative value in a different
+ * range. Used to normalize distances in this class.
+ *
+ * @param valueToNormalize the value to place within the new range
+ * @param minimum the min of the set to be transposed
+ * @param maximum the max of the set to be transposed
+ * @return
+ */
+ private Double normalize(int valueToNormalize, int minimum, int maximum) {
+ Double d = (double) ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
+ d = d == null ? 0d : d;
+ return d;
+ }
+}
diff --git a/jwnl-addon/pom.xml b/jwnl-addon/pom.xml
index 9b0a8ed..ed01100 100644
--- a/jwnl-addon/pom.xml
+++ b/jwnl-addon/pom.xml
@@ -1,78 +1,78 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-addons</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>jwnl-addon</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Apache OpenNLP JWNL Addon</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-tools</artifactId>
- </dependency>
-
- <dependency>
- <groupId>net.sf.jwordnet</groupId>
- <artifactId>jwnl</artifactId>
- <version>1.3.3</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-engine</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-params</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>${maven.compiler.source}</source>
- <target>${maven.compiler.target}</target>
- <compilerArgument>-Xlint</compilerArgument>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>jwnl-addon</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <name>Apache OpenNLP JWNL Addon</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-tools</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>net.sf.jwordnet</groupId>
+ <artifactId>jwnl</artifactId>
+ <version>1.3.3</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerArgument>-Xlint</compilerArgument>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/modelbuilder-addon/pom.xml b/modelbuilder-addon/pom.xml
index b139d3b..c3109e3 100644
--- a/modelbuilder-addon/pom.xml
+++ b/modelbuilder-addon/pom.xml
@@ -1,72 +1,72 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-addons</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>modelbuilder-addon</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <name>Apache OpenNLP ModelBuilder Addon</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-tools</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-engine</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-params</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>${maven.compiler.source}</source>
- <target>${maven.compiler.target}</target>
- <compilerArgument>-Xlint</compilerArgument>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>modelbuilder-addon</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <name>Apache OpenNLP ModelBuilder Addon</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-tools</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerArgument>-Xlint</compilerArgument>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/DefaultModelBuilderUtil.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/DefaultModelBuilderUtil.java
index b613877..72f5d46 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/DefaultModelBuilderUtil.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/DefaultModelBuilderUtil.java
@@ -1,121 +1,121 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import java.io.File;
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-import opennlp.addons.modelbuilder.impls.FileKnownEntityProvider;
-import opennlp.addons.modelbuilder.impls.FileModelValidatorImpl;
-import opennlp.addons.modelbuilder.impls.FileSentenceProvider;
-import opennlp.addons.modelbuilder.impls.GenericModelGenerator;
-import opennlp.addons.modelbuilder.impls.GenericModelableImpl;
-
-/**
- * Utilizes the file-based implementations to produce an NER model from user defined data.
- * <p>
- * The basic processing is such read in the list of known entities
- * <ul>
- * <li>annotate the sentences based on the list of known entities,</li>
- * <li>create a model from the annotations,</li>
- * <li>perform NER with the model on the sentences,</li>
- * <li>add the NER results to the annotations,</li>
- * <li>rebuild the model,</li>
- * <li>loop</li>
- * </ul>
- */
-public class DefaultModelBuilderUtil {
-
- /**
- *
- * @param sentences a file that contains one sentence per line.
- * There should be at least 15K sentences
- * consisting of a representative sample from
- * user data
- * @param knownEntities a file consisting of a simple list of
- * unambiguous entities, one entry per line.
- * For instance, if one was trying to build a
- * person NER model then this file would be a
- * list of person names that are unambiguous
- * and are known to exist in the sentences
- * file
- * @param knownEntitiesBlacklist This file contains a list of known bad hits
- * that the NER phase of this processing might
- * catch early one before the model iterates
- * to maturity
- * @param modelOutFile the location where the model will be
- * written to
- * @param annotatedSentenceOutFile where the annotated sentences produced by
- * this process will be written to
- * @param namedEntityType the type of entity... for example, person,
- * location, organization...
- * @param iterations how many times to repeat the iterative loop
- * of annotation, model generation, and NER
- */
- public static void generateModel(File sentences, File knownEntities, File knownEntitiesBlacklist,
- File modelOutFile, File annotatedSentenceOutFile, String namedEntityType, int iterations) {
- SemiSupervisedModelGenerator modelGenerator = new GenericModelGenerator();
- BaseModelBuilderParams params = new BaseModelBuilderParams();
- params.setAnnotatedTrainingDataFile(annotatedSentenceOutFile);
- params.setSentenceFile(sentences);
- params.setEntityType(namedEntityType);
- params.setKnownEntitiesFile(knownEntities);
- params.setModelFile(modelOutFile);
- params.setKnownEntityBlacklist(knownEntitiesBlacklist);
-
- /*
- * sentence providers feed this process with user data derived sentences
- * this impl just reads line by line through a file
- */
- SentenceProvider sentenceProvider = new FileSentenceProvider();
- sentenceProvider.setParameters(params);
-
- /*
- * KnownEntityProviders provide a seed list of known entities... such as
- * Barack Obama for person, or Germany for location obviously these would
- * want to be prolific, non-ambiguous names
- */
- KnownEntityProvider knownEntityProvider = new FileKnownEntityProvider();
- knownEntityProvider.setParameters(params);
-
- /*
- * ModelGenerationValidators try to weed out bad hits by the iterations of
- * the name finder. Since this is a recursive process, with each iteration
- * the namefinder will get more and more greedy if bad entities are allowed
- * in this provides a mechanism for throwing out obviously bad hits. A good
- * impl may be to make sure a location is actually within a noun phrase
- * etc...users can make this as specific as they need for their dat and
- * their use case
- */
- ModelGenerationValidator validator = new FileModelValidatorImpl();
- validator.setParameters(params);
-
- /*
- * Modelable's write and read the annotated sentences, as well as create and
- * write the NER models
- */
- Modelable modelable = new GenericModelableImpl();
- modelable.setParameters(params);
-
- /*
- * the modelGenerator actually runs the process with a set number of
- * iterations... could be better by actually calculating the diff between
- * runs and stopping based on a thresh, but for extrememly large sentence
- * sets this may be too much.
- */
- modelGenerator.build(sentenceProvider, knownEntityProvider, validator, modelable, iterations);
-
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import java.io.File;
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+import opennlp.addons.modelbuilder.impls.FileKnownEntityProvider;
+import opennlp.addons.modelbuilder.impls.FileModelValidatorImpl;
+import opennlp.addons.modelbuilder.impls.FileSentenceProvider;
+import opennlp.addons.modelbuilder.impls.GenericModelGenerator;
+import opennlp.addons.modelbuilder.impls.GenericModelableImpl;
+
+/**
+ * Utilizes the file-based implementations to produce an NER model from user defined data.
+ * <p>
+ * The basic processing is such read in the list of known entities
+ * <ul>
+ * <li>annotate the sentences based on the list of known entities,</li>
+ * <li>create a model from the annotations,</li>
+ * <li>perform NER with the model on the sentences,</li>
+ * <li>add the NER results to the annotations,</li>
+ * <li>rebuild the model,</li>
+ * <li>loop</li>
+ * </ul>
+ */
+public class DefaultModelBuilderUtil {
+
+ /**
+ *
+ * @param sentences a file that contains one sentence per line.
+ * There should be at least 15K sentences
+ * consisting of a representative sample from
+ * user data
+ * @param knownEntities a file consisting of a simple list of
+ * unambiguous entities, one entry per line.
+ * For instance, if one was trying to build a
+ * person NER model then this file would be a
+ * list of person names that are unambiguous
+ * and are known to exist in the sentences
+ * file
+ * @param knownEntitiesBlacklist This file contains a list of known bad hits
+ * that the NER phase of this processing might
+ * catch early one before the model iterates
+ * to maturity
+ * @param modelOutFile the location where the model will be
+ * written to
+ * @param annotatedSentenceOutFile where the annotated sentences produced by
+ * this process will be written to
+ * @param namedEntityType the type of entity... for example, person,
+ * location, organization...
+ * @param iterations how many times to repeat the iterative loop
+ * of annotation, model generation, and NER
+ */
+ public static void generateModel(File sentences, File knownEntities, File knownEntitiesBlacklist,
+ File modelOutFile, File annotatedSentenceOutFile, String namedEntityType, int iterations) {
+ SemiSupervisedModelGenerator modelGenerator = new GenericModelGenerator();
+ BaseModelBuilderParams params = new BaseModelBuilderParams();
+ params.setAnnotatedTrainingDataFile(annotatedSentenceOutFile);
+ params.setSentenceFile(sentences);
+ params.setEntityType(namedEntityType);
+ params.setKnownEntitiesFile(knownEntities);
+ params.setModelFile(modelOutFile);
+ params.setKnownEntityBlacklist(knownEntitiesBlacklist);
+
+ /*
+ * sentence providers feed this process with user data derived sentences
+ * this impl just reads line by line through a file
+ */
+ SentenceProvider sentenceProvider = new FileSentenceProvider();
+ sentenceProvider.setParameters(params);
+
+ /*
+ * KnownEntityProviders provide a seed list of known entities... such as
+ * Barack Obama for person, or Germany for location obviously these would
+ * want to be prolific, non-ambiguous names
+ */
+ KnownEntityProvider knownEntityProvider = new FileKnownEntityProvider();
+ knownEntityProvider.setParameters(params);
+
+ /*
+ * ModelGenerationValidators try to weed out bad hits by the iterations of
+ * the name finder. Since this is a recursive process, with each iteration
+ * the namefinder will get more and more greedy if bad entities are allowed
+ * in this provides a mechanism for throwing out obviously bad hits. A good
+ * impl may be to make sure a location is actually within a noun phrase
+ * etc...users can make this as specific as they need for their dat and
+ * their use case
+ */
+ ModelGenerationValidator validator = new FileModelValidatorImpl();
+ validator.setParameters(params);
+
+ /*
+ * Modelable's write and read the annotated sentences, as well as create and
+ * write the NER models
+ */
+ Modelable modelable = new GenericModelableImpl();
+ modelable.setParameters(params);
+
+ /*
+ * the modelGenerator actually runs the process with a set number of
+ * iterations... could be better by actually calculating the diff between
+ * runs and stopping based on a thresh, but for extrememly large sentence
+ * sets this may be too much.
+ */
+ modelGenerator.build(sentenceProvider, knownEntityProvider, validator, modelable, iterations);
+
+ }
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/KnownEntityProvider.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/KnownEntityProvider.java
index 5135c2b..9354c50 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/KnownEntityProvider.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/KnownEntityProvider.java
@@ -1,48 +1,48 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-
-import java.util.Set;
-
-
-
-/**
- *
-Supplies a list of known entities (a list of names or locations)
- */
-public interface KnownEntityProvider extends ModelParameter<BaseModelBuilderParams> {
-
- /**
- * returns a list of known non ambiguous entities.
- * @return a set of entities
- */
- Set<String> getKnownEntities();
-
- /**
- * adds to the set of known entities. Overriding classes should hold this list in a class level set.
- * @param unambiguousEntity
- */
- void addKnownEntity(String unambiguousEntity);
-
- /**
- * defines the type of entity that the set contains, ie person, location, organization.
- * @return
- */
- String getKnownEntitiesType();
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+
+import java.util.Set;
+
+
+
+/**
+ *
+Supplies a list of known entities (a list of names or locations)
+ */
+public interface KnownEntityProvider extends ModelParameter<BaseModelBuilderParams> {
+
+ /**
+ * returns a list of known non ambiguous entities.
+ * @return a set of entities
+ */
+ Set<String> getKnownEntities();
+
+ /**
+ * adds to the set of known entities. Overriding classes should hold this list in a class level set.
+ * @param unambiguousEntity
+ */
+ void addKnownEntity(String unambiguousEntity);
+
+ /**
+ * defines the type of entity that the set contains, ie person, location, organization.
+ * @return
+ */
+ String getKnownEntitiesType();
+
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelGenerationValidator.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelGenerationValidator.java
index d05a1d5..a66f36f 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelGenerationValidator.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelGenerationValidator.java
@@ -1,35 +1,35 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-
-import java.util.Collection;
-
-/**
- *
-Validates results from the iterative namefinding
- */
-public interface ModelGenerationValidator extends ModelParameter<BaseModelBuilderParams> {
-
- Boolean validSentence(String sentence);
-
- Boolean validNamedEntity(String namedEntity);
-
-
-
- Collection<String> getBlackList();
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+
+import java.util.Collection;
+
+/**
+ *
+Validates results from the iterative namefinding
+ */
+public interface ModelGenerationValidator extends ModelParameter<BaseModelBuilderParams> {
+
+ Boolean validSentence(String sentence);
+
+ Boolean validNamedEntity(String namedEntity);
+
+
+
+ Collection<String> getBlackList();
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelParameter.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelParameter.java
index 136e775..5bc8e15 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelParameter.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelParameter.java
@@ -1,28 +1,28 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-
-/**
- *
- */
-public interface ModelParameter<T extends BaseModelBuilderParams>{
-
- void setParameters(T params);
-
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+
+/**
+ *
+ */
+public interface ModelParameter<T extends BaseModelBuilderParams>{
+
+ void setParameters(T params);
+
+
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/Modelable.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/Modelable.java
index 8d2e06c..36a33a9 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/Modelable.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/Modelable.java
@@ -1,45 +1,45 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import java.util.Set;
-
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-import opennlp.tools.namefind.TokenNameFinderModel;
-
-/**
- *
- */
-public interface Modelable extends ModelParameter<BaseModelBuilderParams> {
-
-
- String annotate(String sentence, String namedEntity, String entityType);
-
- void writeAnnotatedSentences();
-
- Set<String> getAnnotatedSentences();
-
- void setAnnotatedSentences(Set<String> annotatedSentences);
-
- void addAnnotatedSentence(String annotatedSentence);
-
- void buildModel( String entityType);
-
- TokenNameFinderModel getModel();
-
- String[] tokenizeSentenceToWords(String sentence);
-
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import java.util.Set;
+
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+import opennlp.tools.namefind.TokenNameFinderModel;
+
+/**
+ *
+ */
+public interface Modelable extends ModelParameter<BaseModelBuilderParams> {
+
+
+ String annotate(String sentence, String namedEntity, String entityType);
+
+ void writeAnnotatedSentences();
+
+ Set<String> getAnnotatedSentences();
+
+ void setAnnotatedSentences(Set<String> annotatedSentences);
+
+ void addAnnotatedSentence(String annotatedSentence);
+
+ void buildModel( String entityType);
+
+ TokenNameFinderModel getModel();
+
+ String[] tokenizeSentenceToWords(String sentence);
+
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SemiSupervisedModelGenerator.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SemiSupervisedModelGenerator.java
index c97a4c1..3bb303a 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SemiSupervisedModelGenerator.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SemiSupervisedModelGenerator.java
@@ -1,28 +1,28 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-
-/**
- *
-
- */
-public interface SemiSupervisedModelGenerator extends ModelParameter<BaseModelBuilderParams> {
-
- void build(SentenceProvider sentenceProvider, KnownEntityProvider knownEntityProvider,
- ModelGenerationValidator validator, Modelable modelable, int iterations);
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+
+/**
+ *
+
+ */
+public interface SemiSupervisedModelGenerator extends ModelParameter<BaseModelBuilderParams> {
+
+ void build(SentenceProvider sentenceProvider, KnownEntityProvider knownEntityProvider,
+ ModelGenerationValidator validator, Modelable modelable, int iterations);
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SentenceProvider.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SentenceProvider.java
index 5610224..83af594 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SentenceProvider.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/SentenceProvider.java
@@ -1,27 +1,27 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder;
-
-import java.util.Set;
-import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
-
-/**
- *
- */
-public interface SentenceProvider extends ModelParameter<BaseModelBuilderParams> {
-
- Set<String> getSentences();
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder;
+
+import java.util.Set;
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+
+/**
+ *
+ */
+public interface SentenceProvider extends ModelParameter<BaseModelBuilderParams> {
+
+ Set<String> getSentences();
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/BaseModelBuilderParams.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/BaseModelBuilderParams.java
index fcb2384..fff456b 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/BaseModelBuilderParams.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/BaseModelBuilderParams.java
@@ -1,90 +1,90 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder.impls;
-
-import java.io.File;
-import java.util.Map;
-
-/**
- *
- * Used to pass params through the processing
- */
-public class BaseModelBuilderParams {
-
- private File modelFile;
- private File sentenceFile;
- private File knownEntitiesFile;
- private File knownEntityBlacklist;
- private File annotatedTrainingDataFile;
- private String entityType;
- private Map<String, String> additionalParams;
-
- public File getModelFile() {
- return modelFile;
- }
-
- public void setModelFile(File modelFile) {
- this.modelFile = modelFile;
- }
-
- public File getSentenceFile() {
- return sentenceFile;
- }
-
- public void setSentenceFile(File sentenceFile) {
- this.sentenceFile = sentenceFile;
- }
-
- public File getKnownEntitiesFile() {
- return knownEntitiesFile;
- }
-
- public void setKnownEntitiesFile(File knownEntitiesFile) {
- this.knownEntitiesFile = knownEntitiesFile;
- }
-
- public File getKnownEntityBlacklist() {
- return knownEntityBlacklist;
- }
-
- public void setKnownEntityBlacklist(File knownEntityBlacklist) {
- this.knownEntityBlacklist = knownEntityBlacklist;
- }
-
- public Map<String, String> getAdditionalParams() {
- return additionalParams;
- }
-
- public void setAdditionalParams(Map<String, String> additionalParams) {
- this.additionalParams = additionalParams;
- }
-
- public String getEntityType() {
- return entityType;
- }
-
- public void setEntityType(String entityType) {
- this.entityType = entityType;
- }
-
- public File getAnnotatedTrainingDataFile() {
- return annotatedTrainingDataFile;
- }
-
- public void setAnnotatedTrainingDataFile(File annotatedTrainingDataFile) {
- this.annotatedTrainingDataFile = annotatedTrainingDataFile;
- }
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder.impls;
+
+import java.io.File;
+import java.util.Map;
+
+/**
+ *
+ * Used to pass params through the processing
+ */
+public class BaseModelBuilderParams {
+
+ private File modelFile;
+ private File sentenceFile;
+ private File knownEntitiesFile;
+ private File knownEntityBlacklist;
+ private File annotatedTrainingDataFile;
+ private String entityType;
+ private Map<String, String> additionalParams;
+
+ public File getModelFile() {
+ return modelFile;
+ }
+
+ public void setModelFile(File modelFile) {
+ this.modelFile = modelFile;
+ }
+
+ public File getSentenceFile() {
+ return sentenceFile;
+ }
+
+ public void setSentenceFile(File sentenceFile) {
+ this.sentenceFile = sentenceFile;
+ }
+
+ public File getKnownEntitiesFile() {
+ return knownEntitiesFile;
+ }
+
+ public void setKnownEntitiesFile(File knownEntitiesFile) {
+ this.knownEntitiesFile = knownEntitiesFile;
+ }
+
+ public File getKnownEntityBlacklist() {
+ return knownEntityBlacklist;
+ }
+
+ public void setKnownEntityBlacklist(File knownEntityBlacklist) {
+ this.knownEntityBlacklist = knownEntityBlacklist;
+ }
+
+ public Map<String, String> getAdditionalParams() {
+ return additionalParams;
+ }
+
+ public void setAdditionalParams(Map<String, String> additionalParams) {
+ this.additionalParams = additionalParams;
+ }
+
+ public String getEntityType() {
+ return entityType;
+ }
+
+ public void setEntityType(String entityType) {
+ this.entityType = entityType;
+ }
+
+ public File getAnnotatedTrainingDataFile() {
+ return annotatedTrainingDataFile;
+ }
+
+ public void setAnnotatedTrainingDataFile(File annotatedTrainingDataFile) {
+ this.annotatedTrainingDataFile = annotatedTrainingDataFile;
+ }
}
\ No newline at end of file
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileKnownEntityProvider.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileKnownEntityProvider.java
index 66c1c0e..7aa7d0c 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileKnownEntityProvider.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileKnownEntityProvider.java
@@ -1,80 +1,80 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder.impls;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import opennlp.addons.modelbuilder.KnownEntityProvider;
-
-/**
- *
- */
-public class FileKnownEntityProvider implements KnownEntityProvider {
-
- Set<String> knownEntities = new HashSet<>();
- BaseModelBuilderParams params;
- @Override
- public Set<String> getKnownEntities() {
- if (knownEntities.isEmpty()) {
- try {
- InputStream fis;
- BufferedReader br;
- String line;
-
- fis = new FileInputStream(params.getKnownEntitiesFile());
- br = new BufferedReader(new InputStreamReader(fis, Charset.forName("UTF-8")));
- while ((line = br.readLine()) != null) {
- knownEntities.add(line);
- }
-
- // Done with the file
- br.close();
- br = null;
- fis = null;
- } catch (IOException ex) {
- Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- return knownEntities;
- }
-
- @Override
- public void addKnownEntity(String unambiguousEntity) {
- knownEntities.add(unambiguousEntity);
- }
-
- @Override
- public String getKnownEntitiesType() {
-
- return params.getEntityType();
- }
-
-
-
- @Override
- public void setParameters(BaseModelBuilderParams params) {
- this.params = params;
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder.impls;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import opennlp.addons.modelbuilder.KnownEntityProvider;
+
+/**
+ *
+ */
+public class FileKnownEntityProvider implements KnownEntityProvider {
+
+ Set<String> knownEntities = new HashSet<>();
+ BaseModelBuilderParams params;
+ @Override
+ public Set<String> getKnownEntities() {
+ if (knownEntities.isEmpty()) {
+ try {
+ InputStream fis;
+ BufferedReader br;
+ String line;
+
+ fis = new FileInputStream(params.getKnownEntitiesFile());
+ br = new BufferedReader(new InputStreamReader(fis, Charset.forName("UTF-8")));
+ while ((line = br.readLine()) != null) {
+ knownEntities.add(line);
+ }
+
+ // Done with the file
+ br.close();
+ br = null;
+ fis = null;
+ } catch (IOException ex) {
+ Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ }
+ return knownEntities;
+ }
+
+ @Override
+ public void addKnownEntity(String unambiguousEntity) {
+ knownEntities.add(unambiguousEntity);
+ }
+
+ @Override
+ public String getKnownEntitiesType() {
+
+ return params.getEntityType();
+ }
+
+
+
+ @Override
+ public void setParameters(BaseModelBuilderParams params) {
+ this.params = params;
+ }
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileModelValidatorImpl.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileModelValidatorImpl.java
index 3951eef..a953c76 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileModelValidatorImpl.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileModelValidatorImpl.java
@@ -1,86 +1,86 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder.impls;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import opennlp.addons.modelbuilder.ModelGenerationValidator;
-
-/**
- *Validates NER results input before inclusion into the model
- */
-public class FileModelValidatorImpl implements ModelGenerationValidator {
-
- private final Set<String> badentities = new HashSet<>();
- BaseModelBuilderParams params;
-
- @Override
- public void setParameters(BaseModelBuilderParams params) {
- this.params = params;
- }
-
- @Override
- public Boolean validSentence(String sentence) {
- //returning true by default, because the sentence provider will return only "valid" sentences in this case
- return true;
- }
-
- @Override
- public Boolean validNamedEntity(String namedEntity) {
-
- if (badentities.isEmpty()) {
- getBlackList();
- }
-//
-// Pattern p = Pattern.compile("[0-9]", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
-// if (p.matcher(namedEntity).find()) {
-// return false;
-// }
- boolean b = true;
- if (badentities.contains(namedEntity.toLowerCase())) {
- b = false;
- }
- return b;
- }
-
- @Override
- public Collection<String> getBlackList() {
- if (params.getKnownEntityBlacklist() == null) {
- return badentities;
- }
- if (!badentities.isEmpty()) {
- try (BufferedReader br = new BufferedReader(new InputStreamReader(
- new FileInputStream(params.getKnownEntityBlacklist()), StandardCharsets.UTF_8))){
- String line;
- while ((line = br.readLine()) != null) {
- badentities.add(line);
- }
- } catch (IOException ex) {
- Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- return badentities;
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder.impls;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import opennlp.addons.modelbuilder.ModelGenerationValidator;
+
+/**
+ *Validates NER results input before inclusion into the model
+ */
+public class FileModelValidatorImpl implements ModelGenerationValidator {
+
+ private final Set<String> badentities = new HashSet<>();
+ BaseModelBuilderParams params;
+
+ @Override
+ public void setParameters(BaseModelBuilderParams params) {
+ this.params = params;
+ }
+
+ @Override
+ public Boolean validSentence(String sentence) {
+ //returning true by default, because the sentence provider will return only "valid" sentences in this case
+ return true;
+ }
+
+ @Override
+ public Boolean validNamedEntity(String namedEntity) {
+
+ if (badentities.isEmpty()) {
+ getBlackList();
+ }
+//
+// Pattern p = Pattern.compile("[0-9]", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
+// if (p.matcher(namedEntity).find()) {
+// return false;
+// }
+ boolean b = true;
+ if (badentities.contains(namedEntity.toLowerCase())) {
+ b = false;
+ }
+ return b;
+ }
+
+ @Override
+ public Collection<String> getBlackList() {
+ if (params.getKnownEntityBlacklist() == null) {
+ return badentities;
+ }
+ if (!badentities.isEmpty()) {
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(
+ new FileInputStream(params.getKnownEntityBlacklist()), StandardCharsets.UTF_8))){
+ String line;
+ while ((line = br.readLine()) != null) {
+ badentities.add(line);
+ }
+ } catch (IOException ex) {
+ Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ }
+ return badentities;
+ }
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileSentenceProvider.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileSentenceProvider.java
index 9f1f5e1..cb6715f 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileSentenceProvider.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileSentenceProvider.java
@@ -1,57 +1,57 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder.impls;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import opennlp.addons.modelbuilder.SentenceProvider;
-
-/**
- * Provides user sentences via a simple text file
- */
-public class FileSentenceProvider implements SentenceProvider {
-
- BaseModelBuilderParams params;
- private final Set<String> sentences = new HashSet<>();
-
- @Override
- public Set<String> getSentences() {
- if (sentences.isEmpty()) {
- try (BufferedReader br = new BufferedReader(new InputStreamReader(
- new FileInputStream(params.getSentenceFile()), StandardCharsets.UTF_8))){
- String line;
- while ((line = br.readLine()) != null) {
- sentences.add(line);
- }
- } catch (IOException ex) {
- Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
- }
- }
- return sentences;
- }
-
- public void setParameters(BaseModelBuilderParams params) {
- this.params = params;
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder.impls;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import opennlp.addons.modelbuilder.SentenceProvider;
+
+/**
+ * Provides user sentences via a simple text file
+ */
+public class FileSentenceProvider implements SentenceProvider {
+
+ BaseModelBuilderParams params;
+ private final Set<String> sentences = new HashSet<>();
+
+ @Override
+ public Set<String> getSentences() {
+ if (sentences.isEmpty()) {
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(
+ new FileInputStream(params.getSentenceFile()), StandardCharsets.UTF_8))){
+ String line;
+ while ((line = br.readLine()) != null) {
+ sentences.add(line);
+ }
+ } catch (IOException ex) {
+ Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ }
+ return sentences;
+ }
+
+ public void setParameters(BaseModelBuilderParams params) {
+ this.params = params;
+ }
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelGenerator.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelGenerator.java
index d09a664..126157f 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelGenerator.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelGenerator.java
@@ -1,103 +1,103 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder.impls;
-
-import java.util.HashMap;
-import java.util.Map;
-import opennlp.addons.modelbuilder.KnownEntityProvider;
-import opennlp.addons.modelbuilder.ModelGenerationValidator;
-import opennlp.addons.modelbuilder.Modelable;
-import opennlp.addons.modelbuilder.SemiSupervisedModelGenerator;
-import opennlp.addons.modelbuilder.SentenceProvider;
-import opennlp.tools.namefind.NameFinderME;
-import opennlp.tools.util.Span;
-
-/**
- *
- * Generic impl that handles all processing using the default file implementations
- */
-public class GenericModelGenerator implements SemiSupervisedModelGenerator {
-
- private Map<String, String> params = new HashMap<>();
-
- @Override
- public void setParameters(BaseModelBuilderParams params) {
- this.params = params.getAdditionalParams();
- }
-
- @Override
- public void build(SentenceProvider sentenceProvider, KnownEntityProvider knownEntityProvider,
- ModelGenerationValidator validator, Modelable modelable, int iterations) {
- for (int iteration = 0; iteration < iterations; iteration++) {
- System.out.println("ITERATION: " + iteration);
- System.out.println("\tPerfoming Known Entity Annotation");
- System.out.println("\t\tknowns: " + knownEntityProvider.getKnownEntities().size());
- System.out.println("\t\treading data....: ");
- for (String sentence : sentenceProvider.getSentences()) {
- for (String knownEntity : knownEntityProvider.getKnownEntities()) {
- if (sentence.contains(knownEntity)) {
- //if the same sentence has multiple hits should they be annotated separately?
- modelable.addAnnotatedSentence(modelable.annotate(sentence, knownEntity, knownEntityProvider.getKnownEntitiesType()));
- }
- }
- }
- if (sentenceProvider.getSentences().isEmpty()) {
- System.out.println("No sentences in file");
- return;
- }
- if (knownEntityProvider.getKnownEntities().isEmpty()) {
- System.out.println("No known entities in file");
- return;
- }
- System.out.println("\t\twriting annotated sentences....: ");
- modelable.writeAnnotatedSentences();
- System.out.println("\t\tbuilding model.... ");
- modelable.buildModel(knownEntityProvider.getKnownEntitiesType());
- System.out.println("\t\tmodel building complete.... ");
- NameFinderME nf = new NameFinderME(modelable.getModel());
- System.out.println("\t\tannotated sentences: " + modelable.getAnnotatedSentences().size());
- System.out.println("\tPerforming NER with new model");
- System.out.println("\t\tPrinting NER Results. Add undesired results to the blacklist file and start over");
- for (String sentence : sentenceProvider.getSentences()) {
- if (!validator.validSentence(sentence)) {
- continue;
- }
- String[] tokens = modelable.tokenizeSentenceToWords(sentence);
-
- Span[] find = nf.find(tokens);
- nf.clearAdaptiveData();
-
- String[] namedEntities = Span.spansToStrings(find, tokens);
-
- for (String namedEntity : namedEntities) {
- System.out.println("\t\t" + namedEntity);
- if (validator.validNamedEntity(namedEntity)) {
-
- knownEntityProvider.addKnownEntity(namedEntity);
- modelable.addAnnotatedSentence(modelable.annotate(sentence, namedEntity, knownEntityProvider.getKnownEntitiesType()));
-
- } else {
- System.out.println("\t\t" + namedEntity + "...already blacklisted");
- }
- }
- }
- System.out.println("\t\tannotated sentences: " + modelable.getAnnotatedSentences().size());
- System.out.println("\t\tknowns: " + knownEntityProvider.getKnownEntities().size());
- }
- modelable.writeAnnotatedSentences();
- modelable.buildModel(knownEntityProvider.getKnownEntitiesType());
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder.impls;
+
+import java.util.HashMap;
+import java.util.Map;
+import opennlp.addons.modelbuilder.KnownEntityProvider;
+import opennlp.addons.modelbuilder.ModelGenerationValidator;
+import opennlp.addons.modelbuilder.Modelable;
+import opennlp.addons.modelbuilder.SemiSupervisedModelGenerator;
+import opennlp.addons.modelbuilder.SentenceProvider;
+import opennlp.tools.namefind.NameFinderME;
+import opennlp.tools.util.Span;
+
+/**
+ *
+ * Generic impl that handles all processing using the default file implementations
+ */
+public class GenericModelGenerator implements SemiSupervisedModelGenerator {
+
+ private Map<String, String> params = new HashMap<>();
+
+ @Override
+ public void setParameters(BaseModelBuilderParams params) {
+ this.params = params.getAdditionalParams();
+ }
+
+ @Override
+ public void build(SentenceProvider sentenceProvider, KnownEntityProvider knownEntityProvider,
+ ModelGenerationValidator validator, Modelable modelable, int iterations) {
+ for (int iteration = 0; iteration < iterations; iteration++) {
+ System.out.println("ITERATION: " + iteration);
+ System.out.println("\tPerfoming Known Entity Annotation");
+ System.out.println("\t\tknowns: " + knownEntityProvider.getKnownEntities().size());
+ System.out.println("\t\treading data....: ");
+ for (String sentence : sentenceProvider.getSentences()) {
+ for (String knownEntity : knownEntityProvider.getKnownEntities()) {
+ if (sentence.contains(knownEntity)) {
+ //if the same sentence has multiple hits should they be annotated separately?
+ modelable.addAnnotatedSentence(modelable.annotate(sentence, knownEntity, knownEntityProvider.getKnownEntitiesType()));
+ }
+ }
+ }
+ if (sentenceProvider.getSentences().isEmpty()) {
+ System.out.println("No sentences in file");
+ return;
+ }
+ if (knownEntityProvider.getKnownEntities().isEmpty()) {
+ System.out.println("No known entities in file");
+ return;
+ }
+ System.out.println("\t\twriting annotated sentences....: ");
+ modelable.writeAnnotatedSentences();
+ System.out.println("\t\tbuilding model.... ");
+ modelable.buildModel(knownEntityProvider.getKnownEntitiesType());
+ System.out.println("\t\tmodel building complete.... ");
+ NameFinderME nf = new NameFinderME(modelable.getModel());
+ System.out.println("\t\tannotated sentences: " + modelable.getAnnotatedSentences().size());
+ System.out.println("\tPerforming NER with new model");
+ System.out.println("\t\tPrinting NER Results. Add undesired results to the blacklist file and start over");
+ for (String sentence : sentenceProvider.getSentences()) {
+ if (!validator.validSentence(sentence)) {
+ continue;
+ }
+ String[] tokens = modelable.tokenizeSentenceToWords(sentence);
+
+ Span[] find = nf.find(tokens);
+ nf.clearAdaptiveData();
+
+ String[] namedEntities = Span.spansToStrings(find, tokens);
+
+ for (String namedEntity : namedEntities) {
+ System.out.println("\t\t" + namedEntity);
+ if (validator.validNamedEntity(namedEntity)) {
+
+ knownEntityProvider.addKnownEntity(namedEntity);
+ modelable.addAnnotatedSentence(modelable.annotate(sentence, namedEntity, knownEntityProvider.getKnownEntitiesType()));
+
+ } else {
+ System.out.println("\t\t" + namedEntity + "...already blacklisted");
+ }
+ }
+ }
+ System.out.println("\t\tannotated sentences: " + modelable.getAnnotatedSentences().size());
+ System.out.println("\t\tknowns: " + knownEntityProvider.getKnownEntities().size());
+ }
+ modelable.writeAnnotatedSentences();
+ modelable.buildModel(knownEntityProvider.getKnownEntitiesType());
+ }
+}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelableImpl.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelableImpl.java
index 8c1f754..2630bc4 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelableImpl.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelableImpl.java
@@ -1,121 +1,121 @@
-/*
- * Copyright 2013 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package opennlp.addons.modelbuilder.impls;
-
-import java.io.BufferedOutputStream;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import opennlp.addons.modelbuilder.Modelable;
-import opennlp.tools.namefind.TokenNameFinderFactory;
-import opennlp.tools.util.MarkableFileInputStreamFactory;
-
-import opennlp.tools.namefind.NameFinderME;
-import opennlp.tools.namefind.NameSample;
-import opennlp.tools.namefind.NameSampleDataStream;
-import opennlp.tools.namefind.TokenNameFinderModel;
-
-import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
-import opennlp.tools.util.TrainingParameters;
-
-/**
- * Creates annotations, writes annotations to file, and creates a model and writes to a file
- */
-public class GenericModelableImpl implements Modelable {
-
- private Set<String> annotatedSentences = new HashSet<>();
- BaseModelBuilderParams params;
-
- @Override
- public void setParameters(BaseModelBuilderParams params) {
- this.params = params;
- }
-
- @Override
- public String annotate(String sentence, String namedEntity, String entityType) {
- return sentence.replace(namedEntity, " <START:" + entityType + "> " + namedEntity + " <END> ");
- }
-
- @Override
- public void writeAnnotatedSentences() {
- try (Writer bw = new BufferedWriter(new FileWriter(params.getAnnotatedTrainingDataFile(), false))) {
- for (String s : annotatedSentences) {
- bw.write(s.replace("\n", " ").trim() + "\n");
- }
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- @Override
- public Set<String> getAnnotatedSentences() {
- return annotatedSentences;
- }
-
- @Override
- public void setAnnotatedSentences(Set<String> annotatedSentences) {
- this.annotatedSentences = annotatedSentences;
- }
-
- @Override
- public void addAnnotatedSentence(String annotatedSentence) {
- annotatedSentences.add(annotatedSentence);
- }
-
- @Override
- public void buildModel(String entityType) {
- try (ObjectStream<NameSample> sampleStream = new NameSampleDataStream(new PlainTextByLineStream(
- new MarkableFileInputStreamFactory(params.getAnnotatedTrainingDataFile()), StandardCharsets.UTF_8));
- OutputStream modelOut = new BufferedOutputStream(new FileOutputStream(params.getModelFile()))) {
- System.out.println("\tBuilding Model using " + annotatedSentences.size() + " annotations");
- System.out.println("\t\treading training data...");
- TokenNameFinderModel model;
- model = NameFinderME.train("en", entityType, sampleStream,
- TrainingParameters.defaultParams(), new TokenNameFinderFactory());
- sampleStream.close();
- model.serialize(modelOut);
- System.out.println("\tmodel generated");
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public TokenNameFinderModel getModel() {
- TokenNameFinderModel nerModel = null;
- try {
- nerModel = new TokenNameFinderModel(new FileInputStream(params.getModelFile()));
- } catch (IOException ex) {
- Logger.getLogger(GenericModelableImpl.class.getName()).log(Level.SEVERE, null, ex);
- }
- return nerModel;
- }
-
- @Override
- public String[] tokenizeSentenceToWords(String sentence) {
- return sentence.split("\\s+");
- }
-}
+/*
+ * Copyright 2013 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.addons.modelbuilder.impls;
+
+import java.io.BufferedOutputStream;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import opennlp.addons.modelbuilder.Modelable;
+import opennlp.tools.namefind.TokenNameFinderFactory;
+import opennlp.tools.util.MarkableFileInputStreamFactory;
+
+import opennlp.tools.namefind.NameFinderME;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.namefind.NameSampleDataStream;
+import opennlp.tools.namefind.TokenNameFinderModel;
+
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.PlainTextByLineStream;
+import opennlp.tools.util.TrainingParameters;
+
+/**
+ * Creates annotations, writes annotations to file, and creates a model and writes to a file
+ */
+public class GenericModelableImpl implements Modelable {
+
+ private Set<String> annotatedSentences = new HashSet<>();
+ BaseModelBuilderParams params;
+
+ @Override
+ public void setParameters(BaseModelBuilderParams params) {
+ this.params = params;
+ }
+
+ @Override
+ public String annotate(String sentence, String namedEntity, String entityType) {
+ return sentence.replace(namedEntity, " <START:" + entityType + "> " + namedEntity + " <END> ");
+ }
+
+ @Override
+ public void writeAnnotatedSentences() {
+ try (Writer bw = new BufferedWriter(new FileWriter(params.getAnnotatedTrainingDataFile(), false))) {
+ for (String s : annotatedSentences) {
+ bw.write(s.replace("\n", " ").trim() + "\n");
+ }
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ @Override
+ public Set<String> getAnnotatedSentences() {
+ return annotatedSentences;
+ }
+
+ @Override
+ public void setAnnotatedSentences(Set<String> annotatedSentences) {
+ this.annotatedSentences = annotatedSentences;
+ }
+
+ @Override
+ public void addAnnotatedSentence(String annotatedSentence) {
+ annotatedSentences.add(annotatedSentence);
+ }
+
+ @Override
+ public void buildModel(String entityType) {
+ try (ObjectStream<NameSample> sampleStream = new NameSampleDataStream(new PlainTextByLineStream(
+ new MarkableFileInputStreamFactory(params.getAnnotatedTrainingDataFile()), StandardCharsets.UTF_8));
+ OutputStream modelOut = new BufferedOutputStream(new FileOutputStream(params.getModelFile()))) {
+ System.out.println("\tBuilding Model using " + annotatedSentences.size() + " annotations");
+ System.out.println("\t\treading training data...");
+ TokenNameFinderModel model;
+ model = NameFinderME.train("en", entityType, sampleStream,
+ TrainingParameters.defaultParams(), new TokenNameFinderFactory());
+ sampleStream.close();
+ model.serialize(modelOut);
+ System.out.println("\tmodel generated");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public TokenNameFinderModel getModel() {
+ TokenNameFinderModel nerModel = null;
+ try {
+ nerModel = new TokenNameFinderModel(new FileInputStream(params.getModelFile()));
+ } catch (IOException ex) {
+ Logger.getLogger(GenericModelableImpl.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ return nerModel;
+ }
+
+ @Override
+ public String[] tokenizeSentenceToWords(String sentence) {
+ return sentence.split("\\s+");
+ }
+}
diff --git a/morfologik-addon/pom.xml b/morfologik-addon/pom.xml
index 92b6f72..87928ff 100644
--- a/morfologik-addon/pom.xml
+++ b/morfologik-addon/pom.xml
@@ -1,152 +1,152 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-addons</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>morfologik-addon</artifactId>
- <version>2.2.1-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Apache OpenNLP Morfologik Addon</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-tools</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.carrot2</groupId>
- <artifactId>morfologik-stemming</artifactId>
- <version>2.1.9</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.carrot2</groupId>
- <artifactId>morfologik-tools</artifactId>
- <version>2.1.9</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-engine</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-params</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>${maven.compiler.source}</source>
- <target>${maven.compiler.target}</target>
- <compilerArgument>-Xlint</compilerArgument>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>bundle-project-sources</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <descriptors>
- <descriptor>src/main/assembly/bin.xml</descriptor>
- <descriptor>src/main/assembly/src.xml</descriptor>
- </descriptors>
- <!-- Tar package is only compatible with gnu tar,
- many file have more than 100 chars.
- Right now only javadoc files are too long.
- -->
- <tarLongFileMode>gnu</tarLongFileMode>
-
- <finalName>apache-opennlp-morfologik-addon-${project.version}</finalName>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.6</version>
- <executions>
- <execution>
- <id>generate checksums for binary artifacts</id>
- <goals><goal>run</goal></goals>
- <phase>verify</phase>
- <configuration>
- <target>
- <checksum algorithm="sha1" format="MD5SUM">
- <fileset dir="${project.build.directory}">
- <include name="*.zip" />
- <include name="*.gz" />
- </fileset>
- </checksum>
- <checksum algorithm="md5" format="MD5SUM">
- <fileset dir="${project.build.directory}">
- <include name="*.zip" />
- <include name="*.gz" />
- </fileset>
- </checksum>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
... 142 lines suppressed ...
[opennlp-addons] 01/02: adjusts 'opennlp-addons' to be compatible with latest opennlp-tools release
Posted by rz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/opennlp-addons.git
commit 618eb9a2fc1a06c22966bb921f32a672a7fa0b79
Author: Martin Wiesner <ma...@hs-heilbronn.de>
AuthorDate: Sun Apr 30 19:11:26 2023 +0200
adjusts 'opennlp-addons' to be compatible with latest opennlp-tools release
- adjusts opennlp-tools to 2.2.0
- adjusts parent project (org.apache.apache) to version 29
- adjusts Java language level to 11
- fixes compile incompatibilities
- transforms existing JUnit tests towards JUnit 5.x
- updates morfologik dependency to version 2.1.9
- improves resource handling in some spots
- modernizes code style along the path
- adds GH actions config
- adjusts .gitignore to cover an additional IDE flavor
- improves JavaDoc along the path
---
.github/CONTRIBUTING.md | 11 +
.github/PULL_REQUEST_TEMPLATE.md | 27 ++
.github/workflows/maven.yml | 51 ++
.gitignore | 2 +
NOTICE | 2 +-
geoentitylinker-addon/pom.xml | 77 ++-
.../geoentitylinker/AdminBoundaryContext.java | 2 +-
.../AdminBoundaryContextGenerator.java | 60 +--
.../addons/geoentitylinker/GazetteerEntry.java | 37 +-
.../addons/geoentitylinker/GazetteerSearcher.java | 77 ++-
.../addons/geoentitylinker/GeoEntityLinker.java | 69 ++-
.../geoentitylinker/indexing/GazetteerIndexer.java | 84 ++--
.../indexing/GeonamesFileDownloader.java | 51 +-
.../indexing/GeonamesProcessor.java | 14 +-
.../geoentitylinker/indexing/RegionProcessor.java | 2 +-
.../geoentitylinker/indexing/USGSProcessor.java | 161 +++----
.../scoring/CountryProximityScorer.java | 38 +-
.../scoring/FuzzyStringMatchScorer.java | 3 +-
.../scoring/GeoHashBinningScorer.java | 5 +-
.../scoring/LinkedEntityScorer.java | 2 +-
.../geoentitylinker/scoring/ModelBasedScorer.java | 29 +-
.../geoentitylinker/scoring/PlacetypeScorer.java | 4 +-
.../geoentitylinker/scoring/PointClustering.java | 12 +-
.../scoring/ProvinceProximityScorer.java | 18 +-
.../test/java/apache/opennlp/addons/AppTest.java | 38 --
japanese-addon/build.xml | 2 +-
japanese-addon/pom.xml | 89 +++-
.../AuxiliaryInfoNameContextGenerator.java | 2 +-
.../lang/jpn/BrownTokenClassFeatureGenerator.java | 4 +-
.../tools/namefind/AuxiliaryInfoUtilTest.java | 53 +-
...liaryInfoAwareDelegateFeatureGeneratorTest.java | 17 +-
.../lang/jpn/BigramNameFeatureGeneratorTest.java | 36 +-
.../lang/jpn/FeatureGeneratorUtilTest.java | 59 +--
.../lang/jpn/TokenClassFeatureGeneratorTest.java | 21 +-
.../lang/jpn/TokenPatternFeatureGeneratorTest.java | 45 +-
.../lang/jpn/TrigramNameFeatureGeneratorTest.java | 41 +-
jwnl-addon/pom.xml | 86 ++--
.../opennlp/jwnl/lemmatizer/JWNLLemmatizer.java | 35 +-
liblinear-addon/pom.xml | 41 +-
liblinear-addon/src/main/java/LiblinearModel.java | 135 +++---
.../src/main/java/LiblinearTrainer.java | 86 +---
modelbuilder-addon/pom.xml | 73 ++-
.../modelbuilder/DefaultModelBuilderUtil.java | 38 +-
.../addons/modelbuilder/KnownEntityProvider.java | 33 +-
.../modelbuilder/ModelGenerationValidator.java | 4 +-
.../opennlp/addons/modelbuilder/Modelable.java | 6 +-
.../impls/FileKnownEntityProvider.java | 4 +-
.../modelbuilder/impls/FileModelValidatorImpl.java | 22 +-
.../modelbuilder/impls/FileSentenceProvider.java | 27 +-
.../modelbuilder/impls/GenericModelGenerator.java | 2 +-
.../modelbuilder/impls/GenericModelableImpl.java | 39 +-
.../src/test/java/modelbuilder/AppTest.java | 38 --
morfologik-addon/pom.xml | 181 ++++---
.../main/java/opennlp/morfologik/cmdline/CLI.java | 4 +-
.../cmdline/builder/XMLDictionaryToTableTool.java | 4 +-
.../lemmatizer/MorfologikLemmatizer.java | 36 +-
.../tagdict/MorfologikPOSTaggerFactory.java | 36 +-
.../tagdict/MorfologikTagDictionary.java | 26 +-
.../builder/POSDictionayBuilderTest.java | 8 +-
.../lemmatizer/MorfologikLemmatizerTest.java | 36 +-
.../tagdict/MorfologikTagDictionaryTest.java | 33 +-
.../morfologik/tagdict/POSTaggerFactoryTest.java | 57 ++-
pom.xml | 532 +++++++++++++++++++++
63 files changed, 1747 insertions(+), 1120 deletions(-)
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
new file mode 100644
index 0000000..5e06386
--- /dev/null
+++ b/.github/CONTRIBUTING.md
@@ -0,0 +1,11 @@
+# How to contribute to Apache OpenNLP
+
+Thank you for your intention to contribute to the Apache OpenNLP project. As an open-source community, we highly appreciate external contributions to our project.
+
+To make the process smooth for the project *committers* (those who review and accept changes) and *contributors* (those who propose new changes via pull requests), there are a few rules to follow.
+
+## Contribution Guidelines
+
+Please check out the [How to get involved](https://opennlp.apache.org/get-involved.html) to understand how contributions are made.
+A detailed list of coding standards can be found at [Apache OpenNLP Code Conventions](https://opennlp.apache.org/code-conventions.html) which also contains a list of coding guidelines that you should follow.
+For pull requests, there is a [checklist](PULL_REQUEST_TEMPLATE.md) with criteria for acceptable contributions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 0000000..3f6f388
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,27 @@
+Thank you for contributing to Apache OpenNLP.
+
+In order to streamline the review of the contribution we ask you
+to ensure the following steps have been taken:
+
+### For all changes:
+- [ ] Is there a JIRA ticket associated with this PR? Is it referenced
+ in the commit message?
+
+- [ ] Does your PR title start with OPENNLP-XXXX where XXXX is the JIRA number you are trying to resolve? Pay particular attention to the hyphen "-" character.
+
+- [ ] Has your PR been rebased against the latest commit within the target branch (typically main)?
+
+- [ ] Is your initial contribution a single, squashed commit?
+
+### For code changes:
+- [ ] Have you ensured that the full suite of tests is executed via `mvn clean install` at the root opennlp-sandbox folder?
+- [ ] Have you written or updated unit tests to verify your changes?
+- [ ] If adding new dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under [ASF 2.0](https://www.apache.org/legal/resolved.html#category-a)?
+- [ ] If applicable, have you updated the LICENSE file, including the main LICENSE file in opennlp-sandbox folder?
+- [ ] If applicable, have you updated the NOTICE file, including the main NOTICE file found in opennlp-sandbox folder?
+
+### For documentation related changes:
+- [ ] Have you ensured that format looks appropriate for the output in which it is rendered?
+
+### Note:
+Please ensure that once the PR is submitted, you check GitHub Actions for build issues and submit an update to your PR as soon as possible.
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
new file mode 100644
index 0000000..a31237c
--- /dev/null
+++ b/.github/workflows/maven.yml
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Java CI
+
+on: [push, pull_request]
+
+jobs:
+ build:
+ runs-on: ${{ matrix.os }}
+ continue-on-error: ${{ matrix.experimental }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-latest]
+ java: [ 11, 17, 19 ]
+ experimental: [false]
+# include:
+# - java: 18-ea
+# os: ubuntu-latest
+# experimental: true
+
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/cache@v3
+ with:
+ path: ~/.m2/repository
+ key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-maven-
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v3
+ with:
+ distribution: adopt
+ java-version: ${{ matrix.java }}
+ - name: Build with Maven
+ run: mvn -V clean test install --no-transfer-progress -Pjacoco
+ - name: Jacoco
+ run: mvn jacoco:report
diff --git a/.gitignore b/.gitignore
index 1f7965a..a70e935 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+*.iml
+.idea
target
.classpath
.project
diff --git a/NOTICE b/NOTICE
index deb40c8..a892eb2 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
Apache OpenNLP
-Copyright 2010, 2014 The Apache Software Foundation
+Copyright 2010, 2023 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/geoentitylinker-addon/pom.xml b/geoentitylinker-addon/pom.xml
index 07d03cc..75be328 100644
--- a/geoentitylinker-addon/pom.xml
+++ b/geoentitylinker-addon/pom.xml
@@ -1,3 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -22,48 +24,21 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp</artifactId>
- <version>1.6.0</version>
- <relativePath>../opennlp/pom.xml</relativePath>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
</parent>
<artifactId>geoentitylinker-addon</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <version>2.2.1-SNAPSHOT</version>
<packaging>jar</packaging>
- <name>geoentitylinker-addon</name>
-
- <url>http://maven.apache.org</url>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>2.3.2</version>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
+ <name>Apache OpenNLP GeoentityLinker Addon</name>
<dependencies>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>3.8.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>1.2.16</version>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-tools</artifactId>
</dependency>
-
-
+
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
@@ -79,16 +54,40 @@
<artifactId>lucene-queryparser</artifactId>
<version>6.0.0</version>
</dependency>
- <dependency>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-tools</artifactId>
- <version>1.6.0</version>
- </dependency>
<dependency>
<groupId>com.spatial4j</groupId>
<artifactId>spatial4j</artifactId>
<version>0.4.1</version>
<type>jar</type>
</dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ </dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerArgument>-Xlint</compilerArgument>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContext.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContext.java
index 4fccffb..beadf4e 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContext.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContext.java
@@ -110,7 +110,7 @@ public class AdminBoundaryContext {
private Set<String> setWhereClauses() {
Set<String> clauses = new HashSet<>();
for (String countryCode : this.getCountryHits()) {
- String gazType = countryCode.toLowerCase().equals("us") ? " AND gazsource:usgs" : " AND gazsource:geonames";
+ String gazType = countryCode.equalsIgnoreCase("us") ? " AND gazsource:usgs" : " AND gazsource:geonames";
if (countryCode.toLowerCase().matches(".*rg[0-9].*")) {
gazType = " AND gazsource:region";
}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContextGenerator.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContextGenerator.java
index b645156..a6741fe 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContextGenerator.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/AdminBoundaryContextGenerator.java
@@ -19,6 +19,7 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -30,7 +31,8 @@ import java.util.logging.Level;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import opennlp.tools.entitylinker.EntityLinkerProperties;
-import org.apache.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Finds instances of country mentions in a String, typically a document text.
@@ -39,19 +41,19 @@ import org.apache.log4j.Logger;
*/
public class AdminBoundaryContextGenerator {
- private static final Logger LOGGER = Logger.getLogger(AdminBoundaryContextGenerator.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private List<CountryContextEntry> countrydata;
private Map<String, Set<String>> nameCodesMap = new HashMap<>();
- private Map<String, Set<Integer>> countryMentions = new HashMap<>();
+ private final Map<String, Set<Integer>> countryMentions = new HashMap<>();
Map<String, String> countryRegexMap = new HashMap<>();
Map<String, String> provinceRegexMap = new HashMap<>();
Map<String, String> countyRegexMap = new HashMap<>();
- private Set<CountryContextEntry> countryHits = new HashSet<>();
- private EntityLinkerProperties properties;
- private List<AdminBoundary> adminBoundaryData= new ArrayList<>();
- private Set<AdminBoundary> adminBoundaryHits = new HashSet<>();
+ private final Set<CountryContextEntry> countryHits = new HashSet<>();
+ private final EntityLinkerProperties properties;
+ private final List<AdminBoundary> adminBoundaryData= new ArrayList<>();
+ private final Set<AdminBoundary> adminBoundaryHits = new HashSet<>();
private AdminBoundaryContext context;
public AdminBoundaryContext getContext(String text) {
@@ -62,16 +64,16 @@ public class AdminBoundaryContextGenerator {
return context;
}
- private Set<String> countryHitSet = new HashSet<>();
- private Map<String, String> countryMap = new HashMap<>();
- private Map<String, Map<String, String>> provMap = new HashMap<>();
- private Map<String, Map<String, String>> countyMap = new HashMap<>();
+ private final Set<String> countryHitSet = new HashSet<>();
+ private final Map<String, String> countryMap = new HashMap<>();
+ private final Map<String, Map<String, String>> provMap = new HashMap<>();
+ private final Map<String, Map<String, String>> countyMap = new HashMap<>();
private Map<String, Set<Integer>> provMentions = new HashMap<>();
private Map<String, Set<Integer>> countyMentions = new HashMap<>();
- private Set<String> provHits = new HashSet<String>();
- private Set<String> countyHits = new HashSet<String>();
+ private final Set<String> provHits = new HashSet<>();
+ private final Set<String> countyHits = new HashSet<>();
public static void main(String[] args) {
try {
@@ -109,18 +111,14 @@ public class AdminBoundaryContextGenerator {
}
/**
- * returns the last set of hits after calling regexFind
- *
- * @return
+ * @return returns the last set of hits after calling regexFind
*/
public Set<CountryContextEntry> getCountryHits() {
return countryHits;
}
/**
- * returns the last name to codes map after calling regexFind
- *
- * @return
+ * @return returns the last name to codes map after calling regexFind
*/
public Map<String, Set<String>> getNameCodesMap() {
return nameCodesMap;
@@ -148,11 +146,9 @@ public class AdminBoundaryContextGenerator {
* downstream. The full text of a document should be passed in here.
*
* @param text the full text of the document (block of text).
- * @return
*/
private AdminBoundaryContext process(String text) {
try {
-
reset();
Map<String, Set<Integer>> countryhitMap = regexfind(text, countryMap, countryHitSet, "country");
if (!countryhitMap.isEmpty()) {
@@ -203,13 +199,10 @@ public class AdminBoundaryContextGenerator {
}
}
- AdminBoundaryContext context
- = new AdminBoundaryContext(countryhitMap, provMentions, countyMentions, countryHitSet, provHits, countyHits,
+ return new AdminBoundaryContext(countryhitMap, provMentions, countyMentions, countryHitSet, provHits, countyHits,
countryRefMap, provMap, countyMap, nameCodesMap, countryRegexMap, provinceRegexMap, countyRegexMap);
-
- return context;
} catch (Exception e) {
- e.printStackTrace();
+ LOG.error(e.getLocalizedMessage(), e);
}
return null;
}
@@ -221,7 +214,6 @@ public class AdminBoundaryContextGenerator {
* @param lookupMap a map to use to find names. the key=a location code, the
* value is an actual name.
* @param hitsRef a reference to a set that stores the hits by id
- * @return
*/
private Map<String, Set<Integer>> regexfind(String docText, Map<String, String> lookupMap, Set<String> hitsRef, String locationType) {
Map<String, Set<Integer>> mentions = new HashMap<>();
@@ -268,7 +260,7 @@ public class AdminBoundaryContextGenerator {
if (mentions.containsKey(code)) {
mentions.get(code).add(start);
} else {
- Set<Integer> newset = new HashSet<Integer>();
+ Set<Integer> newset = new HashSet<>();
newset.add(start);
mentions.put(code, newset);
}
@@ -276,7 +268,7 @@ public class AdminBoundaryContextGenerator {
if (this.nameCodesMap.containsKey(hit)) {
nameCodesMap.get(hit).add(code);
} else {
- HashSet<String> newset = new HashSet<String>();
+ HashSet<String> newset = new HashSet<>();
newset.add(code);
nameCodesMap.put(hit, newset);
}
@@ -290,9 +282,7 @@ public class AdminBoundaryContextGenerator {
}
} catch (Exception ex) {
- LOGGER.error(ex);
- ex.printStackTrace();
-
+ LOG.error(ex.getLocalizedMessage(), ex);
}
return mentions;
@@ -306,7 +296,7 @@ public class AdminBoundaryContextGenerator {
BufferedReader reader;
try {
reader = new BufferedReader(new FileReader(countryContextFile));
- String line = "";
+ String line;
int lineNum = 0;
while ((line = reader.readLine()) != null) {
String[] values = line.split("\t");
@@ -334,7 +324,7 @@ public class AdminBoundaryContextGenerator {
}
reader.close();
} catch (IOException ex) {
- LOGGER.error(ex);
+ LOG.error(ex.getLocalizedMessage(), ex);
}
loadMaps(this.adminBoundaryData);
@@ -365,7 +355,7 @@ public class AdminBoundaryContextGenerator {
provMap.put(adm.getCountryCode(), provs);
// }
- if (!adm.getCountyCode().toLowerCase().equals("no_data_found") && !adm.getCountyName().toLowerCase().equals("no_data_found")) {
+ if (!adm.getCountyCode().equalsIgnoreCase("no_data_found") && !adm.getCountyName().equalsIgnoreCase("no_data_found")) {
Map<String, String> counties = countyMap.get(adm.getCountryCode() + "." + adm.getProvCode());
if (counties == null) {
counties = new HashMap<>();
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerEntry.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerEntry.java
index 6f3ac87..56e6eef 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerEntry.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerEntry.java
@@ -21,8 +21,7 @@ import java.util.Objects;
import opennlp.tools.entitylinker.BaseLink;
/**
- *
- * Stores a minimal amount of information from a geographic placenames gazateer
+ * Stores a minimal amount of information from a geographic placenames gazetteer.
*/
public class GazetteerEntry extends BaseLink {
@@ -35,43 +34,39 @@ public class GazetteerEntry extends BaseLink {
private String provinceCode;
private String hierarchy;
+ public GazetteerEntry(String parentID, String itemID, String itemName, String itemType) {
+ super(parentID, itemID, itemName, itemType);
+ }
+
/**
- * returns the id from the lucene document
- *
- * @return
+ * @return returns the id from the lucene document
*/
public String getIndexID() {
return indexID;
}
- /*
+ /**
* sets the id from the lucene document
*/
-
public void setIndexID(String indexID) {
this.indexID = indexID;
}
/**
- * returns the latitude from the gazateer
- *
- * @return
+ * @return Retrieves the latitude from the gazetteer
*/
public Double getLatitude() {
return latitude;
}
/**
- * sets the latitude from the gazateer
- *
+ * sets the latitude from the gazetteer
*/
public void setLatitude(Double latitude) {
this.latitude = latitude;
}
/**
- * returns the longitude from the gaz
- *
- * @return
+ * @return Retrieves the longitude from the gaz
*/
public Double getLongitude() {
return longitude;
@@ -87,16 +82,14 @@ public class GazetteerEntry extends BaseLink {
}
/**
- * returns the source of the gazateer data
- *
- * @return
+ * @return Retrieves the source of the gazetteer data
*/
public String getSource() {
return source;
}
/**
- * sets the source (the source of the gazateer data)
+ * sets the source (the source of the gazetteer data)
*
* @param source
*/
@@ -105,16 +98,14 @@ public class GazetteerEntry extends BaseLink {
}
/**
- * Returns all the other fields in the gazateer in the form of a map
- *
- * @return
+ * @return Retrieves all the other fields in the gazetteer in the form of a map
*/
public Map<String, String> getIndexData() {
return indexData;
}
/**
- * sets the other fields in the gazeteer in the form of a map
+ * sets the other fields in the gazetteer in the form of a map
*
* @param indexData stores all fields in the index as fieldname:value
*/
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearcher.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearcher.java
index e18253d..06e0365 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearcher.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GazetteerSearcher.java
@@ -17,12 +17,12 @@ package opennlp.addons.geoentitylinker;
import java.io.File;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.logging.Level;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
@@ -37,12 +37,12 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
-import org.apache.lucene.util.Version;
import opennlp.tools.entitylinker.EntityLinkerProperties;
-import org.apache.log4j.Logger;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
@@ -53,12 +53,12 @@ import org.apache.lucene.analysis.util.CharArraySet;
public class GazetteerSearcher {
private final String REGEX_CLEAN = "[^\\p{L}\\p{Nd}]";
- private static final Logger LOGGER = Logger.getLogger(GazetteerSearcher.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private double scoreCutoff = .70;
- private boolean doubleQuoteAllSearchTerms = false;
+ private final boolean doubleQuoteAllSearchTerms = false;
private boolean useHierarchyField = false;
- private EntityLinkerProperties properties;
+ private final EntityLinkerProperties properties;
private Directory opennlpIndex;//= new MMapDirectory(new File(indexloc));
private IndexReader opennlpReader;// = DirectoryReader.open(geonamesIndex);
@@ -67,13 +67,10 @@ public class GazetteerSearcher {
public static void main(String[] args) {
try {
- boolean b = Boolean.valueOf("true");
-
+ boolean b = true;
new GazetteerSearcher(new EntityLinkerProperties(new File("c:\\temp\\entitylinker.properties"))).find("alabama", 5, " countrycode:us AND gazsource:usgs");
} catch (IOException ex) {
- java.util.logging.Logger.getLogger(GazetteerSearcher.class.getName()).log(Level.SEVERE, null, ex);
- } catch (Exception ex) {
- java.util.logging.Logger.getLogger(GazetteerSearcher.class.getName()).log(Level.SEVERE, null, ex);
+ LOG.error(ex.getLocalizedMessage(), ex);
}
}
@@ -98,7 +95,7 @@ public class GazetteerSearcher {
return linkedData;
}
try {
- /**
+ /*
* build the search string Sometimes no country context is found. In this
* case the code variables will be empty strings
*/
@@ -108,7 +105,7 @@ public class GazetteerSearcher {
+ " AND " + whereClause;
}
- /**
+ /*
* check the cache and go no further if the records already exist
*/
ArrayList<GazetteerEntry> get = GazetteerSearchCache.get(placeNameQueryString);
@@ -116,7 +113,7 @@ public class GazetteerSearcher {
return get;
}
- /**
+ /*
* search the placename
*/
QueryParser parser = new QueryParser(placeNameQueryString, opennlpAnalyzer);
@@ -126,17 +123,12 @@ public class GazetteerSearcher {
TopDocs bestDocs = opennlpSearcher.search(q, rowsReturned);
Double maxscore = 0d;
for (int i = 0; i < bestDocs.scoreDocs.length; ++i) {
- GazetteerEntry entry = new GazetteerEntry();
int docId = bestDocs.scoreDocs[i].doc;
double sc = bestDocs.scoreDocs[i].score;
if (maxscore.compareTo(sc) < 0) {
maxscore = sc;
}
- entry.getScoreMap().put("lucene", sc);
- entry.setIndexID(docId + "");
-
Document d = opennlpSearcher.doc(docId);
-
List<IndexableField> fields = d.getFields();
String lat = d.get("latitude");
@@ -147,41 +139,39 @@ public class GazetteerSearcher {
String itemtype = d.get("loctype");
String source = d.get("gazsource");
String hier = d.get("hierarchy");
- entry.setSource(source);
- entry.setItemID(docId + "");
- entry.setLatitude(Double.valueOf(lat));
- entry.setLongitude(Double.valueOf(lon));
- entry.setItemType(itemtype);
- entry.setItemParentID(parentid);
- entry.setProvinceCode(provid);
- entry.setCountryCode(parentid);
- entry.setItemName(placename);
- entry.setHierarchy(hier);
- for (int idx = 0; idx < fields.size(); idx++) {
- entry.getIndexData().put(fields.get(idx).name(), d.get(fields.get(idx).name()));
+ GazetteerEntry ge = new GazetteerEntry(parentid, String.valueOf(docId), placename, itemtype);
+ ge.getScoreMap().put("lucene", sc);
+ ge.setIndexID(String.valueOf(docId));
+ ge.setSource(source);
+ ge.setLatitude(Double.valueOf(lat));
+ ge.setLongitude(Double.valueOf(lon));
+ ge.setProvinceCode(provid);
+ ge.setCountryCode(parentid);
+ ge.setHierarchy(hier);
+ for (IndexableField field : fields) {
+ ge.getIndexData().put(field.name(), d.get(field.name()));
}
- /**
- * only want hits above the levenstein thresh. This should be a low
+ /*
+ * only want hits above the levenshtein thresh. This should be a low
* thresh due to the use of the hierarchy field in the index
*/
// if (normLev > scoreCutoff) {
- if (entry.getItemParentID().toLowerCase().equals(parentid.toLowerCase()) || parentid.toLowerCase().equals("")) {
+ if (ge.getItemParentID().equalsIgnoreCase(parentid) || parentid.equalsIgnoreCase("")) {
//make sure we don't produce a duplicate
- if (!linkedData.contains(entry)) {
- linkedData.add(entry);
- /**
+ if (!linkedData.contains(ge)) {
+ linkedData.add(ge);
+ /*
* add the records to the cache for this query
*/
GazetteerSearchCache.put(placeNameQueryString, linkedData);
}
}
- //}
}
} catch (IOException | ParseException ex) {
- LOGGER.error(ex);
+ LOG.error(ex.getLocalizedMessage(), ex);
}
return linkedData;
@@ -210,8 +200,7 @@ public class GazetteerSearcher {
if (opennlpIndex == null) {
String indexloc = properties.getProperty("opennlp.geoentitylinker.gaz", "");
if (indexloc.equals("")) {
- LOGGER.error(new Exception("Opennlp combined Gaz directory location not found"));
-
+ LOG.error("Opennlp combined Gaz directory location not found!");
}
opennlpIndex = new MMapDirectory(Paths.get(indexloc));
@@ -219,7 +208,7 @@ public class GazetteerSearcher {
opennlpSearcher = new IndexSearcher(opennlpReader);
opennlpAnalyzer
= //new StandardAnalyzer(Version.LUCENE_48, new CharArraySet(Version.LUCENE_48, new ArrayList(), true));
- new StandardAnalyzer(new CharArraySet(new ArrayList(), true));
+ new StandardAnalyzer(new CharArraySet(new ArrayList<>(), true));
Map<String, Analyzer> analyMap = new HashMap<>();
analyMap.put("countrycode", new KeywordAnalyzer());
@@ -234,10 +223,10 @@ public class GazetteerSearcher {
String cutoff = properties.getProperty("opennlp.geoentitylinker.gaz.lucenescore.min", String.valueOf(scoreCutoff));
String usehierarchy = properties.getProperty("opennlp.geoentitylinker.gaz.hierarchyfield", String.valueOf("0"));
if (cutoff != null && !cutoff.isEmpty()) {
- scoreCutoff = Double.valueOf(cutoff);
+ scoreCutoff = Double.parseDouble(cutoff);
}
if (usehierarchy != null && !usehierarchy.isEmpty()) {
- useHierarchyField = Boolean.valueOf(usehierarchy);
+ useHierarchyField = Boolean.parseBoolean(usehierarchy);
}
// opennlp.geoentitylinker.gaz.doublequote=false
//opennlp.geoentitylinker.gaz.hierarchyfield=false
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GeoEntityLinker.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GeoEntityLinker.java
index 43be5d5..36de509 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GeoEntityLinker.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/GeoEntityLinker.java
@@ -23,10 +23,10 @@ import opennlp.addons.geoentitylinker.scoring.GeoHashBinningScorer;
import opennlp.addons.geoentitylinker.scoring.FuzzyStringMatchScorer;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+
import opennlp.addons.geoentitylinker.scoring.PlacetypeScorer;
import opennlp.addons.geoentitylinker.scoring.ProvinceProximityScorer;
import opennlp.tools.entitylinker.BaseLink;
@@ -47,11 +47,11 @@ public class GeoEntityLinker implements EntityLinker<LinkedSpan> {
private AdminBoundaryContextGenerator countryContext;
private EntityLinkerProperties linkerProperties;
private GazetteerSearcher gazateerSearcher;
- private List<LinkedEntityScorer<AdminBoundaryContext>> scorers = new ArrayList<>();
+ private final List<LinkedEntityScorer<AdminBoundaryContext>> scorers = new ArrayList<>();
@Override
public List<LinkedSpan> find(String doctext, Span[] sentences, Span[][] tokensBySentence, Span[][] namesBySentence) {
- ArrayList<LinkedSpan> spans = new ArrayList<LinkedSpan>();
+ ArrayList<LinkedSpan> spans = new ArrayList<>();
if (linkerProperties == null) {
throw new IllegalArgumentException("EntityLinkerProperties cannot be null");
@@ -90,7 +90,7 @@ public class GeoEntityLinker implements EntityLinker<LinkedSpan> {
if (geoNamesEntries.isEmpty()) {
continue;
}
- /**
+ /*
* Normalize the returned scores for this name... this will assist the
* sort
*/
@@ -109,7 +109,7 @@ public class GeoEntityLinker implements EntityLinker<LinkedSpan> {
gazetteerEntry.getScoreMap().put("normlucene", normalize);
}
}
- LinkedSpan newspan = new LinkedSpan(geoNamesEntries, names[i], 0);
+ LinkedSpan<BaseLink> newspan = new LinkedSpan<>(geoNamesEntries, names[i], 0);
newspan.setSearchTerm(matches[i]);
newspan.setLinkedEntries(geoNamesEntries);
newspan.setSentenceid(s);
@@ -123,40 +123,37 @@ public class GeoEntityLinker implements EntityLinker<LinkedSpan> {
scorer.score(spans, doctext, sentences, linkerProperties, context);
}
}
- /**
+ /*
* sort the data with the best score on top based on the sum of the scores
* below from the score map for each baselink object
*/
for (LinkedSpan<BaseLink> s : spans) {
ArrayList<BaseLink> linkedData = s.getLinkedEntries();
- Collections.sort(linkedData, Collections.reverseOrder(new Comparator<BaseLink>() {
- @Override
- public int compare(BaseLink o1, BaseLink o2) {
- HashMap<String, Double> o1scoreMap = o1.getScoreMap();
- HashMap<String, Double> o2scoreMap = o2.getScoreMap();
- if (o1scoreMap.size() != o2scoreMap.size()) {
- return 0;
- }
- double sumo1 = 0d;
- double sumo2 = 0d;
- for (String object : o1scoreMap.keySet()) {
- if (object.equals("typescore")
- || object.equals("countrycontext")
- || object.equals("placenamedicecoef")
- || object.equals("provincecontext")
- || object.equals("geohashbin")
- || object.equals("normlucene")) {
- sumo1 += o1scoreMap.get(object);
- sumo2 += o2scoreMap.get(object);
- }
+ linkedData.sort(Collections.reverseOrder((o1, o2) -> {
+ Map<String, Double> o1scoreMap = o1.getScoreMap();
+ Map<String, Double> o2scoreMap = o2.getScoreMap();
+ if (o1scoreMap.size() != o2scoreMap.size()) {
+ return 0;
+ }
+ double sumo1 = 0d;
+ double sumo2 = 0d;
+ for (String object : o1scoreMap.keySet()) {
+ if (object.equals("typescore")
+ || object.equals("countrycontext")
+ || object.equals("placenamedicecoef")
+ || object.equals("provincecontext")
+ || object.equals("geohashbin")
+ || object.equals("normlucene")) {
+ sumo1 += o1scoreMap.get(object);
+ sumo2 += o2scoreMap.get(object);
}
-
- return Double.compare(sumo1,
- sumo2);
}
+
+ return Double.compare(sumo1,
+ sumo2);
}));
//prune the list to topN
- Iterator iterator = linkedData.iterator();
+ Iterator<BaseLink> iterator = linkedData.iterator();
int n = 0;
while (iterator.hasNext()) {
if (n >= topN) {
@@ -179,9 +176,9 @@ public class GeoEntityLinker implements EntityLinker<LinkedSpan> {
* @param maximum the max of the set to be transposed
* @return
*/
- private Double normalize(Double valueToNormalize, Double minimum, Double maximum) {
- Double d = (double) ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
- d = d == Double.NaN ? 0d : d;
+ private Double normalize(Double valueToNormalize, double minimum, double maximum) {
+ double d = ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
+ d = Double.isNaN(d) ? 0d : d;
return d;
}
@@ -203,9 +200,9 @@ public class GeoEntityLinker implements EntityLinker<LinkedSpan> {
countryContext = new AdminBoundaryContextGenerator(this.linkerProperties);
gazateerSearcher = new GazetteerSearcher(this.linkerProperties);
String rowsRetStr = this.linkerProperties.getProperty("opennlp.geoentitylinker.gaz.rowsreturned", "2");
- Integer rws = 2;
+ int rws;
try {
- rws = Integer.valueOf(rowsRetStr);
+ rws = Integer.parseInt(rowsRetStr);
} catch (NumberFormatException e) {
rws = 2;
}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GazetteerIndexer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GazetteerIndexer.java
index 88ca56f..66e7bfe 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GazetteerIndexer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GazetteerIndexer.java
@@ -18,6 +18,7 @@ package opennlp.addons.geoentitylinker.indexing;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
+import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
@@ -76,7 +77,7 @@ public class GazetteerIndexer {
}
- public static interface Separable {
+ public interface Separable {
String getSeparator();
}
@@ -110,37 +111,44 @@ public class GazetteerIndexer {
/**
*
* @param geonamesData the actual Geonames gazetteer data downloaded from
- * here: http://download.geonames.org/export/dump/ then click on this link
+ * <a href="https://download.geonames.org/export/dump/">here</a>, then click on this link
* 'allCountries.zip'
+ *
* @param geoNamesCountryInfo the countryinfo lookup table that can be
- * downloaded from here
- * http://download.geonames.org/export/dump/countryInfo.txt You'll need to
+ * downloaded from
+ * <a href="https://download.geonames.org/export/dump/countryInfo.txt">here</a>. You'll need to
* copy the page into a file or scrape it
+ *
* @param geonamesAdmin1CodesASCII The lookup data for the province names for
- * each place found here:
- * http://download.geonames.org/export/dump/admin1CodesASCII.txt highlight the
- * table view, and copy results into a text file. Make sure the tab delimitted
+ * each place found
+ * <a href="https://download.geonames.org/export/dump/admin1CodesASCII.txt">here</a>, highlight the
+ * table view, and copy results into a text file. Make sure the tab delimited
* format is maintained.
- * @param usgsDataFile the actual USGS gazetteer downloaded from here:
- * http://geonames.usgs.gov/domestic/download_data.htm click on the
+ *
+ * @param usgsDataFile the actual USGS gazetteer downloaded from
+ * <a href="https://geonames.usgs.gov/domestic/download_data.htm">here</a> click on the
* national_file####.zip link to get all the most recent features
*
- * @param usgsGovUnitsFile go to here:
- * http://geonames.usgs.gov/domestic/download_data.htm in the section titled
- * "Topical Gazetteers -- File Format" click on the drop down list and select
+ * @param usgsGovUnitsFile go to
+ * <a href="https://geonames.usgs.gov/domestic/download_data.htm">here</a> in the section titled
+ * "Topical Gazetteers -- File Format" click on the dropdown list and select
* "Government Units". The downloaded file is what you need for this param.
- * @param outputIndexDir where you want the final index. Must be a directory,
- * not an actual file.
- * @param outputCountryContextFile The output countrycontext file. THis is a
+ *
+ * @param outputIndexDir where you want the final index. Must be a directory, not an actual file.
+ *
+ * @param outputCountryContextFile The output countrycontext file. This is a
* very important file used inside the GeoEntityLinker to assist in toponym
* resolution.
+ *
* @param regionsFile this file contains a list of regions in the following
- * format: tab delimitted text with index 0 as the name of the region, index 1
+ * format: tab delimited text with index 0 as the name of the region, index 1
* as the longitude, and index 2 as the latitude
+ *
* @throws Exception
*/
- public void index(File geonamesData, File geoNamesCountryInfo, File geonamesAdmin1CodesASCII,
- File usgsDataFile, File usgsGovUnitsFile, File outputIndexDir, File outputCountryContextFile, File regionsFile) throws Exception {
+ public void index(File geonamesData, File geoNamesCountryInfo, File geonamesAdmin1CodesASCII, File usgsDataFile,
+ File usgsGovUnitsFile, File outputIndexDir, File outputCountryContextFile, File regionsFile)
+ throws Exception {
if (!outputIndexDir.isDirectory()) {
throw new IllegalArgumentException("outputIndexDir must be a directory.");
}
@@ -169,7 +177,7 @@ public class GazetteerIndexer {
String indexloc = outputIndexDir.getPath() + "/opennlp_geoentitylinker_gazetteer";
Directory index = new MMapDirectory(Paths.get(indexloc));
- Analyzer a = new StandardAnalyzer(new CharArraySet(new ArrayList(), true));
+ Analyzer a = new StandardAnalyzer(new CharArraySet(new ArrayList<>(), true));
Map<String, Analyzer> analyMap = new HashMap<>();
analyMap.put("countrycode", new KeywordAnalyzer());
@@ -178,29 +186,25 @@ public class GazetteerIndexer {
analyMap.put("countycode", new KeywordAnalyzer());
analyMap.put("gazsource", new KeywordAnalyzer());
- PerFieldAnalyzerWrapper aWrapper
- = new PerFieldAnalyzerWrapper(a, analyMap);
-
+ PerFieldAnalyzerWrapper aWrapper = new PerFieldAnalyzerWrapper(a, analyMap);
IndexWriterConfig config = new IndexWriterConfig(aWrapper);
+ try (IndexWriter w = new IndexWriter(index, config)) {
+ //write the column headers for the countryContextFile
+ try (FileWriter countryContextFileWriter = new FileWriter(outputCountryContextFile, false)) {
+ String colNamesForCountryContextFile = "countrycode\tprovcode\tcountycode\tcountryname\tprovincename\tcountyname\tcountryregex\tprovregex\tcountyregex\n";
+ countryContextFileWriter.write(colNamesForCountryContextFile);
+ countryContextFileWriter.flush();
+ }
+
+ USGSProcessor.process(usgsGovUnitsFile, usgsDataFile, outputCountryContextFile, w);
+ GeonamesProcessor.process(geoNamesCountryInfo, geonamesAdmin1CodesASCII, geonamesData, outputCountryContextFile, w);
+
+ RegionProcessor.process(regionsFile, outputCountryContextFile, w);
+ w.commit();
+ }
- IndexWriter w = new IndexWriter(index, config);
-
- //write the column headers for the countryContextFile
- FileWriter countryContextFileWriter = new FileWriter(outputCountryContextFile, false);
- String colNamesForCountryContextFile = "countrycode\tprovcode\tcountycode\tcountryname\tprovincename\tcountyname\tcountryregex\tprovregex\tcountyregex\n";
- countryContextFileWriter.write(colNamesForCountryContextFile);
- countryContextFileWriter.flush();
- countryContextFileWriter.close();
-
-
- USGSProcessor.process(usgsGovUnitsFile, usgsDataFile, outputCountryContextFile, w);
-
- GeonamesProcessor.process(geoNamesCountryInfo, geonamesAdmin1CodesASCII, geonamesData, outputCountryContextFile, w);
-
- RegionProcessor.process(regionsFile, outputCountryContextFile, w);
- w.commit();
- w.close();
- System.out.println("\nIndexing complete. Be sure to add '" + indexloc + "' and context file '" + outputCountryContextFile.getPath() + "' to entitylinker.properties file");
+ System.out.println("\nIndexing complete. Be sure to add '" + indexloc + "' and context file '" +
+ outputCountryContextFile.getPath() + "' to entitylinker.properties file");
}
}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesFileDownloader.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesFileDownloader.java
index 4afd96d..7d27e3f 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesFileDownloader.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesFileDownloader.java
@@ -16,13 +16,11 @@
package opennlp.addons.geoentitylinker.indexing;
import java.io.BufferedOutputStream;
-import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
-import java.net.URLConnection;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
@@ -48,7 +46,7 @@ public class GeonamesFileDownloader {
}
- public static final void writeFile(InputStream in, OutputStream out)
+ public static void writeFile(InputStream in, OutputStream out)
throws IOException {
byte[] buffer = new byte[1024];
int len;
@@ -61,17 +59,16 @@ public class GeonamesFileDownloader {
out.close();
}
- public static void unzipMyZip(String zipFileName,
- String directoryToExtractTo) {
- Enumeration entriesEnum;
+ public static void unzipMyZip(String zipFileName, String directoryToExtractTo) {
+ Enumeration<? extends ZipEntry> entriesEnum;
ZipFile zip;
try {
zip = new ZipFile(zipFileName);
entriesEnum = zip.entries();
while (entriesEnum.hasMoreElements()) {
- ZipEntry entry = (ZipEntry) entriesEnum.nextElement();
+ ZipEntry entry = entriesEnum.nextElement();
InputStream is = zip.getInputStream(entry); // get the input stream
- OutputStream os = new java.io.FileOutputStream(new File(zipFileName.replace("\\.zip", ".txt")));
+ OutputStream os = new FileOutputStream(zipFileName.replace("\\.zip", ".txt"));
byte[] buf = new byte[4096];
int r;
while ((r = is.read(buf)) != -1) {
@@ -83,40 +80,24 @@ public class GeonamesFileDownloader {
} catch (IOException ioe) {
System.err.println("Some Exception Occurred:");
ioe.printStackTrace();
- return;
}
}
- public static String fileUrl(String fAddress, String localFileName, String destinationDir) {
- OutputStream outStream = null;
- URLConnection uCon = null;
- String filename = destinationDir + "\\" + localFileName;
- InputStream is = null;
- try {
- URL Url;
- byte[] buf;
- int ByteRead, ByteWritten = 0;
- Url = new URL(fAddress);
- outStream = new BufferedOutputStream(new FileOutputStream(destinationDir + "\\" + localFileName));
-
- uCon = Url.openConnection();
- is = uCon.getInputStream();
- buf = new byte[size];
- while ((ByteRead = is.read(buf)) != -1) {
- outStream.write(buf, 0, ByteRead);
- ByteWritten += ByteRead;
+ public static String fileUrl(String fAddress, String localFileName, String destDir) {
+ String filename = destDir + "\\" + localFileName;
+ try (InputStream is = new URL(fAddress).openConnection().getInputStream();
+ OutputStream outStream = new BufferedOutputStream(new FileOutputStream(destDir + "\\" + localFileName))) {
+
+ byte[] buf = new byte[size];
+ int byteRead, byteWritten = 0;
+ while ((byteRead = is.read(buf)) != -1) {
+ outStream.write(buf, 0, byteRead);
+ byteWritten += byteRead;
}
System.out.println("Downloaded Successfully.");
- System.out.println("File name:\"" + localFileName + "\"\nNo ofbytes :" + ByteWritten);
+ System.out.println("File name:\"" + localFileName + "\"\nNo ofbytes :" + byteWritten);
} catch (Exception e) {
e.printStackTrace();
- } finally {
- try {
- is.close();
- outStream.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
}
return filename;
}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesProcessor.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesProcessor.java
index cd032e8..f560d33 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesProcessor.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/GeonamesProcessor.java
@@ -62,7 +62,7 @@ public class GeonamesProcessor {
reader = new BufferedReader(new FileReader(adm1CodesLookupFile));
int i = 0;
- String line = "";
+ String line;
while ((line = reader.readLine()) != null) {
// String line = reader.readLine();
@@ -116,7 +116,7 @@ public class GeonamesProcessor {
reader = new BufferedReader(new FileReader(countryContextFile));
int i = 0;
- String line = "";
+ String line;
boolean start = false;
while ((line = reader.readLine()) != null) {
if (!line.toLowerCase().startsWith("#iso\t") && !start) {
@@ -157,7 +157,7 @@ public class GeonamesProcessor {
String province = adm.getProvinceName();
String country = adm.getCountryName();
- String line = adm.getCountryCode() + "\t" + adm.getProvCode() + "\t" + "" + "\t" + country + "\t" + province + "\t" + "" + "\t" + "(" + country + ")" + "\t"
+ String line = adm.getCountryCode() + "\t" + adm.getProvCode() + "\t" + "\t" + country + "\t" + province + "\t" + "\t" + "(" + country + ")" + "\t"
+ adm.getProvinceName() + "\t" + adm.getCountyName() + "\n";
bw.write(line);
// System.out.println(line);
@@ -211,7 +211,7 @@ public class GeonamesProcessor {
List<String> fields = Arrays.asList(fieldStrings);
int counter = 0;
System.out.println("reading gazetteer data from file...........");
- String line = "";
+ String line;
while ((line = reader.readLine()) != null) {
String[] values = line.split(type.getSeparator());
@@ -232,8 +232,8 @@ public class GeonamesProcessor {
String dsg = values[7].toLowerCase();
String id = values[0];
- String concatIndexEntry = "";
- String countryname = "";
+ String concatIndexEntry;
+ String countryname;
if (adm != null) {
concatIndexEntry = adm.getCountryName() + ", " + adm.getProvinceName() + ", " + placeName;
countryname = adm.getCountryName();
@@ -259,7 +259,7 @@ public class GeonamesProcessor {
System.out.println("placename: " + placeName + " RESET TO: " + countryname);
placeName = countryname;
}
- /**
+ /*
* add standard fields to the index
*/
doc.add(new TextField("hierarchy", concatIndexEntry, Field.Store.YES));
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/RegionProcessor.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/RegionProcessor.java
index de91e0c..f1d5d9f 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/RegionProcessor.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/RegionProcessor.java
@@ -57,7 +57,7 @@ public class RegionProcessor {
List<String> fields = new ArrayList<>();
int counter = 0;
System.out.println("reading gazetteer data from Regions file...........");
- String line = "";
+ String line;
while ((line = reader.readLine()) != null) {
String[] values = line.split("\t");
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/USGSProcessor.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/USGSProcessor.java
index 59c94b7..588befb 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/USGSProcessor.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/indexing/USGSProcessor.java
@@ -55,84 +55,87 @@ public class USGSProcessor {
public static void readFile(File gazateerInputData, IndexWriter w, GazetteerIndexer.GazType type, Map<String, AdminBoundary> lookupMap) throws Exception {
Map<String, StateCentroid> states = new HashMap<>();
- BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData));
- List<String> fields = new ArrayList<>();
- int counter = 0;
- System.out.println("reading gazetteer data from USGS file...........");
- String line = "";
- while ((line = reader.readLine()) != null) {
-
- String[] values = line.split(type.getSeparator());
- if (counter == 0) {
- for (String columnName : values) {
- fields.add(columnName.replace("»¿", "").trim());
- }
-
- } else {
- Document doc = new Document();
- for (int i = 0; i < fields.size() - 1; i++) {
- doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
- }
- String placeName = values[1];
- String lat = values[9];
- String lon = values[10];
- String dsg = values[2];
- String id = values[0];
-
- String ccode = values[6];
- String admincode = values[3];
- AdminBoundary get = lookupMap.get(admincode + "." + ccode);
- String countyname = "";
- if (get == null) {
- System.out.println("null...continuing to index" + " ccode: " + ccode + " , admincode: " + admincode + " , placename: " + placeName);
- continue;
+ try (BufferedReader reader = new BufferedReader(new FileReader(gazateerInputData))) {
- }
- String countyCode = get.getCountyCode();
+ List<String> fields = new ArrayList<>();
+ int counter = 0;
+ System.out.println("reading gazetteer data from USGS file...........");
+ String line;
+ while ((line = reader.readLine()) != null) {
- if (!get.getCountyName().equals("NO_DATA_FOUND_VALUE")) {
- countyname = get.getCountyName();
- }
- if (!get.getCountyCode().equals("NO_DATA_FOUND_VALUE")) {
- countyCode = get.getCountyCode();
- }
- String hierarchy = get.getCountryName() + ", " + get.getProvinceName() + ", " + countyname + ", " + placeName;
+ String[] values = line.split(type.getSeparator());
+ if (counter == 0) {
+ for (String columnName : values) {
+ fields.add(columnName.replace("»¿", "").trim());
+ }
- if (states.containsKey(get.getProvinceName())) {
- StateCentroid entry = states.get(get.getProvinceName());
- entry.count++;
- entry.latSum += Double.valueOf(lat);
- entry.longSum += Double.valueOf(lon);
} else {
- StateCentroid centroid = new StateCentroid();
- centroid.statecode = get.getProvCode();
- centroid.count = 1;
- centroid.latSum = Double.valueOf(lat);
- centroid.longSum = Double.valueOf(lon);
- states.put(get.getProvinceName(), centroid);
+ Document doc = new Document();
+ for (int i = 0; i < fields.size() - 1; i++) {
+ doc.add(new TextField(fields.get(i), values[i].trim(), Field.Store.YES));
+ }
+ String placeName = values[1];
+ String lat = values[9];
+ String lon = values[10];
+ String dsg = values[2];
+ String id = values[0];
+
+ String ccode = values[6];
+ String admincode = values[3];
+ AdminBoundary get = lookupMap.get(admincode + "." + ccode);
+ String countyname = "";
+ if (get == null) {
+ System.out.println("null...continuing to index" + " ccode: " + ccode + " , admincode: " + admincode + " , placename: " + placeName);
+ continue;
+
+ }
+ String countyCode = get.getCountyCode();
+
+ if (!get.getCountyName().equals("NO_DATA_FOUND_VALUE")) {
+ countyname = get.getCountyName();
+ }
+ if (!get.getCountyCode().equals("NO_DATA_FOUND_VALUE")) {
+ countyCode = get.getCountyCode();
+ }
+ String hierarchy = get.getCountryName() + ", " + get.getProvinceName() + ", " + countyname + ", " + placeName;
+
+ if (states.containsKey(get.getProvinceName())) {
+ StateCentroid entry = states.get(get.getProvinceName());
+ entry.count++;
+ entry.latSum += Double.parseDouble(lat);
+ entry.longSum += Double.parseDouble(lon);
+ } else {
+ StateCentroid centroid = new StateCentroid();
+ centroid.statecode = get.getProvCode();
+ centroid.count = 1;
+ centroid.latSum = Double.parseDouble(lat);
+ centroid.longSum = Double.parseDouble(lon);
+ states.put(get.getProvinceName(), centroid);
+ }
+
+ doc.add(new TextField("hierarchy", hierarchy, Field.Store.YES));
+ doc.add(new TextField("placename", placeName, Field.Store.YES));
+ doc.add(new TextField("latitude", lat, Field.Store.YES));
+ doc.add(new TextField("longitude", lon, Field.Store.YES));
+ doc.add(new StringField("loctype", dsg, Field.Store.YES));
+ doc.add(new StringField("admincode", (get.getCountryCode() + "." + get.getProvCode()).toLowerCase(), Field.Store.YES));
+ doc.add(new StringField("countrycode", get.getCountryCode().toLowerCase(), Field.Store.YES));
+ doc.add(new StringField("countycode", (get.getCountryCode() + "." + get.getProvCode() + "." + countyCode).toLowerCase(), Field.Store.YES));
+
+ doc.add(new StringField("locid", id, Field.Store.YES));
+ doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
+ w.addDocument(doc);
+ }
+ counter++;
+ if (counter % 100000 == 0) {
+ w.commit();
+ System.out.println(counter + " .........USGS entries committed to index..............");
}
- doc.add(new TextField("hierarchy", hierarchy, Field.Store.YES));
- doc.add(new TextField("placename", placeName, Field.Store.YES));
- doc.add(new TextField("latitude", lat, Field.Store.YES));
- doc.add(new TextField("longitude", lon, Field.Store.YES));
- doc.add(new StringField("loctype", dsg, Field.Store.YES));
- doc.add(new StringField("admincode", (get.getCountryCode() + "." + get.getProvCode()).toLowerCase(), Field.Store.YES));
- doc.add(new StringField("countrycode", get.getCountryCode().toLowerCase(), Field.Store.YES));
- doc.add(new StringField("countycode", (get.getCountryCode() + "." + get.getProvCode() + "." + countyCode).toLowerCase(), Field.Store.YES));
-
- doc.add(new StringField("locid", id, Field.Store.YES));
- doc.add(new StringField("gazsource", "usgs", Field.Store.YES));
- w.addDocument(doc);
}
- counter++;
- if (counter % 100000 == 0) {
- w.commit();
- System.out.println(counter + " .........USGS entries committed to index..............");
- }
-
}
+
for (String state : states.keySet()) {
StateCentroid get = states.get(state);
Document doc = new Document();
@@ -184,13 +187,10 @@ public class USGSProcessor {
private static Map<String, AdminBoundary> getProvData(File govUnitsFile, GazetteerIndexer.GazType type) {
System.out.println("Attempting to read USGS province (State) data from: " + govUnitsFile.getPath());
Map<String, AdminBoundary> outmap = new HashMap<>();
- BufferedReader reader;
-
- try {
- reader = new BufferedReader(new FileReader(govUnitsFile));
+ try (BufferedReader reader = new BufferedReader(new FileReader(govUnitsFile))) {
int i = 0;
- String line = "";
+ String line;
String[] fields = null;
while ((line = reader.readLine()) != null) {
@@ -208,12 +208,12 @@ public class USGSProcessor {
String stateName = values[6];
String countryCode = values[7];
String countryName = values[8];
- AdminBoundary adminBoundary = new AdminBoundary(countryCode, countryName, stateCode, stateName, countyCode, countyName, null, null, null);
+ AdminBoundary adminBoundary = new AdminBoundary(countryCode, countryName, stateCode, stateName, countyCode,
+ countyName, null, null, null);
outmap.put(stateCode + "." + countyCode, adminBoundary);
// System.out.println(adminBoundary);
}
- reader.close();
} catch (IOException ex) {
ex.printStackTrace();
}
@@ -224,10 +224,8 @@ public class USGSProcessor {
}
public static void writeCountryContextFile(File outfile, Map<String, AdminBoundary> adms) {
- // FileWriter writer = null;
- try (FileWriter writer = new FileWriter(outfile, true)) {
- BufferedWriter bw = new BufferedWriter(writer);
+ try (BufferedWriter bw = new BufferedWriter(new FileWriter(outfile, true))) {
for (String admkey : adms.keySet()) {
AdminBoundary adm = adms.get(admkey);
if (adm == null) {
@@ -235,15 +233,14 @@ public class USGSProcessor {
}
String province = adm.getProvinceName();
String country = adm.getCountryName();
- /**
+ /*
* this is the standard format of the country context file... Geonames
* data will have an empty string for the county
*/
String line = adm.getCountryCode() + "\t" + adm.getProvCode() + "\t" + adm.getCountyCode() + "\t" + country + "\t" + province + "\t" + adm.getCountyName() + "\t"
+ "(U\\.S\\.[ $]|U\\.S\\.A\\.[ $]|United States|the US[ $]|a us[ $])" + "\t" + adm.getProvinceName() + "\t" + adm.getCountyName() + "\n";
- bw.write(line);i
+ bw.write(line);
/// System.out.println(line);
-
}
} catch (IOException ex) {
Logger.getLogger(GeonamesProcessor.class.getName()).log(Level.SEVERE, null, ex);
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/CountryProximityScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/CountryProximityScorer.java
index aea8f9b..df69fb7 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/CountryProximityScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/CountryProximityScorer.java
@@ -35,8 +35,6 @@ import opennlp.tools.util.Span;
* country mentions. For instance, if the toponym Berlin is mentioned near an
* indicator of Germany, it is more likely to be Berlin Germany than Berlin
* Connecticut (if Connecticut is mentioned further down in the article).
- *
- *
*/
public class CountryProximityScorer implements LinkedEntityScorer<AdminBoundaryContext> {
@@ -54,7 +52,7 @@ public class CountryProximityScorer implements LinkedEntityScorer<AdminBoundaryC
/**
* Assigns a score to each BaseLink in each linkedSpan's set of N best
- * matches. Currently the scoring indicates the probability that the toponym
+ * matches. Currently, the scoring indicates the probability that the toponym
* is correct based on the country context in the document
*
* @param linkedData the linked spans, holds the Namefinder results, and the
@@ -110,20 +108,20 @@ public class CountryProximityScorer implements LinkedEntityScorer<AdminBoundaryC
private LinkedSpan<BaseLink> simpleProximityAnalysis(Span[] sentences, Map<String, Set<Integer>> countryHits, LinkedSpan<BaseLink> span, Integer maxAllowedDistance) {
Double score = 0.0;
/*
- * get the index of the actual span, begining of sentence //should generate
+ * get the index of the actual span, beginning of sentence //should generate
* tokens from sentence and create a char offset... //could have large
* sentences due to poor sentence detection or wonky doc text
*/
int sentenceIdx = span.getSentenceid();
int sentIndexInDoc = sentences[sentenceIdx].getStart();
- /**
+ /*
* create a map of all the span's proximal country mentions in the document
* Map< countrycode, set of <distances from this NamedEntity>>
*/
- Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<String, Set<Integer>>();
+ Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<>();
//map = Map<countrycode, Set <of distances this span is from all the mentions of the code>>
for (String cCode : countryHits.keySet()) {
-//iterate over all the regex start values and calculate an offset
+ // iterate over all the regex start values and calculate an offset
for (Integer cHit : countryHits.get(cCode)) {
Integer absDist = Math.abs(sentIndexInDoc - cHit);
//only include near mentions based on a heuristic
@@ -132,17 +130,15 @@ public class CountryProximityScorer implements LinkedEntityScorer<AdminBoundaryC
if (distancesFromCodeMap.containsKey(cCode)) {
distancesFromCodeMap.get(cCode).add(absDist);
} else {
- HashSet<Integer> newset = new HashSet<Integer>();
+ HashSet<Integer> newset = new HashSet<>();
newset.add(absDist);
distancesFromCodeMap.put(cCode, newset);
}
}
-
- //}
}
//we now know how far this named entity is from every country mention in the document
- /**
+ /*
* the gaz matches that have a country code that have mentions in the doc
* that are closest to the Named Entity should return the best score.
* Analyzemap generates a likelihood score that the toponym from the gaz is
@@ -160,7 +156,7 @@ public class CountryProximityScorer implements LinkedEntityScorer<AdminBoundaryC
if (nameCodesMap.containsKey(link.getItemName().toLowerCase()) || regexMatch(link.getItemName(), link.getItemParentID())) {
//if so, is it the correct country code for that name?
if (nameCodesMap.get(link.getItemName().toLowerCase()).contains(link.getItemParentID())) {
- //boost the score becuase it is likely that this is the location in the text, so add 50% to the score or set to 1
+ //boost the score because it is likely that this is the location in the text, so add 50% to the score or set to 1
score = (score + .75) > 1.0 ? 1d : (score + .75);
if (link.getItemParentID().equals(dominantCode)) {
@@ -188,24 +184,24 @@ public class CountryProximityScorer implements LinkedEntityScorer<AdminBoundaryC
*/
private Map<String, Double> analyzeMap(Map<String, Set<Integer>> distanceMap, Span[] sentences, LinkedSpan<BaseLink> span) {
- Map<String, Double> scoreMap = new HashMap<String, Double>();
+ Map<String, Double> scoreMap = new HashMap<>();
if (distanceMap.isEmpty()) {
return scoreMap;
}
- TreeSet<Integer> all = new TreeSet<Integer>();
+ TreeSet<Integer> all = new TreeSet<>();
for (String key : distanceMap.keySet()) {
all.addAll(distanceMap.get(key));
}
- //get min max for normalization, this could be more efficient
- Integer min = all.first();
- Integer max = all.last();
+ // get min max for normalization, this could be more efficient
+ int min = all.first();
+ int max = all.last();
if (min == max) {
min = 0;
}
- for (String key : distanceMap.keySet()) {
- TreeSet<Double> normalizedDistances = new TreeSet<Double>();
+ for (String key : distanceMap.keySet()) {
+ TreeSet<Double> normalizedDistances = new TreeSet<>();
for (Integer i : distanceMap.get(key)) {
Double norm = normalize(i, min, max);
//reverse the normed distance so low numbers (closer) are better
@@ -214,7 +210,7 @@ public class CountryProximityScorer implements LinkedEntityScorer<AdminBoundaryC
normalizedDistances.add(reverse);
}
- List<Double> doubles = new ArrayList<Double>(normalizedDistances);
+ List<Double> doubles = new ArrayList<>(normalizedDistances);
scoreMap.put(key, slidingDistanceAverage(doubles));
}
return scoreMap;
@@ -241,7 +237,7 @@ public class CountryProximityScorer implements LinkedEntityScorer<AdminBoundaryC
* @return
*/
private Double slidingDistanceAverage(List<Double> normDis) {
- List<Double> windowOfAverages = new ArrayList<Double>();
+ List<Double> windowOfAverages = new ArrayList<>();
if (normDis.size() < 3) {
windowOfAverages.addAll(normDis);
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/FuzzyStringMatchScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/FuzzyStringMatchScorer.java
index e9634d9..64bd788 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/FuzzyStringMatchScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/FuzzyStringMatchScorer.java
@@ -34,6 +34,7 @@ public class FuzzyStringMatchScorer implements LinkedEntityScorer<AdminBoundaryC
@Override
public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
+
for (LinkedSpan<BaseLink> linkedSpan : linkedSpans) {
for (BaseLink link : linkedSpan.getLinkedEntries()) {
if (link instanceof GazetteerEntry) {
@@ -87,7 +88,7 @@ public class FuzzyStringMatchScorer implements LinkedEntityScorer<AdminBoundaryC
s2Grams.add(token);
}
- Set<String> overlap = new HashSet<String>(s1Grams);
+ Set<String> overlap = new HashSet<>(s1Grams);
overlap.retainAll(s2Grams);
double totcombigrams = overlap.size();
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/GeoHashBinningScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/GeoHashBinningScorer.java
index d3494e0..adf48fd 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/GeoHashBinningScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/GeoHashBinningScorer.java
@@ -38,10 +38,9 @@ public class GeoHashBinningScorer implements LinkedEntityScorer<AdminBoundaryCon
@Override
public void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, AdminBoundaryContext additionalContext) {
- //Map<Double, Double> latLongs = new HashMap<Double, Double>();
List<GazetteerEntry> allGazEntries = new ArrayList<>();
- /**
+ /*
* collect all the gaz entry references
*/
for (LinkedSpan<BaseLink> ls : linkedSpans) {
@@ -51,7 +50,7 @@ public class GeoHashBinningScorer implements LinkedEntityScorer<AdminBoundaryCon
}
}
}
- /**
+ /*
* use the point clustering to score each hit
*/
Map<String, List<GazetteerEntry>> cluster = CLUSTERER.cluster(allGazEntries, PRECISION);
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/LinkedEntityScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/LinkedEntityScorer.java
index 5fb9c5d..5741af0 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/LinkedEntityScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/LinkedEntityScorer.java
@@ -33,7 +33,7 @@ public interface LinkedEntityScorer<T> {
* @param linkedSpans the spans that have been linked to some external source and have all the data they need to be scored
* @param docText the full text of the document.
* @param sentenceSpans the sentence spans the correspond to the document text
- * @param properties the entitylinker properties config file
+ * @param properties the entitylinker properties config file
* @param additionalContext any additional data required to perform the scoring operation
*/
void score(List<LinkedSpan> linkedSpans, String docText, Span[] sentenceSpans, EntityLinkerProperties properties, T additionalContext);
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ModelBasedScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ModelBasedScorer.java
index 01b3269..8d473d0 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ModelBasedScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ModelBasedScorer.java
@@ -16,8 +16,7 @@
package opennlp.addons.geoentitylinker.scoring;
import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -28,7 +27,8 @@ import opennlp.tools.entitylinker.EntityLinkerProperties;
import opennlp.tools.entitylinker.BaseLink;
import opennlp.tools.entitylinker.LinkedSpan;
import opennlp.tools.util.Span;
-import org.apache.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
@@ -36,7 +36,8 @@ import org.apache.log4j.Logger;
*/
public class ModelBasedScorer implements LinkedEntityScorer<AdminBoundaryContext> {
- private static final Logger LOGGER = Logger.getLogger(ModelBasedScorer.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
DocumentCategorizerME documentCategorizerME;
DoccatModel doccatModel;
public static final int RADIUS = 200;
@@ -66,12 +67,8 @@ public class ModelBasedScorer implements LinkedEntityScorer<AdminBoundaryContext
}
}
- } catch (FileNotFoundException ex) {
- LOGGER.error(ex);
- } catch (IOException ex) {
- LOGGER.error(ex);
} catch (Exception ex) {
- LOGGER.error(ex);
+ LOG.error(ex.getLocalizedMessage(), ex);
}
}
@@ -89,17 +86,17 @@ public class ModelBasedScorer implements LinkedEntityScorer<AdminBoundaryContext
public Map<Integer, String> generateProximalFeatures(List<LinkedSpan> linkedSpans, Span[] sentenceSpans, String docText, int radius) {
Map<Integer, String> featureBags = new HashMap<>();
Map<Integer, Integer> nameMentionMap = new HashMap<>();
- /**
+ /*
* iterator over the map that contains a mapping of every country code to
* all of its mentions in the document
*/
for (int i = 0; i < linkedSpans.size(); i++) {
- LinkedSpan span = linkedSpans.get(i);
+ LinkedSpan<?> span = linkedSpans.get(i);
if (span.getLinkedEntries().isEmpty()) {
//don't care about spans that did not get linked to anything at all; nothing to work with
continue;
}
- /**
+ /*
* get the sentence the name span was found in, the beginning of the
* sentence will suffice as a centroid for feature generation around the
* named entity
@@ -107,7 +104,7 @@ public class ModelBasedScorer implements LinkedEntityScorer<AdminBoundaryContext
Integer mentionIdx = sentenceSpans[span.getSentenceid()].getStart();
nameMentionMap.put(i, mentionIdx);
}
- /**
+ /*
* now associate each span to a string that will be used for categorization
* against the model.
*/
@@ -127,7 +124,7 @@ public class ModelBasedScorer implements LinkedEntityScorer<AdminBoundaryContext
if (right <= left) {
chunk = "";
} else {
- /**
+ /*
* don't want to chop any words in half, so take fron the first space to
* the last space in the chunk string
*/
@@ -136,7 +133,7 @@ public class ModelBasedScorer implements LinkedEntityScorer<AdminBoundaryContext
left = chunk.indexOf(" ");
}
right = chunk.lastIndexOf(" ");
- /**
+ /*
* now get the substring again with only whole words
*/
if (left < right) {
@@ -149,7 +146,7 @@ public class ModelBasedScorer implements LinkedEntityScorer<AdminBoundaryContext
private Map<String, Double> getScore(String text) throws Exception {
Map<String, Double> scoreMap = new HashMap<>();
- double[] categorize = documentCategorizerME.categorize(text);
+ double[] categorize = documentCategorizerME.categorize(List.of(text).toArray(new String[0]));
int catSize = documentCategorizerME.getNumberOfCategories();
for (int i = 0; i < catSize; i++) {
String category = documentCategorizerME.getCategory(i);
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PlacetypeScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PlacetypeScorer.java
index dede558..4c8e151 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PlacetypeScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PlacetypeScorer.java
@@ -67,10 +67,10 @@ public class PlacetypeScorer implements LinkedEntityScorer<AdminBoundaryContext>
boosetedTypes.put(type.toLowerCase(), .5d);
} else if (type.startsWith("ADM")) {
boosetedTypes.put(type.toLowerCase(), .75d);
- }else if (type.toLowerCase().equals("civil")){
+ }else if (type.equalsIgnoreCase("civil")){
boosetedTypes.put(type.toLowerCase(), .9d);
}
- if(type.toLowerCase().equals("populated_place")){
+ if(type.equalsIgnoreCase("populated_place")){
boosetedTypes.put("Populated Place", .75d);
}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PointClustering.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PointClustering.java
index 908df1e..8cd1440 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PointClustering.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/PointClustering.java
@@ -41,8 +41,7 @@ public class PointClustering {
*/
public Map<String, List<GazetteerEntry>> cluster(List<GazetteerEntry> entries, int precision) {
Map<String, List<GazetteerEntry>> map = new HashMap<>();
- for (int i = 0; i < entries.size(); i++) {
- GazetteerEntry entry = entries.get(i);
+ for (GazetteerEntry entry : entries) {
Double latw = entry.getLatitude();
Double lonw = entry.getLongitude();
@@ -64,12 +63,12 @@ public class PointClustering {
for (String key : clusters.keySet()) {
int size = clusters.get(key).size();
if (size > max) {
- max = Double.valueOf(size);
+ max = (double) size;
}
}
for (String key : clusters.keySet()) {
int size = clusters.get(key).size();
- Double score = normalize(Double.valueOf(size), min, max);
+ Double score = normalize((double) size, min, max);
for (GazetteerEntry entry : clusters.get(key)) {
entry.getScoreMap().put("geohashbin", score);
}
@@ -116,8 +115,7 @@ public class PointClustering {
}
- private Double normalize(Double valueToNormalize, Double minimum, Double maximum) {
- Double d = (double) ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
- return d;
+ private Double normalize(Double valueToNormalize, double minimum, double maximum) {
+ return ((1 - 0) * (valueToNormalize - minimum)) / (maximum - minimum) + 0;
}
}
diff --git a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ProvinceProximityScorer.java b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ProvinceProximityScorer.java
index d9e7d19..a21af45 100644
--- a/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ProvinceProximityScorer.java
+++ b/geoentitylinker-addon/src/main/java/opennlp/addons/geoentitylinker/scoring/ProvinceProximityScorer.java
@@ -129,7 +129,7 @@ public class ProvinceProximityScorer implements LinkedEntityScorer<AdminBoundary
* create a map of all the span's proximal country mentions in the document
* Map< countrycode, set of <distances from this NamedEntity>>
*/
- Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<String, Set<Integer>>();
+ Map<String, Set<Integer>> distancesFromCodeMap = new HashMap<>();
//map = Map<countrycode, Set <of distances this span is from all the mentions of the code>>
for (String cCode : countryHits.keySet()) {
//iterate over all the regex start values and calculate an offset
@@ -141,7 +141,7 @@ public class ProvinceProximityScorer implements LinkedEntityScorer<AdminBoundary
if (distancesFromCodeMap.containsKey(cCode)) {
distancesFromCodeMap.get(cCode).add(absDist);
} else {
- HashSet<Integer> newset = new HashSet<Integer>();
+ HashSet<Integer> newset = new HashSet<>();
newset.add(absDist);
distancesFromCodeMap.put(cCode, newset);
}
@@ -211,24 +211,24 @@ public class ProvinceProximityScorer implements LinkedEntityScorer<AdminBoundary
*/
private Map<String, Double> analyzeMap(Map<String, Set<Integer>> distanceMap, Span[] sentences, LinkedSpan<BaseLink> span) {
- Map<String, Double> scoreMap = new HashMap<String, Double>();
+ Map<String, Double> scoreMap = new HashMap<>();
if (distanceMap.isEmpty()) {
return scoreMap;
}
- TreeSet<Integer> all = new TreeSet<Integer>();
+ TreeSet<Integer> all = new TreeSet<>();
for (String key : distanceMap.keySet()) {
all.addAll(distanceMap.get(key));
}
//get min max for normalization, this could be more efficient
- Integer min = all.first();
- Integer max = all.last();
+ int min = all.first();
+ int max = all.last();
if (min == max) {
min = 0;
}
for (String key : distanceMap.keySet()) {
- TreeSet<Double> normalizedDistances = new TreeSet<Double>();
+ TreeSet<Double> normalizedDistances = new TreeSet<>();
for (Integer i : distanceMap.get(key)) {
Double norm = normalize(i, min, max);
//reverse the normed distance so low numbers (closer) are better
@@ -237,7 +237,7 @@ public class ProvinceProximityScorer implements LinkedEntityScorer<AdminBoundary
normalizedDistances.add(reverse);
}
- List<Double> doubles = new ArrayList<Double>(normalizedDistances);
+ List<Double> doubles = new ArrayList<>(normalizedDistances);
scoreMap.put(key, slidingDistanceAverage(doubles));
}
return scoreMap;
@@ -254,7 +254,7 @@ public class ProvinceProximityScorer implements LinkedEntityScorer<AdminBoundary
* @return
*/
private Double slidingDistanceAverage(List<Double> normDis) {
- List<Double> windowOfAverages = new ArrayList<Double>();
+ List<Double> windowOfAverages = new ArrayList<>();
if (normDis.size() < 3) {
windowOfAverages.addAll(normDis);
diff --git a/geoentitylinker-addon/src/test/java/apache/opennlp/addons/AppTest.java b/geoentitylinker-addon/src/test/java/apache/opennlp/addons/AppTest.java
deleted file mode 100644
index 60ea0f2..0000000
--- a/geoentitylinker-addon/src/test/java/apache/opennlp/addons/AppTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package apache.opennlp.addons;
-
-import junit.framework.Test;
-import junit.framework.TestCase;
-import junit.framework.TestSuite;
-
-/**
- * Unit test for simple App.
- */
-public class AppTest
- extends TestCase
-{
- /**
- * Create the test case
- *
- * @param testName name of the test case
- */
- public AppTest( String testName )
- {
- super( testName );
- }
-
- /**
- * @return the suite of tests being tested
- */
- public static Test suite()
- {
- return new TestSuite( AppTest.class );
- }
-
- /**
- * Rigourous Test :-)
- */
- public void testApp()
- {
- assertTrue( true );
- }
-}
diff --git a/japanese-addon/build.xml b/japanese-addon/build.xml
index 925f888..1bb60b5 100644
--- a/japanese-addon/build.xml
+++ b/japanese-addon/build.xml
@@ -23,7 +23,7 @@
<property name="cls.dir" value="classes"/>
<property name="lib.dir" value="lib"/>
<property name="test.result.dir" value="test-result"/>
- <property name="product.jar" value="opennlp-japanese-addon-1.0-SNAPSHOT.jar"/>
+ <property name="product.jar" value="opennlp-japanese-addon-2.2.1-SNAPSHOT.jar"/>
<target name="compile" description="compile source and test code">
<mkdir dir="${cls.dir}"/>
diff --git a/japanese-addon/pom.xml b/japanese-addon/pom.xml
index 36ac3d1..d10c307 100644
--- a/japanese-addon/pom.xml
+++ b/japanese-addon/pom.xml
@@ -1,31 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.opennlp</groupId>
+ <parent>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ </parent>
+
<artifactId>japanese-addon</artifactId>
<packaging>jar</packaging>
- <version>1.0-SNAPSHOT</version>
- <name>japanese-addon</name>
- <url>http://maven.apache.org</url>
-
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <maven.compiler.source>1.8</maven.compiler.source>
- <maven.compiler.target>1.8</maven.compiler.target>
- </properties>
+ <version>2.2.1-SNAPSHOT</version>
+ <name>Apache OpenNLP Japanese Addon</name>
<dependencies>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.13.1</version>
- <scope>test</scope>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-tools</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-tools</artifactId>
- <version>1.9.1-SNAPSHOT</version>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerArgument>-Xlint</compilerArgument>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoNameContextGenerator.java b/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoNameContextGenerator.java
index a45a7c7..bda691b 100644
--- a/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoNameContextGenerator.java
+++ b/japanese-addon/src/main/java/opennlp/tools/namefind/AuxiliaryInfoNameContextGenerator.java
@@ -76,6 +76,6 @@ public class AuxiliaryInfoNameContextGenerator extends DefaultNameContextGenerat
features.add("ppo=" + ppo);
}
- return features.toArray(new String[features.size()]);
+ return features.toArray(new String[0]);
}
}
diff --git a/japanese-addon/src/main/java/opennlp/tools/util/featuregen/lang/jpn/BrownTokenClassFeatureGenerator.java b/japanese-addon/src/main/java/opennlp/tools/util/featuregen/lang/jpn/BrownTokenClassFeatureGenerator.java
index 341f46a..2d12171 100644
--- a/japanese-addon/src/main/java/opennlp/tools/util/featuregen/lang/jpn/BrownTokenClassFeatureGenerator.java
+++ b/japanese-addon/src/main/java/opennlp/tools/util/featuregen/lang/jpn/BrownTokenClassFeatureGenerator.java
@@ -40,8 +40,8 @@ public class BrownTokenClassFeatureGenerator implements AdaptiveFeatureGenerator
String wordShape = FeatureGeneratorUtil.tokenFeature(tokens[index]);
List<String> wordClasses = BrownTokenClasses.getWordClasses(tokens[index], brownLexicon);
- for (int i = 0; i < wordClasses.size(); i++) {
- features.add("c," + "browncluster" + "=" + wordShape + "," + wordClasses.get(i));
+ for (String wordClass : wordClasses) {
+ features.add("c," + "browncluster" + "=" + wordShape + "," + wordClass);
}
}
diff --git a/japanese-addon/src/test/java/opennlp/tools/namefind/AuxiliaryInfoUtilTest.java b/japanese-addon/src/test/java/opennlp/tools/namefind/AuxiliaryInfoUtilTest.java
index 91bd756..4c0bcc1 100644
--- a/japanese-addon/src/test/java/opennlp/tools/namefind/AuxiliaryInfoUtilTest.java
+++ b/japanese-addon/src/test/java/opennlp/tools/namefind/AuxiliaryInfoUtilTest.java
@@ -17,56 +17,59 @@
package opennlp.tools.namefind;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
public class AuxiliaryInfoUtilTest {
@Test
public void testGetSeparatorIndex() throws Exception {
- Assert.assertEquals(0, AuxiliaryInfoUtil.getSeparatorIndex("/POStag"));
- Assert.assertEquals(1, AuxiliaryInfoUtil.getSeparatorIndex("1/POStag"));
- Assert.assertEquals(10, AuxiliaryInfoUtil.getSeparatorIndex("word/stuff/POStag"));
+ assertEquals(0, AuxiliaryInfoUtil.getSeparatorIndex("/POStag"));
+ assertEquals(1, AuxiliaryInfoUtil.getSeparatorIndex("1/POStag"));
+ assertEquals(10, AuxiliaryInfoUtil.getSeparatorIndex("word/stuff/POStag"));
}
- @Test(expected = RuntimeException.class)
+ @Test
public void testGetSeparatorIndexNoPos() throws Exception {
- AuxiliaryInfoUtil.getSeparatorIndex("NOPOStags");
+ Assertions.assertThrows(RuntimeException.class, () ->
+ AuxiliaryInfoUtil.getSeparatorIndex("NOPOStags"));
}
@Test
public void testGetWordPart() throws Exception {
- Assert.assertEquals(" ", AuxiliaryInfoUtil.getWordPart("/POStag"));
- Assert.assertEquals("1", AuxiliaryInfoUtil.getWordPart("1/POStag"));
- Assert.assertEquals("word", AuxiliaryInfoUtil.getWordPart("word/POStag"));
- Assert.assertEquals("word/stuff", AuxiliaryInfoUtil.getWordPart("word/stuff/POStag"));
+ assertEquals(" ", AuxiliaryInfoUtil.getWordPart("/POStag"));
+ assertEquals("1", AuxiliaryInfoUtil.getWordPart("1/POStag"));
+ assertEquals("word", AuxiliaryInfoUtil.getWordPart("word/POStag"));
+ assertEquals("word/stuff", AuxiliaryInfoUtil.getWordPart("word/stuff/POStag"));
}
@Test
public void testGetWordParts() throws Exception {
String[] results = AuxiliaryInfoUtil.getWordParts(new String[]{"1/A", "234/B", "3456/C", "/D"});
- Assert.assertEquals(4, results.length);
- Assert.assertEquals("1", results[0]);
- Assert.assertEquals("234", results[1]);
- Assert.assertEquals("3456", results[2]);
- Assert.assertEquals(" ", results[3]);
+ assertEquals(4, results.length);
+ assertEquals("1", results[0]);
+ assertEquals("234", results[1]);
+ assertEquals("3456", results[2]);
+ assertEquals(" ", results[3]);
}
@Test
public void testGetAuxPart() throws Exception {
- Assert.assertEquals("POStag", AuxiliaryInfoUtil.getAuxPart("/POStag"));
- Assert.assertEquals("POStag", AuxiliaryInfoUtil.getAuxPart("1/POStag"));
- Assert.assertEquals("POStag", AuxiliaryInfoUtil.getAuxPart("word/POStag"));
- Assert.assertEquals("POStag", AuxiliaryInfoUtil.getAuxPart("word/stuff/POStag"));
+ assertEquals("POStag", AuxiliaryInfoUtil.getAuxPart("/POStag"));
+ assertEquals("POStag", AuxiliaryInfoUtil.getAuxPart("1/POStag"));
+ assertEquals("POStag", AuxiliaryInfoUtil.getAuxPart("word/POStag"));
+ assertEquals("POStag", AuxiliaryInfoUtil.getAuxPart("word/stuff/POStag"));
}
@Test
public void testGetAuxParts() throws Exception {
String[] results = AuxiliaryInfoUtil.getAuxParts(new String[] {"1/ABC", "234/B", "3456/CD", "/DEFGH"});
- Assert.assertEquals(4, results.length);
- Assert.assertEquals("ABC", results[0]);
- Assert.assertEquals("B", results[1]);
- Assert.assertEquals("CD", results[2]);
- Assert.assertEquals("DEFGH", results[3]);
+ assertEquals(4, results.length);
+ assertEquals("ABC", results[0]);
+ assertEquals("B", results[1]);
+ assertEquals("CD", results[2]);
+ assertEquals("DEFGH", results[3]);
}
}
diff --git a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGeneratorTest.java b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGeneratorTest.java
index aa03110..8c8f44d 100644
--- a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGeneratorTest.java
+++ b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/AuxiliaryInfoAwareDelegateFeatureGeneratorTest.java
@@ -20,9 +20,10 @@ package opennlp.tools.util.featuregen;
import java.util.ArrayList;
import java.util.List;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
public class AuxiliaryInfoAwareDelegateFeatureGeneratorTest {
@@ -30,7 +31,7 @@ public class AuxiliaryInfoAwareDelegateFeatureGeneratorTest {
private List<String> features;
- @Before
+ @BeforeEach
public void setUp() throws Exception {
features = new ArrayList<>();
}
@@ -41,8 +42,8 @@ public class AuxiliaryInfoAwareDelegateFeatureGeneratorTest {
new IdentityFeatureGenerator(), false);
featureGenerator.createFeatures(features, testSentence, 2, null);
- Assert.assertEquals(1, features.size());
- Assert.assertEquals("w3", features.get(0));
+ assertEquals(1, features.size());
+ assertEquals("w3", features.get(0));
}
@Test
@@ -51,8 +52,8 @@ public class AuxiliaryInfoAwareDelegateFeatureGeneratorTest {
new IdentityFeatureGenerator(), true);
featureGenerator.createFeatures(features, testSentence, 3, null);
- Assert.assertEquals(1, features.size());
- Assert.assertEquals("pos4", features.get(0));
+ assertEquals(1, features.size());
+ assertEquals("pos4", features.get(0));
}
static class IdentityFeatureGenerator implements AdaptiveFeatureGenerator {
diff --git a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/BigramNameFeatureGeneratorTest.java b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/BigramNameFeatureGeneratorTest.java
index ddfa024..46d952e 100644
--- a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/BigramNameFeatureGeneratorTest.java
+++ b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/BigramNameFeatureGeneratorTest.java
@@ -20,18 +20,19 @@ package opennlp.tools.util.featuregen.lang.jpn;
import java.util.ArrayList;
import java.util.List;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
public class BigramNameFeatureGeneratorTest {
private List<String> features;
static String[] testSentence = new String[] {"This", "is", "an", "example", "sentence"};
- @Before
+ @BeforeEach
public void setUp() throws Exception {
features = new ArrayList<>();
}
@@ -45,9 +46,9 @@ public class BigramNameFeatureGeneratorTest {
generator.createFeatures(features, testSentence, testTokenIndex, null);
- Assert.assertEquals(2, features.size());
- Assert.assertEquals("w,nw=This,is", features.get(0));
- Assert.assertEquals("wc,nc=alpha,alpha", features.get(1));
+ assertEquals(2, features.size());
+ assertEquals("w,nw=This,is", features.get(0));
+ assertEquals("wc,nc=alpha,alpha", features.get(1));
}
@Test
@@ -59,11 +60,11 @@ public class BigramNameFeatureGeneratorTest {
generator.createFeatures(features, testSentence, testTokenIndex, null);
- Assert.assertEquals(4, features.size());
- Assert.assertEquals("pw,w=is,an", features.get(0));
- Assert.assertEquals("pwc,wc=alpha,alpha", features.get(1));
- Assert.assertEquals("w,nw=an,example", features.get(2));
- Assert.assertEquals("wc,nc=alpha,alpha", features.get(3));
+ assertEquals(4, features.size());
+ assertEquals("pw,w=is,an", features.get(0));
+ assertEquals("pwc,wc=alpha,alpha", features.get(1));
+ assertEquals("w,nw=an,example", features.get(2));
+ assertEquals("wc,nc=alpha,alpha", features.get(3));
}
@Test
@@ -75,9 +76,9 @@ public class BigramNameFeatureGeneratorTest {
generator.createFeatures(features, testSentence, testTokenIndex, null);
- Assert.assertEquals(2, features.size());
- Assert.assertEquals("pw,w=example,sentence", features.get(0));
- Assert.assertEquals("pwc,wc=alpha,alpha", features.get(1));
+ assertEquals(2, features.size());
+ assertEquals("pw,w=example,sentence", features.get(0));
+ assertEquals("pwc,wc=alpha,alpha", features.get(1));
}
@Test
@@ -88,9 +89,8 @@ public class BigramNameFeatureGeneratorTest {
final int testTokenIndex = 0;
AdaptiveFeatureGenerator generator = new BigramNameFeatureGenerator();
-
generator.createFeatures(features, shortSentence, testTokenIndex, null);
- Assert.assertEquals(0, features.size());
+ assertEquals(0, features.size());
}
}
diff --git a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/FeatureGeneratorUtilTest.java b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/FeatureGeneratorUtilTest.java
index ce5816f..1b52161 100644
--- a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/FeatureGeneratorUtilTest.java
+++ b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/FeatureGeneratorUtilTest.java
@@ -17,51 +17,52 @@
package opennlp.tools.util.featuregen.lang.jpn;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
public class FeatureGeneratorUtilTest {
@Test
public void test() {
// digits
- Assert.assertEquals("digit", FeatureGeneratorUtil.tokenFeature("12"));
- Assert.assertEquals("digit", FeatureGeneratorUtil.tokenFeature("1234"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("abcd234"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("1234-56"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("4/6/2017"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("1,234,567"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("12.34567"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("123(456)7890"));
+ assertEquals("digit", FeatureGeneratorUtil.tokenFeature("12"));
+ assertEquals("digit", FeatureGeneratorUtil.tokenFeature("1234"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("abcd234"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("1234-56"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("4/6/2017"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("1,234,567"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("12.34567"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("123(456)7890"));
// letters
- Assert.assertEquals("alpha", FeatureGeneratorUtil.tokenFeature("opennlp"));
- Assert.assertEquals("alpha", FeatureGeneratorUtil.tokenFeature("O"));
- Assert.assertEquals("alpha", FeatureGeneratorUtil.tokenFeature("OPENNLP"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("A."));
- Assert.assertEquals("alpha", FeatureGeneratorUtil.tokenFeature("Mike"));
- Assert.assertEquals("alpha", FeatureGeneratorUtil.tokenFeature("somethingStupid"));
+ assertEquals("alpha", FeatureGeneratorUtil.tokenFeature("opennlp"));
+ assertEquals("alpha", FeatureGeneratorUtil.tokenFeature("O"));
+ assertEquals("alpha", FeatureGeneratorUtil.tokenFeature("OPENNLP"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("A."));
+ assertEquals("alpha", FeatureGeneratorUtil.tokenFeature("Mike"));
+ assertEquals("alpha", FeatureGeneratorUtil.tokenFeature("somethingStupid"));
// symbols
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature(","));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("."));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("?"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("!"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("#"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("%"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("&"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature(","));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("."));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("?"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("!"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("#"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("%"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("&"));
}
@Test
public void testJapanese() {
// Hiragana
- Assert.assertEquals("hira", FeatureGeneratorUtil.tokenFeature("そういえば"));
- Assert.assertEquals("hira", FeatureGeneratorUtil.tokenFeature("おーぷん・そ〜す・そふとうぇあ"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("あぱっち・そふとうぇあ財団"));
+ assertEquals("hira", FeatureGeneratorUtil.tokenFeature("そういえば"));
+ assertEquals("hira", FeatureGeneratorUtil.tokenFeature("おーぷん・そ〜す・そふとうぇあ"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("あぱっち・そふとうぇあ財団"));
// Katakana
- Assert.assertEquals("kata", FeatureGeneratorUtil.tokenFeature("ジャパン"));
- Assert.assertEquals("kata", FeatureGeneratorUtil.tokenFeature("オープン・ソ〜ス・ソフトウェア"));
- Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("アパッチ・ソフトウェア財団"));
+ assertEquals("kata", FeatureGeneratorUtil.tokenFeature("ジャパン"));
+ assertEquals("kata", FeatureGeneratorUtil.tokenFeature("オープン・ソ〜ス・ソフトウェア"));
+ assertEquals("other", FeatureGeneratorUtil.tokenFeature("アパッチ・ソフトウェア財団"));
}
}
diff --git a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/TokenClassFeatureGeneratorTest.java b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/TokenClassFeatureGeneratorTest.java
index be9359f..dc6962d 100644
--- a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/TokenClassFeatureGeneratorTest.java
+++ b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/TokenClassFeatureGeneratorTest.java
@@ -20,18 +20,19 @@ package opennlp.tools.util.featuregen.lang.jpn;
import java.util.ArrayList;
import java.util.List;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
public class TokenClassFeatureGeneratorTest {
private List<String> features;
static String[] testSentence = new String[] {"This", "is", "an", "Example", "sentence"};
- @Before
+ @BeforeEach
public void setUp() throws Exception {
features = new ArrayList<>();
}
@@ -45,9 +46,9 @@ public class TokenClassFeatureGeneratorTest {
generator.createFeatures(features, testSentence, testTokenIndex, null);
- Assert.assertEquals(2, features.size());
- Assert.assertEquals("wc=alpha", features.get(0));
- Assert.assertEquals("w&c=example,alpha", features.get(1));
+ assertEquals(2, features.size());
+ assertEquals("wc=alpha", features.get(0));
+ assertEquals("w&c=example,alpha", features.get(1));
}
@Test
@@ -59,7 +60,7 @@ public class TokenClassFeatureGeneratorTest {
generator.createFeatures(features, testSentence, testTokenIndex, null);
- Assert.assertEquals(1, features.size());
- Assert.assertEquals("wc=alpha", features.get(0));
+ assertEquals(1, features.size());
+ assertEquals("wc=alpha", features.get(0));
}
}
diff --git a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/TokenPatternFeatureGeneratorTest.java b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/TokenPatternFeatureGeneratorTest.java
index d74051e..24509ef 100644
--- a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/TokenPatternFeatureGeneratorTest.java
+++ b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/TokenPatternFeatureGeneratorTest.java
@@ -20,17 +20,18 @@ package opennlp.tools.util.featuregen.lang.jpn;
import java.util.ArrayList;
import java.util.List;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
public class TokenPatternFeatureGeneratorTest {
private List<String> features;
- @Before
+ @BeforeEach
public void setUp() throws Exception {
features = new ArrayList<>();
}
@@ -44,8 +45,8 @@ public class TokenPatternFeatureGeneratorTest {
AdaptiveFeatureGenerator generator = new TokenPatternFeatureGenerator();
generator.createFeatures(features, testSentence, testTokenIndex, null);
- Assert.assertEquals(1, features.size());
- Assert.assertEquals("st=example", features.get(0));
+ assertEquals(1, features.size());
+ assertEquals("st=example", features.get(0));
}
@Test
@@ -57,20 +58,20 @@ public class TokenPatternFeatureGeneratorTest {
AdaptiveFeatureGenerator generator = new TokenPatternFeatureGenerator();
generator.createFeatures(features, testSentence, testTokenIndex, null);
- Assert.assertEquals(14, features.size());
- Assert.assertEquals("stn=5", features.get(0));
- Assert.assertEquals("pt2=alphaalpha", features.get(1));
- Assert.assertEquals("pt3=alphaalphaalpha", features.get(2));
- Assert.assertEquals("st=this", features.get(3));
- Assert.assertEquals("pt2=alphaalpha", features.get(4));
- Assert.assertEquals("pt3=alphaalphaalpha", features.get(5));
- Assert.assertEquals("st=is", features.get(6));
- Assert.assertEquals("pt2=alphaalpha", features.get(7));
- Assert.assertEquals("pt3=alphaalphaalpha", features.get(8));
- Assert.assertEquals("st=an", features.get(9));
- Assert.assertEquals("pt2=alphaalpha", features.get(10));
- Assert.assertEquals("st=example", features.get(11));
- Assert.assertEquals("st=sentence", features.get(12));
- Assert.assertEquals("pta=alphaalphaalphaalphaalpha", features.get(13));
+ assertEquals(14, features.size());
+ assertEquals("stn=5", features.get(0));
+ assertEquals("pt2=alphaalpha", features.get(1));
+ assertEquals("pt3=alphaalphaalpha", features.get(2));
+ assertEquals("st=this", features.get(3));
+ assertEquals("pt2=alphaalpha", features.get(4));
+ assertEquals("pt3=alphaalphaalpha", features.get(5));
+ assertEquals("st=is", features.get(6));
+ assertEquals("pt2=alphaalpha", features.get(7));
+ assertEquals("pt3=alphaalphaalpha", features.get(8));
+ assertEquals("st=an", features.get(9));
+ assertEquals("pt2=alphaalpha", features.get(10));
+ assertEquals("st=example", features.get(11));
+ assertEquals("st=sentence", features.get(12));
+ assertEquals("pta=alphaalphaalphaalphaalpha", features.get(13));
}
}
diff --git a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/TrigramNameFeatureGeneratorTest.java b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/TrigramNameFeatureGeneratorTest.java
index 546a0bd..789c508 100644
--- a/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/TrigramNameFeatureGeneratorTest.java
+++ b/japanese-addon/src/test/java/opennlp/tools/util/featuregen/lang/jpn/TrigramNameFeatureGeneratorTest.java
@@ -20,18 +20,19 @@ package opennlp.tools.util.featuregen.lang.jpn;
import java.util.ArrayList;
import java.util.List;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
public class TrigramNameFeatureGeneratorTest {
private List<String> features;
static String[] testSentence = new String[] {"This", "is", "an", "example", "sentence"};
- @Before
+ @BeforeEach
public void setUp() throws Exception {
features = new ArrayList<>();
}
@@ -45,9 +46,9 @@ public class TrigramNameFeatureGeneratorTest {
generator.createFeatures(features, testSentence, testTokenIndex, null);
- Assert.assertEquals(2, features.size());
- Assert.assertEquals("w,nw,nnw=This,is,an", features.get(0));
- Assert.assertEquals("wc,nwc,nnwc=alpha,alpha,alpha", features.get(1));
+ assertEquals(2, features.size());
+ assertEquals("w,nw,nnw=This,is,an", features.get(0));
+ assertEquals("wc,nwc,nnwc=alpha,alpha,alpha", features.get(1));
}
@Test
@@ -59,9 +60,9 @@ public class TrigramNameFeatureGeneratorTest {
generator.createFeatures(features, testSentence, testTokenIndex, null);
- Assert.assertEquals(2, features.size());
- Assert.assertEquals("w,nw,nnw=is,an,example", features.get(0));
- Assert.assertEquals("wc,nwc,nnwc=alpha,alpha,alpha", features.get(1));
+ assertEquals(2, features.size());
+ assertEquals("w,nw,nnw=is,an,example", features.get(0));
+ assertEquals("wc,nwc,nnwc=alpha,alpha,alpha", features.get(1));
}
@Test
@@ -73,11 +74,11 @@ public class TrigramNameFeatureGeneratorTest {
generator.createFeatures(features, testSentence, testTokenIndex, null);
- Assert.assertEquals(4, features.size());
- Assert.assertEquals("ppw,pw,w=This,is,an", features.get(0));
- Assert.assertEquals("ppwc,pwc,wc=alpha,alpha,alpha", features.get(1));
- Assert.assertEquals("w,nw,nnw=an,example,sentence", features.get(2));
- Assert.assertEquals("wc,nwc,nnwc=alpha,alpha,alpha", features.get(3));
+ assertEquals(4, features.size());
+ assertEquals("ppw,pw,w=This,is,an", features.get(0));
+ assertEquals("ppwc,pwc,wc=alpha,alpha,alpha", features.get(1));
+ assertEquals("w,nw,nnw=an,example,sentence", features.get(2));
+ assertEquals("wc,nwc,nnwc=alpha,alpha,alpha", features.get(3));
}
@Test
@@ -89,9 +90,9 @@ public class TrigramNameFeatureGeneratorTest {
generator.createFeatures(features, testSentence, testTokenIndex, null);
- Assert.assertEquals(2, features.size());
- Assert.assertEquals("ppw,pw,w=an,example,sentence", features.get(0));
- Assert.assertEquals("ppwc,pwc,wc=alpha,alpha,alpha", features.get(1));
+ assertEquals(2, features.size());
+ assertEquals("ppw,pw,w=an,example,sentence", features.get(0));
+ assertEquals("ppwc,pwc,wc=alpha,alpha,alpha", features.get(1));
}
@Test
@@ -105,6 +106,6 @@ public class TrigramNameFeatureGeneratorTest {
generator.createFeatures(features, shortSentence, testTokenIndex, null);
- Assert.assertEquals(0, features.size());
+ assertEquals(0, features.size());
}
}
diff --git a/jwnl-addon/pom.xml b/jwnl-addon/pom.xml
index be38764..9b0a8ed 100644
--- a/jwnl-addon/pom.xml
+++ b/jwnl-addon/pom.xml
@@ -1,50 +1,78 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
-
- <groupId>org.apache.opennlp</groupId>
+ <parent>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ </parent>
+
<artifactId>jwnl-addon</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <version>2.2.1-SNAPSHOT</version>
<packaging>jar</packaging>
- <name>JWNL Addon</name>
-
- <url>http://maven.apache.org</url>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>2.3.2</version>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
+ <name>Apache OpenNLP JWNL Addon</name>
<dependencies>
+ <dependency>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-tools</artifactId>
+ </dependency>
+
<dependency>
<groupId>net.sf.jwordnet</groupId>
<artifactId>jwnl</artifactId>
<version>1.3.3</version>
<scope>compile</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ </dependency>
<dependency>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-tools</artifactId>
- <version>1.6.0-SNAPSHOT</version>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>3.8.1</version>
- <scope>test</scope>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerArgument>-Xlint</compilerArgument>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/jwnl-addon/src/main/java/opennlp/jwnl/lemmatizer/JWNLLemmatizer.java b/jwnl-addon/src/main/java/opennlp/jwnl/lemmatizer/JWNLLemmatizer.java
index f84530a..ba55a0f 100644
--- a/jwnl-addon/src/main/java/opennlp/jwnl/lemmatizer/JWNLLemmatizer.java
+++ b/jwnl-addon/src/main/java/opennlp/jwnl/lemmatizer/JWNLLemmatizer.java
@@ -18,11 +18,11 @@
package opennlp.jwnl.lemmatizer;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-import opennlp.tools.lemmatizer.DictionaryLemmatizer;
-
import net.didion.jwnl.JWNLException;
import net.didion.jwnl.data.Adjective;
import net.didion.jwnl.data.FileDictionaryElementFactory;
@@ -42,8 +42,9 @@ import net.didion.jwnl.dictionary.morph.Operation;
import net.didion.jwnl.dictionary.morph.TokenizerOperation;
import net.didion.jwnl.princeton.data.PrincetonWN17FileDictionaryElementFactory;
import net.didion.jwnl.princeton.file.PrincetonRandomAccessDictionaryFile;
+import opennlp.tools.lemmatizer.Lemmatizer;
-public class JWNLLemmatizer implements DictionaryLemmatizer {
+public class JWNLLemmatizer implements Lemmatizer {
private net.didion.jwnl.dictionary.Dictionary dict;
private MorphologicalProcessor morphy;
@@ -52,18 +53,18 @@ public class JWNLLemmatizer implements DictionaryLemmatizer {
* Creates JWNL dictionary and morphological processor objects in
* JWNLemmatizer constructor. It also loads the JWNL configuration into the
* constructor.
- *
+ * <p>
* Constructor code based on Apache OpenNLP JWNLDictionary class.
*
* @param wnDirectory
* @throws IOException
- * @throws JWNLException
*/
- public JWNLLemmatizer(String wnDirectory) throws IOException, JWNLException {
+ public JWNLLemmatizer(String wnDirectory) throws IOException {
+ super();
PointerType.initialize();
Adjective.initialize();
VerbFrame.initialize();
- Map<POS, String[][]> suffixMap = new HashMap<POS, String[][]>();
+ Map<POS, String[][]> suffixMap = new HashMap<>();
suffixMap.put(POS.NOUN, new String[][] { { "s", "" }, { "ses", "s" },
{ "xes", "x" }, { "zes", "z" }, { "ches", "ch" }, { "shes", "sh" },
{ "men", "man" }, { "ies", "y" } });
@@ -91,8 +92,7 @@ public class JWNLLemmatizer implements DictionaryLemmatizer {
dict = net.didion.jwnl.dictionary.Dictionary.getInstance();
morphy = dict.getMorphologicalProcessor();
}
-
-
+
/**
* It takes a word and a POS tag and obtains a word's lemma from WordNet.
*
@@ -103,7 +103,7 @@ public class JWNLLemmatizer implements DictionaryLemmatizer {
public String lemmatize(String word, String postag) {
String constantTag = "NNP";
IndexWord baseForm;
- String lemma = null;
+ String lemma;
try {
POS pos;
if (postag.startsWith("N") || postag.startsWith("n")) {
@@ -121,7 +121,7 @@ public class JWNLLemmatizer implements DictionaryLemmatizer {
if (baseForm != null) {
lemma = baseForm.getLemma().toString();
}
- else if (baseForm == null && postag.startsWith(String.valueOf(constantTag))) {
+ else if (baseForm == null && postag.startsWith(constantTag)) {
lemma = word;
}
else {
@@ -134,5 +134,18 @@ public class JWNLLemmatizer implements DictionaryLemmatizer {
return lemma;
}
+ @Override
+ public String[] lemmatize(final String[] tokens, final String[] postags) {
+ List<String> lemmas = new ArrayList<>();
+ for (int i = 0; i < tokens.length; i++) {
+ lemmas.add(this.lemmatize(tokens[i], postags[i]));
+ }
+ return lemmas.toArray(new String[0]);
+ }
+
+ @Override
+ public List<List<String>> lemmatize(final List<String> tokens, final List<String> posTags) {
+ throw new UnsupportedOperationException("Method not implemented here!");
+ }
}
diff --git a/liblinear-addon/pom.xml b/liblinear-addon/pom.xml
index 7fa3e51..52f67a3 100644
--- a/liblinear-addon/pom.xml
+++ b/liblinear-addon/pom.xml
@@ -21,32 +21,20 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
-
<parent>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp</artifactId>
- <version>1.6.0-SNAPSHOT</version>
- <relativePath>../opennlp/pom.xml</relativePath>
- </parent>
-
- <artifactId>opennlp-liblinear-addon</artifactId>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>liblinear-addon</artifactId>
<packaging>jar</packaging>
- <name>Apache OpenNLP Liblinear Addon</name>
-
- <repositories>
- <repository>
- <id>ApacheIncubatorRepository</id>
- <url>
- http://people.apache.org/repo/m2-incubating-repository/
- </url>
- </repository>
- </repositories>
+ <name>Apache OpenNLP LibLinear Addon</name>
<dependencies>
<dependency>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp-tools</artifactId>
- <version>1.6.0-SNAPSHOT</version>
</dependency>
<dependency>
@@ -56,9 +44,18 @@
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
</dependency>
</dependencies>
diff --git a/liblinear-addon/src/main/java/LiblinearModel.java b/liblinear-addon/src/main/java/LiblinearModel.java
index f1037de..ea52e89 100644
--- a/liblinear-addon/src/main/java/LiblinearModel.java
+++ b/liblinear-addon/src/main/java/LiblinearModel.java
@@ -27,6 +27,7 @@ import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -41,21 +42,21 @@ import de.bwaldvogel.liblinear.Model;
// TODO: The features need to be serialized with the model
// the liblinear model only contains the ints and weights,
-// but the string lables get lost ... basically that are two maps.
+// but the string labels get lost ... basically that are two maps.
// One for outcomes, one for the features ...
public class LiblinearModel implements MaxentModel, SerializableArtifact {
- private static final Charset LIBLINEAR_MODEL_ENCODING = Charset.forName("UTF-8");
+ private static final Charset LIBLINEAR_MODEL_ENCODING = StandardCharsets.UTF_8;
- private Model model;
+ private final Model model;
- // Lets read them from disk, when model is loaded ...
- private String outcomeLabels[];
- private Map<String, Integer> predMap;
+ // Let's read them from disk, when model is loaded ...
+ private final String[] outcomeLabels;
+ private final Map<String, Integer> predMap;
- public LiblinearModel(Model model, String outcomes[], Map<String, Integer> predMap) {
+ public LiblinearModel(Model model, String[] outcomes, Map<String, Integer> predMap) {
this.model = model;
this.outcomeLabels = outcomes;
this.predMap = predMap;
@@ -63,43 +64,41 @@ public class LiblinearModel implements MaxentModel, SerializableArtifact {
public LiblinearModel(InputStream in) throws IOException {
- DataInputStream di = new DataInputStream(in);
-
- int modelByteLength = di.readInt();
-
- // TODO: We should have a fixed memory limit here ...
-
- byte modelBytes[] = new byte[modelByteLength];
- di.read(modelBytes);
-
- int outcomeLabelLength = di.readInt();
-
- outcomeLabels = new String[outcomeLabelLength];
- for (int i = 0; i < outcomeLabelLength; i++) {
- outcomeLabels[i] = di.readUTF();
- }
-
- predMap = new HashMap<String, Integer>();
-
- int predMapSize = di.readInt();
- for (int i = 0; i < predMapSize; i++) {
- String key = di.readUTF();
- int value = di.readInt();
- predMap.put(key, value);
+ try (DataInputStream di = new DataInputStream(in)) {
+ int modelByteLength = di.readInt();
+
+ // TODO: We should have a fixed memory limit here ...
+
+ byte[] modelBytes = new byte[modelByteLength];
+ di.read(modelBytes);
+
+ int outcomeLabelLength = di.readInt();
+
+ outcomeLabels = new String[outcomeLabelLength];
+ for (int i = 0; i < outcomeLabelLength; i++) {
+ outcomeLabels[i] = di.readUTF();
+ }
+
+ predMap = new HashMap<>();
+
+ int predMapSize = di.readInt();
+ for (int i = 0; i < predMapSize; i++) {
+ String key = di.readUTF();
+ int value = di.readInt();
+ predMap.put(key, value);
+ }
+
+ model = Linear.loadModel(new InputStreamReader(new ByteArrayInputStream(modelBytes), LIBLINEAR_MODEL_ENCODING));
}
-
- model = Linear.loadModel(new InputStreamReader(new ByteArrayInputStream(modelBytes), LIBLINEAR_MODEL_ENCODING));
}
+ @Override
public double[] eval(String[] features) {
-
// Note: If a feature can't be mapped, it will be ignored!
-
- List<Integer> context = new ArrayList<Integer>(features.length);
-
- for (int i = 0; i < features.length; i++) {
- Integer feature = predMap.get(features[i]);
-
+ List<Integer> context = new ArrayList<>(features.length);
+ for (String s : features) {
+ Integer feature = predMap.get(s);
+
if (feature != null) {
context.add(feature);
}
@@ -108,19 +107,21 @@ public class LiblinearModel implements MaxentModel, SerializableArtifact {
return eval(context);
}
+ @Override
public double[] eval(String[] context, double[] probs) {
return eval(context);
}
+ @Override
public double[] eval(String[] context, float[] values) {
return eval(context);
}
private double[] eval(List<Integer> context) {
- double outcomes[] = new double[outcomeLabels.length];
+ double[] outcomes = new double[outcomeLabels.length];
- Feature vx[] = new Feature[context.size()];
+ Feature[] vx = new Feature[context.size()];
for (int i = 0; i < context.size(); i++) {
vx[i] = new FeatureNode(context.get(i) + 1, 1d);
@@ -130,12 +131,14 @@ public class LiblinearModel implements MaxentModel, SerializableArtifact {
return outcomes;
}
-
+
+ @Override
public String getAllOutcomes(double[] outcomes) {
// TODO: Return prev outcomes ..
return null;
}
+ @Override
public String getBestOutcome(double[] ocs) {
int best = 0;
for (int i = 1; i < ocs.length; i++)
@@ -143,11 +146,7 @@ public class LiblinearModel implements MaxentModel, SerializableArtifact {
return outcomeLabels[best];
}
- // TODO: This method needs to go away from the interface ... !!!
- public Object[] getDataStructures() {
- return null;
- }
-
+ @Override
public int getIndex(String outcome) {
for (int i = 0; i < outcomeLabels.length; i++) {
if (outcomeLabels[i].equals(outcome)) {
@@ -158,38 +157,40 @@ public class LiblinearModel implements MaxentModel, SerializableArtifact {
return -1;
}
+ @Override
public int getNumOutcomes() {
return outcomeLabels.length;
}
+ @Override
public String getOutcome(int i) {
return outcomeLabels[i];
}
public void serialize(OutputStream out) throws IOException {
- DataOutputStream ds = new DataOutputStream(out);
-
- ByteArrayOutputStream modelBytes = new ByteArrayOutputStream();
- Linear.saveModel(new OutputStreamWriter(modelBytes, LIBLINEAR_MODEL_ENCODING), model);
+ try (DataOutputStream ds = new DataOutputStream(out);
+ ByteArrayOutputStream modelBytes = new ByteArrayOutputStream()) {
- ds.writeInt(modelBytes.size());
- ds.write(modelBytes.toByteArray());
-
- // write string array
- // write label count
- ds.writeInt(outcomeLabels.length);
-
- // write each label
- for (String outcomeLabel : outcomeLabels) {
- ds.writeUTF(outcomeLabel);
- }
+ Linear.saveModel(new OutputStreamWriter(modelBytes, LIBLINEAR_MODEL_ENCODING), model);
+ ds.writeInt(modelBytes.size());
+ ds.write(modelBytes.toByteArray());
+
+ // write string array
+ // write label count
+ ds.writeInt(outcomeLabels.length);
- // write entry count
- ds.writeInt(predMap.size());
- for (Map.Entry<String, Integer> entry : predMap.entrySet()) {
- ds.writeUTF(entry.getKey());
- ds.writeInt(entry.getValue());
+ // write each label
+ for (String outcomeLabel : outcomeLabels) {
+ ds.writeUTF(outcomeLabel);
+ }
+
+ // write entry count
+ ds.writeInt(predMap.size());
+ for (Map.Entry<String, Integer> entry : predMap.entrySet()) {
+ ds.writeUTF(entry.getKey());
+ ds.writeInt(entry.getValue());
+ }
}
}
diff --git a/liblinear-addon/src/main/java/LiblinearTrainer.java b/liblinear-addon/src/main/java/LiblinearTrainer.java
index e3bc0fd..abfac72 100644
--- a/liblinear-addon/src/main/java/LiblinearTrainer.java
+++ b/liblinear-addon/src/main/java/LiblinearTrainer.java
@@ -23,9 +23,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import opennlp.tools.ml.AbstractEventTrainer;
-import opennlp.tools.ml.model.DataIndexer;
-import opennlp.tools.ml.model.MaxentModel;
import de.bwaldvogel.liblinear.Feature;
import de.bwaldvogel.liblinear.FeatureNode;
import de.bwaldvogel.liblinear.Linear;
@@ -34,26 +31,25 @@ import de.bwaldvogel.liblinear.Parameter;
import de.bwaldvogel.liblinear.Problem;
import de.bwaldvogel.liblinear.SolverType;
+import opennlp.tools.ml.AbstractEventTrainer;
+import opennlp.tools.ml.model.DataIndexer;
+import opennlp.tools.ml.model.MaxentModel;
+import opennlp.tools.util.TrainingParameters;
+
public class LiblinearTrainer extends AbstractEventTrainer {
- private SolverType solverType;
- private Double c;
- private Double eps;
- private Double p;
-
- private int bias;
+ private final SolverType solverType;
+ private final double c;
+ private final double eps;
+ private final double p;
+ private final int bias;
- public LiblinearTrainer() {
- }
-
- @Override
- public void init(Map<String, String> trainParams,
- Map<String, String> reportMap) {
- String solverTypeName = trainParams.get("solverType");
+ public LiblinearTrainer(TrainingParameters trainParams) {
+ String solverTypeName = trainParams.getStringParameter("solverType", "");
if (solverTypeName != null) {
try {
- solverType = SolverType.valueOf(trainParams.get("solverType"));
+ solverType = SolverType.valueOf(trainParams.getStringParameter("solverType", ""));
}
catch (IllegalArgumentException e) {
throw new IllegalArgumentException("solverType [" + solverTypeName + "] is not available!");
@@ -63,42 +59,10 @@ public class LiblinearTrainer extends AbstractEventTrainer {
throw new IllegalArgumentException("solverType needs to be specified!");
}
- String cValueString = trainParams.get("c");
-
- if (cValueString != null) {
- c = Double.valueOf(cValueString);
- }
- else {
- throw new IllegalArgumentException("c must be specified");
- }
-
- // eps
- String epsValueString = trainParams.get("eps");
-
- if (epsValueString != null) {
- eps = Double.valueOf(epsValueString);
- }
- else {
- throw new IllegalArgumentException("eps must be specified");
- }
-
- String pValueString = trainParams.get("p");
-
- if (pValueString != null) {
- p = Double.valueOf(pValueString);
- }
- else {
- throw new IllegalArgumentException("p must be specified");
- }
-
- String biasValueString = trainParams.get("bias");
-
- if (biasValueString != null) {
- bias = Integer.valueOf(biasValueString);
- }
- else {
- throw new IllegalArgumentException("eps must be specified");
- }
+ c = trainParams.getDoubleParameter("c", 0);
+ eps = trainParams.getDoubleParameter("eps", 0);
+ p = trainParams.getDoubleParameter("p", 0);
+ bias = trainParams.getIntParameter("bias", 0);
}
private static Problem constructProblem(List<Double> vy, List<Feature[]> vx, int maxIndex, double bias) {
@@ -126,7 +90,7 @@ public class LiblinearTrainer extends AbstractEventTrainer {
problem.y = new double[problem.l];
for (int i = 0; i < problem.l; i++) {
- problem.y[i] = vy.get(i).doubleValue();
+ problem.y[i] = vy.get(i);
}
return problem;
@@ -135,8 +99,8 @@ public class LiblinearTrainer extends AbstractEventTrainer {
@Override
public MaxentModel doTrain(DataIndexer indexer) throws IOException {
- List<Double> vy = new ArrayList<Double>();
- List<Feature[]> vx = new ArrayList<Feature[]>();
+ List<Double> vy = new ArrayList<>();
+ List<Feature[]> vx = new ArrayList<>();
// outcomes
int outcomes[] = indexer.getOutcomeList();
@@ -147,9 +111,9 @@ public class LiblinearTrainer extends AbstractEventTrainer {
for (int i = 0; i < indexer.getContexts().length; i++) {
int outcome = outcomes[i];
- vy.add(Double.valueOf(outcome));
+ vy.add((double) outcome);
- int features[] = indexer.getContexts()[i];
+ int[] features = indexer.getContexts()[i];
Feature[] x;
if (bias >= 0) {
@@ -160,7 +124,7 @@ public class LiblinearTrainer extends AbstractEventTrainer {
// for each feature ...
for (int fi = 0; fi < features.length; fi++) {
- // TODO: SHOUDL BE indexer.getNumTimesEventsSeen()[i] and not fi !!!
+ // TODO: SHOULD BE indexer.getNumTimesEventsSeen()[i] and not fi !!!
x[fi] = new FeatureNode(features[fi] + 1, indexer.getNumTimesEventsSeen()[i]);
}
@@ -176,9 +140,9 @@ public class LiblinearTrainer extends AbstractEventTrainer {
Model liblinearModel = Linear.train(problem, parameter);
- Map<String, Integer> predMap = new HashMap<String, Integer>();
+ Map<String, Integer> predMap = new HashMap<>();
- String predLabels[] = indexer.getPredLabels();
+ String[] predLabels = indexer.getPredLabels();
for (int i = 0; i < predLabels.length; i++) {
predMap.put(predLabels[i], i);
}
diff --git a/modelbuilder-addon/pom.xml b/modelbuilder-addon/pom.xml
index 4a9c886..b139d3b 100644
--- a/modelbuilder-addon/pom.xml
+++ b/modelbuilder-addon/pom.xml
@@ -1,35 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <parent>
+ <parent>
<groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp</artifactId>
- <version>1.6.0-SNAPSHOT</version>
- <relativePath>../opennlp/pom.xml</relativePath>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
</parent>
<artifactId>modelbuilder-addon</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <version>2.2.1-SNAPSHOT</version>
<packaging>jar</packaging>
- <name>modelbuilder-addon</name>
- <url>http://maven.apache.org</url>
-
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
+ <name>Apache OpenNLP ModelBuilder Addon</name>
<dependencies>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>3.8.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp-tools</artifactId>
- <version>1.6.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerArgument>-Xlint</compilerArgument>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/DefaultModelBuilderUtil.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/DefaultModelBuilderUtil.java
index 81ff9fd..b613877 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/DefaultModelBuilderUtil.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/DefaultModelBuilderUtil.java
@@ -24,17 +24,17 @@ import opennlp.addons.modelbuilder.impls.GenericModelGenerator;
import opennlp.addons.modelbuilder.impls.GenericModelableImpl;
/**
- *
- * Utilizes the filebased implementations to produce an NER model from user
- * The basic processing is such
- * read in the list of known entities
- * annotate the sentences based on the list of known entities
- * create a model from the annotations
- * perform NER with the model on the sentences
- * add the NER results to the annotations
- * rebuild the model
- * loop
- * defined data
+ * Utilizes the file-based implementations to produce an NER model from user defined data.
+ * <p>
+ * The basic processing is such read in the list of known entities
+ * <ul>
+ * <li>annotate the sentences based on the list of known entities,</li>
+ * <li>create a model from the annotations,</li>
+ * <li>perform NER with the model on the sentences,</li>
+ * <li>add the NER results to the annotations,</li>
+ * <li>rebuild the model,</li>
+ * <li>loop</li>
+ * </ul>
*/
public class DefaultModelBuilderUtil {
@@ -74,20 +74,23 @@ public class DefaultModelBuilderUtil {
params.setKnownEntitiesFile(knownEntities);
params.setModelFile(modelOutFile);
params.setKnownEntityBlacklist(knownEntitiesBlacklist);
- /**
+
+ /*
* sentence providers feed this process with user data derived sentences
* this impl just reads line by line through a file
*/
SentenceProvider sentenceProvider = new FileSentenceProvider();
sentenceProvider.setParameters(params);
- /**
+
+ /*
* KnownEntityProviders provide a seed list of known entities... such as
* Barack Obama for person, or Germany for location obviously these would
- * want to be prolific, non ambiguous names
+ * want to be prolific, non-ambiguous names
*/
KnownEntityProvider knownEntityProvider = new FileKnownEntityProvider();
knownEntityProvider.setParameters(params);
- /**
+
+ /*
* ModelGenerationValidators try to weed out bad hits by the iterations of
* the name finder. Since this is a recursive process, with each iteration
* the namefinder will get more and more greedy if bad entities are allowed
@@ -98,14 +101,15 @@ public class DefaultModelBuilderUtil {
*/
ModelGenerationValidator validator = new FileModelValidatorImpl();
validator.setParameters(params);
- /**
+
+ /*
* Modelable's write and read the annotated sentences, as well as create and
* write the NER models
*/
Modelable modelable = new GenericModelableImpl();
modelable.setParameters(params);
- /**
+ /*
* the modelGenerator actually runs the process with a set number of
* iterations... could be better by actually calculating the diff between
* runs and stopping based on a thresh, but for extrememly large sentence
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/KnownEntityProvider.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/KnownEntityProvider.java
index 694250e..5135c2b 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/KnownEntityProvider.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/KnownEntityProvider.java
@@ -15,6 +15,8 @@
*/
package opennlp.addons.modelbuilder;
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+
import java.util.Set;
@@ -23,23 +25,24 @@ import java.util.Set;
*
Supplies a list of known entities (a list of names or locations)
*/
-public interface KnownEntityProvider extends ModelParameter{
+public interface KnownEntityProvider extends ModelParameter<BaseModelBuilderParams> {
+
/**
- * returns a list of known non ambiguous entities.
- * @return a set of entities
- */
+ * returns a list of known non ambiguous entities.
+ * @return a set of entities
+ */
Set<String> getKnownEntities();
-/**
- * adds to the set of known entities. Overriding classes should hold this list in a class level set.
- * @param unambiguousEntity
- */
+
+ /**
+ * adds to the set of known entities. Overriding classes should hold this list in a class level set.
+ * @param unambiguousEntity
+ */
void addKnownEntity(String unambiguousEntity);
-/**
- * defines the type of entity that the set contains, ie person, location, organization.
- * @return
- */
- String getKnownEntitiesType();
-
-
+ /**
+ * defines the type of entity that the set contains, ie person, location, organization.
+ * @return
+ */
+ String getKnownEntitiesType();
+
}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelGenerationValidator.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelGenerationValidator.java
index 4bd5fe2..d05a1d5 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelGenerationValidator.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/ModelGenerationValidator.java
@@ -15,13 +15,15 @@
*/
package opennlp.addons.modelbuilder;
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
+
import java.util.Collection;
/**
*
Validates results from the iterative namefinding
*/
-public interface ModelGenerationValidator extends ModelParameter {
+public interface ModelGenerationValidator extends ModelParameter<BaseModelBuilderParams> {
Boolean validSentence(String sentence);
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/Modelable.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/Modelable.java
index 80b0170..8d2e06c 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/Modelable.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/Modelable.java
@@ -16,13 +16,14 @@
package opennlp.addons.modelbuilder;
import java.util.Set;
+
+import opennlp.addons.modelbuilder.impls.BaseModelBuilderParams;
import opennlp.tools.namefind.TokenNameFinderModel;
/**
*
*/
-public interface Modelable extends ModelParameter{
-
+public interface Modelable extends ModelParameter<BaseModelBuilderParams> {
String annotate(String sentence, String namedEntity, String entityType);
@@ -40,6 +41,5 @@ public interface Modelable extends ModelParameter{
TokenNameFinderModel getModel();
String[] tokenizeSentenceToWords(String sentence);
-
}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileKnownEntityProvider.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileKnownEntityProvider.java
index 0de043c..66c1c0e 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileKnownEntityProvider.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileKnownEntityProvider.java
@@ -33,7 +33,7 @@ import opennlp.addons.modelbuilder.KnownEntityProvider;
*/
public class FileKnownEntityProvider implements KnownEntityProvider {
- Set<String> knownEntities = new HashSet<String>();
+ Set<String> knownEntities = new HashSet<>();
BaseModelBuilderParams params;
@Override
public Set<String> getKnownEntities() {
@@ -53,8 +53,6 @@ public class FileKnownEntityProvider implements KnownEntityProvider {
br.close();
br = null;
fis = null;
- } catch (FileNotFoundException ex) {
- Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
} catch (IOException ex) {
Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileModelValidatorImpl.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileModelValidatorImpl.java
index ea4bb05..3951eef 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileModelValidatorImpl.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileModelValidatorImpl.java
@@ -17,16 +17,15 @@ package opennlp.addons.modelbuilder.impls;
import java.io.BufferedReader;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
import java.io.InputStreamReader;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
+
import opennlp.addons.modelbuilder.ModelGenerationValidator;
/**
@@ -34,7 +33,7 @@ import opennlp.addons.modelbuilder.ModelGenerationValidator;
*/
public class FileModelValidatorImpl implements ModelGenerationValidator {
- private Set<String> badentities = new HashSet<String>();
+ private final Set<String> badentities = new HashSet<>();
BaseModelBuilderParams params;
@Override
@@ -59,7 +58,7 @@ public class FileModelValidatorImpl implements ModelGenerationValidator {
// if (p.matcher(namedEntity).find()) {
// return false;
// }
- Boolean b = true;
+ boolean b = true;
if (badentities.contains(namedEntity.toLowerCase())) {
b = false;
}
@@ -72,21 +71,12 @@ public class FileModelValidatorImpl implements ModelGenerationValidator {
return badentities;
}
if (!badentities.isEmpty()) {
- try {
- InputStream fis;
- BufferedReader br;
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(
+ new FileInputStream(params.getKnownEntityBlacklist()), StandardCharsets.UTF_8))){
String line;
-
- fis = new FileInputStream(params.getKnownEntityBlacklist());
- br = new BufferedReader(new InputStreamReader(fis, Charset.forName("UTF-8")));
while ((line = br.readLine()) != null) {
badentities.add(line);
}
- br.close();
- br = null;
- fis = null;
- } catch (FileNotFoundException ex) {
- Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
} catch (IOException ex) {
Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileSentenceProvider.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileSentenceProvider.java
index bea55f5..9f1f5e1 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileSentenceProvider.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/FileSentenceProvider.java
@@ -17,15 +17,14 @@ package opennlp.addons.modelbuilder.impls;
import java.io.BufferedReader;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
import java.io.InputStreamReader;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
+
import opennlp.addons.modelbuilder.SentenceProvider;
/**
@@ -33,30 +32,18 @@ import opennlp.addons.modelbuilder.SentenceProvider;
*/
public class FileSentenceProvider implements SentenceProvider {
- BaseModelBuilderParams params ;
- Set<String> sentences = new HashSet<String>();
+ BaseModelBuilderParams params;
+ private final Set<String> sentences = new HashSet<>();
+ @Override
public Set<String> getSentences() {
if (sentences.isEmpty()) {
- try {
- InputStream fis;
- BufferedReader br;
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(
+ new FileInputStream(params.getSentenceFile()), StandardCharsets.UTF_8))){
String line;
-
- fis = new FileInputStream(params.getSentenceFile());
- br = new BufferedReader(new InputStreamReader(fis, Charset.forName("UTF-8")));
- int i=0;
while ((line = br.readLine()) != null) {
-
sentences.add(line);
}
-
- // Done with the file
- br.close();
- br = null;
- fis = null;
- } catch (FileNotFoundException ex) {
- Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
} catch (IOException ex) {
Logger.getLogger(FileKnownEntityProvider.class.getName()).log(Level.SEVERE, null, ex);
}
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelGenerator.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelGenerator.java
index bbd23e1..d09a664 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelGenerator.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelGenerator.java
@@ -31,7 +31,7 @@ import opennlp.tools.util.Span;
*/
public class GenericModelGenerator implements SemiSupervisedModelGenerator {
- private Map<String, String> params = new HashMap<String, String>();
+ private Map<String, String> params = new HashMap<>();
@Override
public void setParameters(BaseModelBuilderParams params) {
diff --git a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelableImpl.java b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelableImpl.java
index ccfddcb..8c1f754 100644
--- a/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelableImpl.java
+++ b/modelbuilder-addon/src/main/java/opennlp/addons/modelbuilder/impls/GenericModelableImpl.java
@@ -22,12 +22,14 @@ import java.io.FileWriter;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStream;
-import java.nio.charset.Charset;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import opennlp.addons.modelbuilder.Modelable;
+import opennlp.tools.namefind.TokenNameFinderFactory;
import opennlp.tools.util.MarkableFileInputStreamFactory;
import opennlp.tools.namefind.NameFinderME;
@@ -37,13 +39,14 @@ import opennlp.tools.namefind.TokenNameFinderModel;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
+import opennlp.tools.util.TrainingParameters;
/**
* Creates annotations, writes annotations to file, and creates a model and writes to a file
*/
public class GenericModelableImpl implements Modelable {
- private Set<String> annotatedSentences = new HashSet<String>();
+ private Set<String> annotatedSentences = new HashSet<>();
BaseModelBuilderParams params;
@Override
@@ -53,21 +56,15 @@ public class GenericModelableImpl implements Modelable {
@Override
public String annotate(String sentence, String namedEntity, String entityType) {
- String annotation = sentence.replace(namedEntity, " <START:" + entityType + "> " + namedEntity + " <END> ");
- return annotation;
+ return sentence.replace(namedEntity, " <START:" + entityType + "> " + namedEntity + " <END> ");
}
@Override
public void writeAnnotatedSentences() {
- try {
-
- FileWriter writer = new FileWriter(params.getAnnotatedTrainingDataFile(), false);
- BufferedWriter bw = new BufferedWriter(writer);
-
+ try (Writer bw = new BufferedWriter(new FileWriter(params.getAnnotatedTrainingDataFile(), false))) {
for (String s : annotatedSentences) {
bw.write(s.replace("\n", " ").trim() + "\n");
}
- bw.close();
} catch (IOException ex) {
ex.printStackTrace();
}
@@ -90,31 +87,24 @@ public class GenericModelableImpl implements Modelable {
@Override
public void buildModel(String entityType) {
- try {
+ try (ObjectStream<NameSample> sampleStream = new NameSampleDataStream(new PlainTextByLineStream(
+ new MarkableFileInputStreamFactory(params.getAnnotatedTrainingDataFile()), StandardCharsets.UTF_8));
+ OutputStream modelOut = new BufferedOutputStream(new FileOutputStream(params.getModelFile()))) {
System.out.println("\tBuilding Model using " + annotatedSentences.size() + " annotations");
System.out.println("\t\treading training data...");
- Charset charset = Charset.forName("UTF-8");
- ObjectStream<String> lineStream =
- new PlainTextByLineStream(new MarkableFileInputStreamFactory(params.getAnnotatedTrainingDataFile()), charset);
- ObjectStream<NameSample> sampleStream = new NameSampleDataStream(lineStream);
-
TokenNameFinderModel model;
- model = NameFinderME.train("en", entityType, sampleStream, null);
+ model = NameFinderME.train("en", entityType, sampleStream,
+ TrainingParameters.defaultParams(), new TokenNameFinderFactory());
sampleStream.close();
- OutputStream modelOut = new BufferedOutputStream(new FileOutputStream(params.getModelFile()));
model.serialize(modelOut);
- if (modelOut != null) {
- modelOut.close();
- }
System.out.println("\tmodel generated");
} catch (Exception e) {
+ e.printStackTrace();
}
}
@Override
public TokenNameFinderModel getModel() {
-
-
TokenNameFinderModel nerModel = null;
try {
nerModel = new TokenNameFinderModel(new FileInputStream(params.getModelFile()));
@@ -126,7 +116,6 @@ public class GenericModelableImpl implements Modelable {
@Override
public String[] tokenizeSentenceToWords(String sentence) {
- return sentence.split(" ");
-
+ return sentence.split("\\s+");
}
}
diff --git a/modelbuilder-addon/src/test/java/modelbuilder/AppTest.java b/modelbuilder-addon/src/test/java/modelbuilder/AppTest.java
deleted file mode 100644
index 2b04731..0000000
--- a/modelbuilder-addon/src/test/java/modelbuilder/AppTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package modelbuilder;
-
-import junit.framework.Test;
-import junit.framework.TestCase;
-import junit.framework.TestSuite;
-
-/**
- * Unit test for simple App.
- */
-public class AppTest
- extends TestCase
-{
- /**
- * Create the test case
- *
- * @param testName name of the test case
- */
- public AppTest( String testName )
- {
- super( testName );
- }
-
- /**
- * @return the suite of tests being tested
- */
- public static Test suite()
- {
- return new TestSuite( AppTest.class );
- }
-
- /**
- * Rigourous Test :-)
- */
- public void testApp()
- {
- assertTrue( true );
- }
-}
diff --git a/morfologik-addon/pom.xml b/morfologik-addon/pom.xml
index f6eef6a..92b6f72 100644
--- a/morfologik-addon/pom.xml
+++ b/morfologik-addon/pom.xml
@@ -1,23 +1,98 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ </parent>
- <groupId>org.apache.opennlp</groupId>
<artifactId>morfologik-addon</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <version>2.2.1-SNAPSHOT</version>
<packaging>jar</packaging>
- <name>Morfologik Addon</name>
+ <name>Apache OpenNLP Morfologik Addon</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-tools</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.carrot2</groupId>
+ <artifactId>morfologik-stemming</artifactId>
+ <version>2.1.9</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.carrot2</groupId>
+ <artifactId>morfologik-tools</artifactId>
+ <version>2.1.9</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </dependency>
+ </dependencies>
- <url>http://maven.apache.org</url>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>2.3.2</version>
<configuration>
- <source>1.7</source>
- <target>1.7</target>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerArgument>-Xlint</compilerArgument>
</configuration>
</plugin>
<plugin>
@@ -38,72 +113,40 @@
many file have more than 100 chars.
Right now only javadoc files are too long.
-->
- <tarLongFileMode>gnu</tarLongFileMode>
-
- <finalName>apache-opennlp-morfologik-addon-${project.version}</finalName>
+ <tarLongFileMode>gnu</tarLongFileMode>
+
+ <finalName>apache-opennlp-morfologik-addon-${project.version}</finalName>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.6</version>
+ <executions>
+ <execution>
+ <id>generate checksums for binary artifacts</id>
+ <goals><goal>run</goal></goals>
+ <phase>verify</phase>
+ <configuration>
+ <target>
+ <checksum algorithm="sha1" format="MD5SUM">
+ <fileset dir="${project.build.directory}">
+ <include name="*.zip" />
+ <include name="*.gz" />
+ </fileset>
+ </checksum>
+ <checksum algorithm="md5" format="MD5SUM">
+ <fileset dir="${project.build.directory}">
+ <include name="*.zip" />
+ <include name="*.gz" />
+ </fileset>
+ </checksum>
+ </target>
</configuration>
</execution>
</executions>
</plugin>
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.6</version>
- <executions>
- <execution>
- <id>generate checksums for binary artifacts</id>
- <goals><goal>run</goal></goals>
- <phase>verify</phase>
- <configuration>
- <target>
- <checksum algorithm="sha1" format="MD5SUM">
- <fileset dir="${project.build.directory}">
- <include name="*.zip" />
- <include name="*.gz" />
- </fileset>
- </checksum>
- <checksum algorithm="md5" format="MD5SUM">
- <fileset dir="${project.build.directory}">
- <include name="*.zip" />
- <include name="*.gz" />
- </fileset>
- </checksum>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.carrot2</groupId>
- <artifactId>morfologik-stemming</artifactId>
- <version>2.1.0</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.carrot2</groupId>
- <artifactId>morfologik-tools</artifactId>
- <version>2.1.0</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.opennlp</groupId>
- <artifactId>opennlp-tools</artifactId>
- <version>1.6.0</version>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.13.1</version>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
</project>
diff --git a/morfologik-addon/src/main/java/opennlp/morfologik/cmdline/CLI.java b/morfologik-addon/src/main/java/opennlp/morfologik/cmdline/CLI.java
index f92d178..5205739 100644
--- a/morfologik-addon/src/main/java/opennlp/morfologik/cmdline/CLI.java
+++ b/morfologik-addon/src/main/java/opennlp/morfologik/cmdline/CLI.java
@@ -40,9 +40,9 @@ public final class CLI {
private static Map<String, CmdLineTool> toolLookupMap;
static {
- toolLookupMap = new LinkedHashMap<String, CmdLineTool>();
+ toolLookupMap = new LinkedHashMap<>();
- List<CmdLineTool> tools = new LinkedList<CmdLineTool>();
+ List<CmdLineTool> tools = new LinkedList<>();
tools.add(new MorfologikDictionaryBuilderTool());
tools.add(new XMLDictionaryToTableTool());
diff --git a/morfologik-addon/src/main/java/opennlp/morfologik/cmdline/builder/XMLDictionaryToTableTool.java b/morfologik-addon/src/main/java/opennlp/morfologik/cmdline/builder/XMLDictionaryToTableTool.java
index 0e7f2d5..edc41c7 100644
--- a/morfologik-addon/src/main/java/opennlp/morfologik/cmdline/builder/XMLDictionaryToTableTool.java
+++ b/morfologik-addon/src/main/java/opennlp/morfologik/cmdline/builder/XMLDictionaryToTableTool.java
@@ -59,7 +59,7 @@ public class XMLDictionaryToTableTool extends BasicCmdLineTool {
CmdLineUtil.checkInputFile("dictionary input file", dictInFile);
CmdLineUtil.checkOutputFile("dictionary output file", dictOutFile);
- POSDictionary tagDictionary = null;
+ POSDictionary tagDictionary;
try {
tagDictionary = POSDictionary.create(new FileInputStream(dictInFile));
} catch (IOException e) {
@@ -117,7 +117,7 @@ public class XMLDictionaryToTableTool extends BasicCmdLineTool {
private String createEntry(String word, String tag) {
- String entry = "" + SEPARATOR +// base
+ String entry = SEPARATOR +// base
word + SEPARATOR +
tag;
diff --git a/morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java b/morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java
index 2090ce5..650f7a6 100644
--- a/morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java
+++ b/morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java
@@ -30,24 +30,22 @@ import morfologik.stemming.Dictionary;
import morfologik.stemming.DictionaryLookup;
import morfologik.stemming.IStemmer;
import morfologik.stemming.WordData;
-import opennlp.tools.lemmatizer.DictionaryLemmatizer;
+import opennlp.tools.lemmatizer.Lemmatizer;
-public class MorfologikLemmatizer implements DictionaryLemmatizer {
+public class MorfologikLemmatizer implements Lemmatizer {
private IStemmer dictLookup;
- public final Set<String> constantTags = new HashSet<String>(Arrays.asList(
- "NNP", "NP00000"));
+ public final Set<String> constantTags = new HashSet<>(Arrays.asList("NNP", "NP00000"));
- public MorfologikLemmatizer(Path dictionaryPath) throws IllegalArgumentException,
- IOException {
+ public MorfologikLemmatizer(Path dictionaryPath) throws IllegalArgumentException, IOException {
dictLookup = new DictionaryLookup(Dictionary.read(dictionaryPath));
}
private HashMap<List<String>, String> getLemmaTagsDict(String word) {
List<WordData> wdList = dictLookup.lookup(word);
- HashMap<List<String>, String> dictMap = new HashMap<List<String>, String>();
+ HashMap<List<String>, String> dictMap = new HashMap<>();
for (WordData wd : wdList) {
- List<String> wordLemmaTags = new ArrayList<String>();
+ List<String> wordLemmaTags = new ArrayList<>();
wordLemmaTags.add(word);
wordLemmaTags.add(wd.getTag().toString());
dictMap.put(wordLemmaTags, wd.getStem().toString());
@@ -56,7 +54,7 @@ public class MorfologikLemmatizer implements DictionaryLemmatizer {
}
private List<String> getDictKeys(String word, String postag) {
- List<String> keys = new ArrayList<String>();
+ List<String> keys = new ArrayList<>();
if (constantTags.contains(postag)) {
keys.addAll(Arrays.asList(word, postag));
} else {
@@ -66,7 +64,7 @@ public class MorfologikLemmatizer implements DictionaryLemmatizer {
}
private HashMap<List<String>, String> getDictMap(String word, String postag) {
- HashMap<List<String>, String> dictMap = new HashMap<List<String>, String>();
+ HashMap<List<String>, String> dictMap;
if (constantTags.contains(postag)) {
dictMap = this.getLemmaTagsDict(word);
@@ -77,7 +75,7 @@ public class MorfologikLemmatizer implements DictionaryLemmatizer {
}
public String lemmatize(String word, String postag) {
- String lemma = null;
+ String lemma;
List<String> keys = this.getDictKeys(word, postag);
HashMap<List<String>, String> dictMap = this.getDictMap(word, postag);
// lookup lemma as value of the map
@@ -86,11 +84,25 @@ public class MorfologikLemmatizer implements DictionaryLemmatizer {
lemma = keyValue;
} else if (keyValue == null && constantTags.contains(postag)) {
lemma = word;
- } else if (keyValue == null && word.toUpperCase() == word) {
+ } else if (keyValue == null && word.toUpperCase().equals(word)) {
lemma = word;
} else {
lemma = word.toLowerCase();
}
return lemma;
}
+
+ @Override
+ public String[] lemmatize(final String[] tokens, final String[] postags) {
+ List<String> lemmas = new ArrayList<>();
+ for (int i = 0; i < tokens.length; i++) {
+ lemmas.add(this.lemmatize(tokens[i], postags[i]));
+ }
+ return lemmas.toArray(new String[0]);
+ }
+
+ @Override
+ public List<List<String>> lemmatize(final List<String> tokens, final List<String> posTags) {
+ throw new UnsupportedOperationException("Method not implemented here!");
+ }
}
diff --git a/morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikPOSTaggerFactory.java b/morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikPOSTaggerFactory.java
index 93d6c61..c3501cf 100644
--- a/morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikPOSTaggerFactory.java
+++ b/morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikPOSTaggerFactory.java
@@ -28,10 +28,8 @@ import java.nio.file.Path;
import java.util.Map;
import morfologik.stemming.DictionaryMetadata;
-import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.postag.POSTaggerFactory;
import opennlp.tools.postag.TagDictionary;
-import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.model.ArtifactSerializer;
import opennlp.tools.util.model.ModelUtil;
@@ -52,9 +50,9 @@ public class MorfologikPOSTaggerFactory extends POSTaggerFactory {
public MorfologikPOSTaggerFactory() {
}
-
- public TagDictionary createTagDictionary(File dictionary)
- throws InvalidFormatException, FileNotFoundException, IOException {
+
+ @Override
+ public TagDictionary createTagDictionary(File dictionary) throws IOException {
if(!dictionary.canRead()) {
throw new FileNotFoundException("Could not read dictionary: " + dictionary.getAbsolutePath());
@@ -72,13 +70,6 @@ public class MorfologikPOSTaggerFactory extends POSTaggerFactory {
return createMorfologikDictionary(dictData, dictInfo);
}
-
-
- @Override
- protected void init(Dictionary ngramDictionary, TagDictionary posDictionary) {
- super.init(ngramDictionary, null);
- this.dict = posDictionary;
- }
@Override
public TagDictionary getTagDictionary() {
@@ -87,10 +78,8 @@ public class MorfologikPOSTaggerFactory extends POSTaggerFactory {
if (artifactProvider != null) {
Object obj = artifactProvider.getArtifact(MORFOLOGIK_POSDICT);
if (obj != null) {
- byte[] data = (byte[]) artifactProvider
- .getArtifact(MORFOLOGIK_POSDICT);
- byte[] info = (byte[]) artifactProvider
- .getArtifact(MORFOLOGIK_DICT_INFO);
+ byte[] data = artifactProvider.getArtifact(MORFOLOGIK_POSDICT);
+ byte[] info = artifactProvider.getArtifact(MORFOLOGIK_DICT_INFO);
try {
this.dict = createMorfologikDictionary(data, info);
@@ -120,8 +109,7 @@ public class MorfologikPOSTaggerFactory extends POSTaggerFactory {
}
@Override
- public TagDictionary createTagDictionary(InputStream in)
- throws InvalidFormatException, IOException {
+ public TagDictionary createTagDictionary(InputStream in) throws IOException {
throw new UnsupportedOperationException(
"Morfologik POS Tagger factory does not support this operation");
}
@@ -129,8 +117,7 @@ public class MorfologikPOSTaggerFactory extends POSTaggerFactory {
@Override
@SuppressWarnings("rawtypes")
public Map<String, ArtifactSerializer> createArtifactSerializersMap() {
- Map<String, ArtifactSerializer> serializers = super
- .createArtifactSerializersMap();
+ Map<String, ArtifactSerializer> serializers = super.createArtifactSerializersMap();
serializers.put(MORFOLOGIK_POSDICT_SUF, new ByteArraySerializer());
serializers.put(MORFOLOGIK_DICT_INFO_SUF, new ByteArraySerializer());
@@ -149,19 +136,18 @@ public class MorfologikPOSTaggerFactory extends POSTaggerFactory {
private TagDictionary createMorfologikDictionary(byte[] data, byte[] info)
throws IOException {
morfologik.stemming.Dictionary dict = morfologik.stemming.Dictionary
- .read(new ByteArrayInputStream(data), new ByteArrayInputStream(
- info));
+ .read(new ByteArrayInputStream(data), new ByteArrayInputStream(info));
return new MorfologikTagDictionary(dict);
}
static class ByteArraySerializer implements ArtifactSerializer<byte[]> {
- public byte[] create(InputStream in) throws IOException,
- InvalidFormatException {
-
+ @Override
+ public byte[] create(InputStream in) throws IOException {
return ModelUtil.read(in);
}
+ @Override
public void serialize(byte[] artifact, OutputStream out) throws IOException {
out.write(artifact);
}
diff --git a/morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikTagDictionary.java b/morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikTagDictionary.java
index b34ca2b..5b6bf13 100644
--- a/morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikTagDictionary.java
+++ b/morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikTagDictionary.java
@@ -28,13 +28,12 @@ import morfologik.stemming.WordData;
import opennlp.tools.postag.TagDictionary;
/**
- * A POS Tagger dictionary implementation based on Morfologik binary
- * dictionaries
+ * A POS Tagger dictionary implementation based on Morfologik binary dictionaries.
*/
public class MorfologikTagDictionary implements TagDictionary {
- private IStemmer dictLookup;
- private boolean isCaseSensitive;
+ private final IStemmer dictLookup;
+ private final boolean isCaseSensitive;
/**
* Creates a case sensitive {@link MorfologikTagDictionary}
@@ -46,8 +45,7 @@ public class MorfologikTagDictionary implements TagDictionary {
* @throws IOException
* could not read dictionary from dictURL
*/
- public MorfologikTagDictionary(Dictionary dict)
- throws IllegalArgumentException, IOException {
+ public MorfologikTagDictionary(Dictionary dict) throws IllegalArgumentException {
this(dict, true);
}
@@ -63,8 +61,7 @@ public class MorfologikTagDictionary implements TagDictionary {
* @throws IOException
* could not read dictionary from dictURL
*/
- public MorfologikTagDictionary(Dictionary dict, boolean caseSensitive)
- throws IllegalArgumentException, IOException {
+ public MorfologikTagDictionary(Dictionary dict, boolean caseSensitive) throws IllegalArgumentException {
this.dictLookup = new DictionaryLookup(dict);
this.isCaseSensitive = caseSensitive;
}
@@ -77,14 +74,19 @@ public class MorfologikTagDictionary implements TagDictionary {
List<WordData> data = dictLookup.lookup(word);
if (data != null && data.size() > 0) {
- List<String> tags = new ArrayList<String>(data.size());
- for (int i = 0; i < data.size(); i++) {
- tags.add(data.get(i).getTag().toString());
+ List<String> tags = new ArrayList<>(data.size());
+ for (WordData datum : data) {
+ tags.add(datum.getTag().toString());
}
if (tags.size() > 0)
- return tags.toArray(new String[tags.size()]);
+ return tags.toArray(new String[0]);
return null;
}
return null;
}
+
+ @Override
+ public boolean isCaseSensitive() {
+ return isCaseSensitive;
+ }
}
diff --git a/morfologik-addon/src/test/java/opennlp/morfologik/builder/POSDictionayBuilderTest.java b/morfologik-addon/src/test/java/opennlp/morfologik/builder/POSDictionayBuilderTest.java
index 0a7ba48..ae2bb46 100644
--- a/morfologik-addon/src/test/java/opennlp/morfologik/builder/POSDictionayBuilderTest.java
+++ b/morfologik-addon/src/test/java/opennlp/morfologik/builder/POSDictionayBuilderTest.java
@@ -22,19 +22,19 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
-import junit.framework.TestCase;
import morfologik.stemming.DictionaryMetadata;
import opennlp.morfologik.lemmatizer.MorfologikLemmatizer;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
-public class POSDictionayBuilderTest extends TestCase {
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class POSDictionayBuilderTest {
@Test
public void testBuildDictionary() throws Exception {
Path output = createMorfologikDictionary();
-
MorfologikLemmatizer ml = new MorfologikLemmatizer(output);
assertNotNull(ml);
diff --git a/morfologik-addon/src/test/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizerTest.java b/morfologik-addon/src/test/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizerTest.java
index 6b7525e..f1212cc 100644
--- a/morfologik-addon/src/test/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizerTest.java
+++ b/morfologik-addon/src/test/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizerTest.java
@@ -1,35 +1,45 @@
-package opennlp.morfologik.lemmatizer;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import static org.junit.Assert.assertEquals;
+package opennlp.morfologik.lemmatizer;
import java.nio.file.Path;
import opennlp.morfologik.builder.POSDictionayBuilderTest;
-import opennlp.tools.lemmatizer.DictionaryLemmatizer;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
public class MorfologikLemmatizerTest {
@Test
public void testLemmatizeInsensitive() throws Exception {
- DictionaryLemmatizer dict = createDictionary(false);
+ MorfologikLemmatizer dict = createDictionary(false);
assertEquals("casar", dict.lemmatize("casa", "V"));
assertEquals("casa", dict.lemmatize("casa", "NOUN"));
-
assertEquals("casa", dict.lemmatize("Casa", "PROP"));
}
- private MorfologikLemmatizer createDictionary(boolean caseSensitive)
- throws Exception {
-
+ private MorfologikLemmatizer createDictionary(boolean caseSensitive) throws Exception {
Path output = POSDictionayBuilderTest.createMorfologikDictionary();
-
- MorfologikLemmatizer ml = new MorfologikLemmatizer(output);
-
- return ml;
+ return new MorfologikLemmatizer(output);
}
}
diff --git a/morfologik-addon/src/test/java/opennlp/morfologik/tagdict/MorfologikTagDictionaryTest.java b/morfologik-addon/src/test/java/opennlp/morfologik/tagdict/MorfologikTagDictionaryTest.java
index c6c9e04..d6bc2fe 100644
--- a/morfologik-addon/src/test/java/opennlp/morfologik/tagdict/MorfologikTagDictionaryTest.java
+++ b/morfologik-addon/src/test/java/opennlp/morfologik/tagdict/MorfologikTagDictionaryTest.java
@@ -1,7 +1,21 @@
-package opennlp.morfologik.tagdict;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+package opennlp.morfologik.tagdict;
import java.util.Arrays;
import java.util.List;
@@ -10,7 +24,10 @@ import morfologik.stemming.Dictionary;
import opennlp.morfologik.builder.POSDictionayBuilderTest;
import opennlp.tools.postag.TagDictionary;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
public class MorfologikTagDictionaryTest {
@@ -21,7 +38,6 @@ public class MorfologikTagDictionaryTest {
List<String> tags = Arrays.asList(dict.getTags("carro"));
assertEquals(1, tags.size());
assertTrue(tags.contains("NOUN"));
-
}
@Test
@@ -66,13 +82,10 @@ public class MorfologikTagDictionaryTest {
return this.createDictionary(caseSensitive, null);
}
- private MorfologikTagDictionary createDictionary(boolean caseSensitive,
- List<String> constant) throws Exception {
+ private MorfologikTagDictionary createDictionary(boolean caseSensitive, List<String> constant) throws Exception {
Dictionary dic = Dictionary.read(POSDictionayBuilderTest.createMorfologikDictionary());
- MorfologikTagDictionary ml = new MorfologikTagDictionary(dic, caseSensitive);
-
- return ml;
+ return new MorfologikTagDictionary(dic, caseSensitive);
}
}
diff --git a/morfologik-addon/src/test/java/opennlp/morfologik/tagdict/POSTaggerFactoryTest.java b/morfologik-addon/src/test/java/opennlp/morfologik/tagdict/POSTaggerFactoryTest.java
index 7341a02..602ffc6 100644
--- a/morfologik-addon/src/test/java/opennlp/morfologik/tagdict/POSTaggerFactoryTest.java
+++ b/morfologik-addon/src/test/java/opennlp/morfologik/tagdict/POSTaggerFactoryTest.java
@@ -17,13 +17,11 @@
package opennlp.morfologik.tagdict;
-import static org.junit.Assert.*;
-
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import opennlp.morfologik.builder.POSDictionayBuilderTest;
@@ -33,31 +31,22 @@ import opennlp.tools.postag.POSTaggerFactory;
import opennlp.tools.postag.POSTaggerME;
import opennlp.tools.postag.TagDictionary;
import opennlp.tools.postag.WordTagSampleStream;
+import opennlp.tools.util.MarkableFileInputStreamFactory;
import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.model.ModelType;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* Tests for the {@link POSTaggerFactory} class.
*/
public class POSTaggerFactoryTest {
- private static ObjectStream<POSSample> createSampleStream()
- throws IOException {
- InputStream in = POSTaggerFactoryTest.class.getClassLoader()
- .getResourceAsStream("AnnotatedSentences.txt");
-
- return new WordTagSampleStream((new InputStreamReader(in)));
- }
-
- static POSModel trainPOSModel(ModelType type, POSTaggerFactory factory)
- throws IOException {
- return POSTaggerME.train("en", createSampleStream(),
- TrainingParameters.defaultParams(), factory);
- }
-
@Test
public void testPOSTaggerWithCustomFactory() throws Exception {
@@ -71,18 +60,28 @@ public class POSTaggerFactoryTest {
POSTaggerFactory factory = posModel.getFactory();
assertTrue(factory.getTagDictionary() instanceof MorfologikTagDictionary);
- factory = null;
-
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- posModel.serialize(out);
- ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+ try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
- POSModel fromSerialized = new POSModel(in);
+ posModel.serialize(out);
+ POSModel fromSerialized = new POSModel(new ByteArrayInputStream(out.toByteArray()));
- factory = fromSerialized.getFactory();
- assertTrue(factory.getTagDictionary() instanceof MorfologikTagDictionary);
-
- assertEquals(2, factory.getTagDictionary().getTags("casa").length);
+ factory = fromSerialized.getFactory();
+ assertTrue(factory.getTagDictionary() instanceof MorfologikTagDictionary);
+
+ assertEquals(2, factory.getTagDictionary().getTags("casa").length);
+ }
+ }
+
+ private static ObjectStream<POSSample> createSampleStream() throws IOException {
+ File data = new File("target/test-classes/AnnotatedSentences.txt");
+ return new WordTagSampleStream(new PlainTextByLineStream(
+ new MarkableFileInputStreamFactory(data), StandardCharsets.UTF_8));
+ }
+
+ static POSModel trainPOSModel(ModelType type, POSTaggerFactory factory)
+ throws IOException {
+ return POSTaggerME.train("en", createSampleStream(),
+ TrainingParameters.defaultParams(), factory);
}
}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..045d0cf
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,532 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache</groupId>
+ <artifactId>apache</artifactId>
+ <version>29</version>
+ <relativePath />
+ </parent>
+
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-addons</artifactId>
+ <version>2.2.1-SNAPSHOT</version>
+ <packaging>pom</packaging>
+
+ <name>Apache OpenNLP Addons</name>
+
+ <scm>
+ <connection>scm:git:https://github.com/apache/opennlp-addons.git</connection>
+ <developerConnection>scm:git:git@github.com:apache/opennlp-addons.git</developerConnection>
+ <url>https://github.com/apache/opennlp-addons.git</url>
+ <tag>HEAD</tag>
+ </scm>
+
+ <repositories>
+ <repository>
+ <id>apache.snapshots</id>
+ <name>Apache Snapshot Repository</name>
+ <url>https://repository.apache.org/snapshots</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+
+ <mailingLists>
+ <mailingList>
+ <name>Apache OpenNLP Users</name>
+ <subscribe>users-subscribe@opennlp.apache.org</subscribe>
+ <unsubscribe>users-unsubscribe@opennlp.apache.org</unsubscribe>
+ <post>users@opennlp.apache.org</post>
+ <archive>http://mail-archives.apache.org/mod_mbox/opennlp-users/</archive>
+ </mailingList>
+
+ <mailingList>
+ <name>Apache OpenNLP Developers</name>
+ <subscribe>dev-subscribe@opennlp.apache.org</subscribe>
+ <unsubscribe>dev-unsubscribe@opennlp.apache.org</unsubscribe>
+ <post>dev@opennlp.apache.org</post>
+ <archive>http://mail-archives.apache.org/mod_mbox/opennlp-dev/</archive>
+ </mailingList>
+
+ <mailingList>
+ <name>Apache OpenNLP Commits</name>
+ <subscribe>commits-subscribe@opennlp.apache.org</subscribe>
+ <unsubscribe>commits-unsubscribe@opennlp.apache.org</unsubscribe>
+ <archive>http://mail-archives.apache.org/mod_mbox/opennlp-commits/</archive>
+ </mailingList>
+
+ <mailingList>
+ <name>Apache OpenNLP Issues</name>
+ <subscribe>issues-subscribe@opennlp.apache.org</subscribe>
+ <unsubscribe>issues-unsubscribe@opennlp.apache.org</unsubscribe>
+ <archive>http://mail-archives.apache.org/mod_mbox/opennlp-issues/</archive>
+ </mailingList>
+ </mailingLists>
+
+ <issueManagement>
+ <system>jira</system>
+ <url>https://issues.apache.org/jira/browse/OPENNLP</url>
+ </issueManagement>
+
+ <modules>
+ <module>geoentitylinker-addon</module>
+ <module>japanese-addon</module>
+ <module>jwnl-addon</module>
+ <module>liblinear-addon</module>
+ <module>modelbuilder-addon</module>
+ <module>morfologik-addon</module>
+ </modules>
+
+ <properties>
+ <!-- Build Properties -->
+ <java.version>11</java.version>
+ <maven.version>3.3.9</maven.version>
+ <maven.compiler.source>${java.version}</maven.compiler.source>
+ <maven.compiler.target>${java.version}</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+
+ <opennlp.tools.version>2.2.0</opennlp.tools.version>
+ <opennlp.forkCount>1.0C</opennlp.forkCount>
+
+ <slf4j.version>1.7.36</slf4j.version>
+ <log4j2.version>2.20.0</log4j2.version>
+
+ <junit.version>5.9.2</junit.version>
+
+ <enforcer.plugin.version>3.0.0-M3</enforcer.plugin.version>
+ <checkstyle.plugin.version>3.2.0</checkstyle.plugin.version>
+ <coveralls.maven.plugin>4.3.0</coveralls.maven.plugin>
+ <jacoco.maven.plugin>0.7.9</jacoco.maven.plugin>
+ <maven.surefire.plugin>3.0.0-M5</maven.surefire.plugin>
+ <maven.failsafe.plugin>3.0.0-M5</maven.failsafe.plugin>
+ <mockito.version>3.9.0</mockito.version>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <artifactId>opennlp-tools</artifactId>
+ <groupId>${project.groupId}</groupId>
+ <version>${opennlp.tools.version}</version>
+ </dependency>
+
+ <dependency>
+ <artifactId>opennlp-tools</artifactId>
+ <groupId>${project.groupId}</groupId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.12.0</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.15</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>1.1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ <version>3.2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.6.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <version>1.9.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>${log4j2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${log4j2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j2.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+ </dependencyManagement>
+
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-release-plugin</artifactId>
+ <configuration>
+ <useReleaseProfile>false</useReleaseProfile>
+ <goals>deploy</goals>
+ <arguments>-Papache-release</arguments>
+ <mavenExecutorId>forked-path</mavenExecutorId>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>3.2.0</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <version>5.1.4</version>
+ </plugin>
+ <!--
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>${checkstyle.plugin.version}</version>
+ <dependencies>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>10.6.0</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>validate</id>
+ <phase>validate</phase>
+ <configuration>
+ <configLocation>checkstyle.xml</configLocation>
+ <consoleOutput>true</consoleOutput>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <testSourceDirectories>${project.basedir}/src/test/java</testSourceDirectories>
+ <violationSeverity>error</violationSeverity>
+ <failOnViolation>true</failOnViolation>
+ </configuration>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ -->
+
+ <!-- Coverage analysis for tests -->
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <version>${jacoco.maven.plugin}</version>
+ <configuration>
+ <excludes>
+ <exclude>**/stemmer/*</exclude>
+ <exclude>**/stemmer/snowball/*</exclude>
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <id>jacoco-prepare-agent</id>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>jacoco-prepare-agent-integration</id>
+ <goals>
+ <goal>prepare-agent-integration</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>jacoco-report</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>report</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${maven.surefire.plugin}</version>
+ <configuration>
+ <argLine>-Xmx2048m -Dfile.encoding=UTF-8</argLine>
+ <forkCount>${opennlp.forkCount}</forkCount>
+ <failIfNoSpecifiedTests>false</failIfNoSpecifiedTests>
+ <excludes>
+ <exclude>**/*IT.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>${maven.failsafe.plugin}</version>
+ <executions>
+ <execution>
+ <id>integration-test</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*Test.java</exclude>
+ </excludes>
+ <includes>
+ <include>**/*IT.java</include>
+ </includes>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>de.thetaphi</groupId>
+ <artifactId>forbiddenapis</artifactId>
+ <version>3.5.1</version>
+ <configuration>
+ <failOnUnsupportedJava>false</failOnUnsupportedJava>
+ <bundledSignatures>
+ <bundledSignature>jdk-deprecated</bundledSignature>
+ <bundledSignature>jdk-non-portable</bundledSignature>
+ <bundledSignature>jdk-internal</bundledSignature>
+ <!-- don't allow unsafe reflective access: -->
+ <bundledSignature>jdk-reflection</bundledSignature>
+ </bundledSignatures>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>validate</phase>
+ <goals>
+ <goal>check</goal>
+ <goal>testCheck</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.10.1</version>
+ <configuration>
+ <release>${java.version}</release>
+ <compilerArgument>-Xlint</compilerArgument>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-cli</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ <phase>verify</phase>
+ <configuration>
+ <excludes>
+ <exclude>release.properties</exclude>
+ <exclude>.gitattributes</exclude>
+ <exclude>.github/*</exclude>
+ <exclude>**/*.md</exclude>
+ <!-- We do not ship files from test/resources, so ok to exclude -->
+ <exclude>**/src/test/resources/**/*.txt</exclude>
+ <exclude>**/src/test/resources/**/*.csv</exclude>
+ <exclude>**/src/test/resources/**/*.cxt</exclude>
+ <exclude>**/src/test/resources/**/*.xml</exclude>
+ <exclude>**/src/test/resources/**/*.info</exclude>
+ <!-- These files do not allow a license header -->
+ <exclude>**/src/main/readme/MORFOLOGIK-LICENSE</exclude>
+ </excludes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>3.5.0</version>
+ <configuration>
+ <doclint>none</doclint>
+ <source>${java.version}</source>
+ <sourcepath>src/main/java</sourcepath>
+ </configuration>
+ <executions>
+ <execution>
+ <id>create-javadoc-jar</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <show>public</show>
+ <quiet>false</quiet>
+ <use>false</use> <!-- Speeds up the build of the javadocs -->
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-source-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-source-jar</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <phase>package</phase>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.10</version>
+ <configuration>
+ <workspace>../</workspace>
+ <workspaceCodeStylesURL>http://opennlp.apache.org/code-formatter/OpenNLP-Eclipse-Formatter.xml</workspaceCodeStylesURL>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>${enforcer.plugin.version}</version>
+ <executions>
+ <execution>
+ <id>enforce-java</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireJavaVersion>
+ <message>Java 11 or higher is required to compile this module</message>
+ <version>[${java.version},)</version>
+ </requireJavaVersion>
+ <requireMavenVersion>
+ <message>Maven 3.3.9 or higher is required to compile this module</message>
+ <version>[${maven.version},)</version>
+ </requireMavenVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>de.thetaphi</groupId>
+ <artifactId>forbiddenapis</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>${checkstyle.plugin.version}</version>
+ </plugin>
+ </plugins>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>jacoco</id>
+ <properties>
+ <opennlp.forkCount>1</opennlp.forkCount>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ </profiles>
+
+</project>
\ No newline at end of file