You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by ar...@apache.org on 2020/08/03 19:46:35 UTC
[incubator-nlpcraft] branch NLPCRAFT-98 updated: Code review.
This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a commit to branch NLPCRAFT-98
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-98 by this push:
new f855d49 Code review.
f855d49 is described below
commit f855d4909c9731c37851b2866d6bae8c1250cd74
Author: Aaron Radzinzski <ar...@datalingvo.com>
AuthorDate: Mon Aug 3 12:46:24 2020 -0700
Code review.
---
external/readme.txt | 2 +-
nlpcraft/src/main/resources/nlpcraft.conf | 19 ++++++------
.../NCExternalConfigHolder.scala} | 14 ++++++---
.../NCExternalConfigManager.scala} | 36 +++++++++++-----------
.../NCExternalConfigType.scala} | 8 +++--
.../nlp/core/opennlp/NCOpenNlpTokenizer.scala | 6 ++--
.../org/apache/nlpcraft/probe/NCProbeBoot.scala | 6 ++--
.../org/apache/nlpcraft/server/NCServer.scala | 6 ++--
.../apache/nlpcraft/server/geo/NCGeoManager.scala | 24 +++++++--------
.../nlp/core/opennlp/NCOpenNlpNerEnricher.scala | 6 ++--
.../server/nlp/core/opennlp/NCOpenNlpParser.scala | 8 ++---
.../server/nlp/enrichers/geo/NCGeoEnricher.scala | 10 +++---
.../server/nlp/spell/NCSpellCheckManager.scala | 6 ++--
13 files changed, 79 insertions(+), 72 deletions(-)
diff --git a/external/readme.txt b/external/readme.txt
index fe255cd..e8a226c 100644
--- a/external/readme.txt
+++ b/external/readme.txt
@@ -15,7 +15,7 @@
# limitations under the License.
#
-This folder contains 3rd party pre-packaged configuration.
+This folder contains external pre-packaged configuration.
'spell' and 'geo' folders contain configuration that is based on external data that is licensed under Creative
Commons Attribution 4.0 International (CC BY 4.0) license (https://creativecommons.org/licenses/by/4.0/).
diff --git a/nlpcraft/src/main/resources/nlpcraft.conf b/nlpcraft/src/main/resources/nlpcraft.conf
index 6a7663a..da972c8 100644
--- a/nlpcraft/src/main/resources/nlpcraft.conf
+++ b/nlpcraft/src/main/resources/nlpcraft.conf
@@ -294,22 +294,21 @@ nlpcraft {
# Due to licensing restrictions of the official ASF release policy some of the
# configuration for NLPCraft cannot be shipped with the official Apache release.
# Instead, NLPCraft will attempt to download these configuration files from the
- # external source upon the first start.
+ # external URL upon the first start.
#
- # NLPCraft will attempt to download locally missing configuration files from URL defined
+ # NLPCraft will attempt to download the missing configuration files from URL defined
# in 'nlpcraft.extConfig.extUrl' property and place them into 'nlpcraft.extConfig.locDir'
- # folder. On subsequent starts, NLPCraft will check if the required file is present locally
- # and skip the download. If 'nlpcraft.extConfig.checkMd5' property is set to 'true' then
- # on each start NLPCraft will check the checksum of each file locally and remote and will
- # re-download such file if the MD5 checksums don't match.
+ # folder on the local file system. On subsequent starts, NLPCraft will check if the required
+ # file is already present locally and skip the download in such case. If 'nlpcraft.extConfig.checkMd5'
+ # property is set to 'true' then on each start NLPCraft will check the checksum of each file
+ # locally and remote and will re-download such file if the MD5 checksums don't match.
#
# By default, the external configuration is stored in the main Git repository for NLPCraft
# project from where it will be downloaded ('/external' folder). See this folder in the Git
# reposotiry for more information: https://github.com/apache/incubator-nlpcraft/tree/master/external
extConfig {
# Mandatory.
- # TODO: change url to master.
- extUrl = "https://github.com/apache/incubator-nlpcraft/raw/NLPCRAFT-98/external"
+ extUrl = "https://github.com/apache/incubator-nlpcraft/raw/master/external"
# Optional.
# Default value is $USER_HOME/.nlpcraft/extcfg
@@ -318,8 +317,8 @@ nlpcraft {
# If 'true', on each start NLPCraft will check the MD5 checksum of the each local and remote
# external configuration file and will re-download such file if the checksum doesn't match.
# Set it to 'false' to speed up the bootstrap of the NLPCrafrt server and the data probe if you
- # are certain that all external configuration files are properly downloaed and available locally
- # in 'nlpcraft.extConfig.locDir' folder.
+ # are certain that all external configuration files are properly downloaded and available
+ # in 'nlpcraft.extConfig.locDir' local folder.
checkMd5 = true
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/resources/NCResourceTxtContent.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigHolder.scala
similarity index 67%
rename from nlpcraft/src/main/scala/org/apache/nlpcraft/common/resources/NCResourceTxtContent.scala
rename to nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigHolder.scala
index 5e26c37..175011b 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/resources/NCResourceTxtContent.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigHolder.scala
@@ -15,9 +15,15 @@
* limitations under the License.
*/
-package org.apache.nlpcraft.common.resources
+package org.apache.nlpcraft.common.extcfg
-import org.apache.nlpcraft.common.resources.NCResourceType.NCResourceType
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigType.NCResourceType
-// TODO:
-case class NCResourceTxtContent(resType: NCResourceType, resource: String, content: String)
+/**
+ * Holder for a single external configuration resource.
+ *
+ * @param resType Resource type.
+ * @param fileName Resource file name.
+ * @param content Resource content.
+ */
+case class NCExternalConfigHolder(resType: NCResourceType, fileName: String, content: String)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/resources/NCExtResourceManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigManager.scala
similarity index 87%
rename from nlpcraft/src/main/scala/org/apache/nlpcraft/common/resources/NCExtResourceManager.scala
rename to nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigManager.scala
index cfb52a8..bb9019e 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/resources/NCExtResourceManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigManager.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.nlpcraft.common.resources
+package org.apache.nlpcraft.common.extcfg
import java.io._
import java.net.URL
@@ -26,16 +26,16 @@ import io.opencensus.trace.Span
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.IOUtils
import org.apache.nlpcraft.common.config.NCConfigurable
-import org.apache.nlpcraft.common.resources.NCResourceType._
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigType._
import org.apache.nlpcraft.common.{NCE, NCService, U}
import resource.managed
import scala.collection.JavaConverters._
import scala.io.Source
/**
- * External resources manager.
+ * External configuration manager.
*/
-object NCExtResourceManager extends NCService {
+object NCExternalConfigManager extends NCService {
private final val DFLT_DIR = ".nlpcraft/extcfg"
private final val MD5_FILE = "md5.txt"
@@ -145,12 +145,12 @@ object NCExtResourceManager extends NCService {
* @param parent Optional parent span.
*/
override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ ⇒
- require(NCResourceType.values.forall(FILES.contains))
+ require(NCExternalConfigType.values.forall(FILES.contains))
val m = new ConcurrentHashMap[NCResourceType, File]
U.executeParallel(
- NCResourceType.values.flatMap(t ⇒ FILES(t).map(FileHolder(_, t))).toSeq.map(f ⇒ () ⇒ processFile(f, m)): _*
+ NCExternalConfigType.values.flatMap(t ⇒ FILES(t).map(FileHolder(_, t))).toSeq.map(f ⇒ () ⇒ processFile(f, m)): _*
)
val downTypes = m.asScala
@@ -199,20 +199,20 @@ object NCExtResourceManager extends NCService {
@throws[NCE]
def getDirContent(
typ: NCResourceType, resDir: String, resFilter: String ⇒ Boolean, parent: Span = null
- ): Stream[NCResourceTxtContent] =
+ ): Stream[NCExternalConfigHolder] =
startScopedSpan("getDirContent", parent, "resDir" → resDir) { _ ⇒
val resDirPath = getResourcePath(typ, resDir)
val d = new File(Config.dir, resDirPath)
if (!d.exists || !d.isDirectory)
- throw new NCE(s"'${d.getAbsolutePath}' is not valid folder")
+ throw new NCE(s"'${d.getAbsolutePath}' is not a valid folder.")
val files =
d.listFiles(new FileFilter { override def accept(f: File): Boolean = f.isFile && resFilter(f.getName) })
if (files != null)
- files.toStream.map(f ⇒ NCResourceTxtContent(typ, f.getName, mkString(U.readFile(f, "UTF-8"))))
+ files.toStream.map(f ⇒ NCExternalConfigHolder(typ, f.getName, mkString(U.readFile(f, "UTF-8"))))
else
Stream.empty
}
@@ -226,12 +226,12 @@ object NCExtResourceManager extends NCService {
private def processFile(h: FileHolder, m: ConcurrentHashMap[NCResourceType, File]): Unit =
if (h.file.exists()) {
if (h.file.isDirectory)
- throw new NCE(s"Unexpected folder: '${h.file.getAbsolutePath}'")
+ throw new NCE(s"Unexpected folder (expecting a file): ${h.file.getAbsolutePath}")
if (h.file.length() == 0 || Config.checkMd5 && !Md5.isValid(h.file, h.typ)) {
logger.warn(
- s"File: '${h.file.getAbsolutePath}' corrupted. " +
- s"All files of: '${h.typ}' will be deleted and downloaded again"
+ s"File '${h.file.getAbsolutePath}' appears to be corrupted. " +
+ s"All related files will be deleted and downloaded again."
)
m.put(h.typ, h.dir)
@@ -255,10 +255,10 @@ object NCExtResourceManager extends NCService {
IOUtils.copy(src, dest)
}
- logger.info(s"File downloaded [url='$url', file='$filePath']")
+ logger.info(s"External config downloaded [url='$url', file='$filePath']")
}
catch {
- case e: IOException ⇒ throw new NCE(s"Error downloading file [url='$url', file='$filePath']", e)
+ case e: IOException ⇒ throw new NCE(s"Failed to download external config [url='$url', file='$filePath']", e)
}
def safeDelete(): Unit =
@@ -277,7 +277,7 @@ object NCExtResourceManager extends NCService {
try {
U.unzip(filePath, destDirPath)
- logger.info(s"File unzipped [file='$filePath', dest='$destDirPath']")
+ logger.trace(s"File unzipped [file='$filePath', dest='$destDirPath']")
}
catch {
case e: NCE ⇒
@@ -301,7 +301,7 @@ object NCExtResourceManager extends NCService {
@throws[NCE]
private def string2Type(s: String) =
try
- NCResourceType.withName(s.toUpperCase)
+ NCExternalConfigType.withName(s.toUpperCase)
catch {
case e: IllegalArgumentException ⇒ throw new NCE(s"Invalid type: '$s'", e)
}
@@ -320,11 +320,11 @@ object NCExtResourceManager extends NCService {
private def checkAndPrepareDir(d: File): Unit =
if (d.exists()) {
if (!d.isDirectory)
- throw new NCE(s"'${d.getAbsolutePath}' is not valid folder")
+ throw new NCE(s"'${d.getAbsolutePath}' is not a valid folder.")
}
else {
if (!d.mkdirs())
- throw new NCE(s"'${d.getAbsolutePath}' folder cannot be created")
+ throw new NCE(s"'${d.getAbsolutePath}' folder cannot be created.")
}
/**
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/resources/NCResourceType.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigType.scala
similarity index 84%
rename from nlpcraft/src/main/scala/org/apache/nlpcraft/common/resources/NCResourceType.scala
rename to nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigType.scala
index 8831be2..b62a8bc 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/resources/NCResourceType.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigType.scala
@@ -15,10 +15,12 @@
* limitations under the License.
*/
-package org.apache.nlpcraft.common.resources
+package org.apache.nlpcraft.common.extcfg
-// TODO:
-object NCResourceType extends Enumeration {
+/**
+ * Internal type for external configuration resource.
+ */
+object NCExternalConfigType extends Enumeration {
type NCResourceType = Value
val SPELL, GEO, OPENNLP = Value
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/core/opennlp/NCOpenNlpTokenizer.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/core/opennlp/NCOpenNlpTokenizer.scala
index 22a0b8e..60ddfc9 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/core/opennlp/NCOpenNlpTokenizer.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/core/opennlp/NCOpenNlpTokenizer.scala
@@ -21,9 +21,9 @@ import io.opencensus.trace.Span
import opennlp.tools.tokenize.{Tokenizer, TokenizerME, TokenizerModel}
import org.apache.nlpcraft.common.NCService
import org.apache.nlpcraft.common.nlp.core.{NCNlpCoreToken, NCNlpTokenizer}
-import org.apache.nlpcraft.common.resources.NCExtResourceManager
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigManager
import resource.managed
-import org.apache.nlpcraft.common.resources.NCResourceType.OPENNLP
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigType.OPENNLP
import scala.language.{implicitConversions, postfixOps}
/**
@@ -35,7 +35,7 @@ object NCOpenNlpTokenizer extends NCNlpTokenizer {
@volatile private var tokenizer: Tokenizer = _
override def start(parent: Span): NCService = {
- tokenizer = managed(NCExtResourceManager.getStream(OPENNLP, RESOURCE)) acquireAndGet { in ⇒
+ tokenizer = managed(NCExternalConfigManager.getStream(OPENNLP, RESOURCE)) acquireAndGet { in ⇒
new TokenizerME(new TokenizerModel(in))
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
index 40abd15..b583100 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
@@ -27,7 +27,7 @@ import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.nlp.dict.NCDictionaryManager
import org.apache.nlpcraft.common.nlp.numeric.NCNumericManager
import org.apache.nlpcraft.common.opencensus.NCOpenCensusTrace
-import org.apache.nlpcraft.common.resources.NCExtResourceManager
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigManager
import org.apache.nlpcraft.common.version.NCVersion
import org.apache.nlpcraft.common.{NCE, NCException, U}
import org.apache.nlpcraft.model.NCModel
@@ -418,7 +418,7 @@ private [probe] object NCProbeBoot extends LazyLogging with NCOpenCensusTrace {
"jarFolder" → cfg.jarsFolder
)
- NCExtResourceManager.start(span)
+ NCExternalConfigManager.start(span)
NCNlpCoreManager.start(span)
NCNumericManager.start(span)
NCDeployManager.start(span)
@@ -464,7 +464,7 @@ private [probe] object NCProbeBoot extends LazyLogging with NCOpenCensusTrace {
NCDeployManager.stop(span)
NCNumericManager.stop(span)
NCNlpCoreManager.stop(span)
- NCExtResourceManager.stop(span)
+ NCExternalConfigManager.stop(span)
}
// Lifecycle callback.
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
index 9021204..1418685 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
@@ -28,7 +28,7 @@ import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.nlp.dict.NCDictionaryManager
import org.apache.nlpcraft.common.nlp.numeric.NCNumericManager
import org.apache.nlpcraft.common.opencensus.NCOpenCensusTrace
-import org.apache.nlpcraft.common.resources.NCExtResourceManager
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigManager
import org.apache.nlpcraft.common.version._
import org.apache.nlpcraft.server.company.NCCompanyManager
import org.apache.nlpcraft.server.feedback.NCFeedbackManager
@@ -89,7 +89,7 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
startScopedSpan("startManagers", "relVer" → ver.version, "relDate" → ver.date) { span ⇒
U.executeParallel(
- () ⇒ NCExtResourceManager.start(span),
+ () ⇒ NCExternalConfigManager.start(span),
() ⇒ NCWordNetManager.start(span),
() ⇒ NCDictionaryManager.start(span),
() ⇒ {
@@ -158,7 +158,7 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
NCSqlManager,
NCTxManager,
NCNlpCoreManager,
- NCExtResourceManager
+ NCExternalConfigManager
).foreach(p ⇒
try
p.stop(span)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/NCGeoManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/NCGeoManager.scala
index 2b42b87..b44ed56 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/NCGeoManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/NCGeoManager.scala
@@ -20,8 +20,8 @@ package org.apache.nlpcraft.server.geo
import com.fasterxml.jackson.core.`type`.TypeReference
import io.opencensus.trace.Span
import org.apache.nlpcraft.common.nlp.dict.{NCDictionaryManager, NCDictionaryType}
-import org.apache.nlpcraft.common.resources.NCExtResourceManager
-import org.apache.nlpcraft.common.resources.NCResourceType.GEO
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigManager
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigType.GEO
import org.apache.nlpcraft.common.{NCService, _}
import scala.collection.{immutable, mutable}
@@ -173,7 +173,7 @@ object NCGeoManager extends NCService {
// | 1. Process metros. |
// +====================+
for (p ← U.extractYamlString(
- NCExtResourceManager.getContent(GEO, METRO_RESOURCE), METRO_RESOURCE, ignoreCase = true, new TypeReference[List[YamlMetro]] {})
+ NCExternalConfigManager.getContent(GEO, METRO_RESOURCE), METRO_RESOURCE, ignoreCase = true, new TypeReference[List[YamlMetro]] {})
)
addEntry(p.name, NCGeoMetro(p.name), lowerCase = true)
@@ -183,7 +183,7 @@ object NCGeoManager extends NCService {
val ctrsIsoToSubconts = mutable.HashMap.empty[String, NCGeoSubContinent]
for ((contName, subMap) ← U.extractYamlString(
- NCExtResourceManager.getContent(GEO, CONT_RESOURCE),
+ NCExternalConfigManager.getContent(GEO, CONT_RESOURCE),
CONT_RESOURCE,
ignoreCase = true,
new TypeReference[immutable.Map[String, immutable.Map[String, List[YamlCountry]]]] {} )
@@ -218,9 +218,9 @@ object NCGeoManager extends NCService {
val cntrMap = mutable.HashMap.empty[String, NCGeoCountry]
val citiesMap = mutable.HashMap.empty[CityKey, NCGeoCity]
- for (res ← NCExtResourceManager.getDirContent(GEO, COUNTRY_DIR, (name: String) ⇒ name.endsWith("yaml"))) {
+ for (res ← NCExternalConfigManager.getDirContent(GEO, COUNTRY_DIR, (name: String) ⇒ name.endsWith("yaml"))) {
val countryYaml =
- U.extractYamlString(res.content, res.resource, ignoreCase = true, new TypeReference[YamlCountryHolder] {})
+ U.extractYamlString(res.content, res.fileName, ignoreCase = true, new TypeReference[YamlCountryHolder] {})
val meta = mutable.HashMap.empty[String, Any]
@@ -347,8 +347,8 @@ object NCGeoManager extends NCService {
}
for (
- res ← NCExtResourceManager.getDirContent(GEO, SYNONYMS_DIR, (name: String) ⇒ name.endsWith("yaml"));
- s ← extractSynonyms(res.content, res.resource, ignoreCase = true)
+ res ← NCExternalConfigManager.getDirContent(GEO, SYNONYMS_DIR, (name: String) ⇒ name.endsWith("yaml"));
+ s ← extractSynonyms(res.content, res.fileName, ignoreCase = true)
)
process(
s,
@@ -358,7 +358,7 @@ object NCGeoManager extends NCService {
// NCGeoSynonym shouldn't be matched with common dictionary word.
// Exception - manually defined synonyms.
syns.filter(s ⇒
- SYNONYMS_MANUAL_FILES.exists(p ⇒ res.resource.endsWith(s"/$p")) ||
+ SYNONYMS_MANUAL_FILES.exists(p ⇒ res.fileName.endsWith(s"/$p")) ||
(!dicts.contains(s) &&
!dicts.contains(s.replaceAll("the ", "")) &&
!dicts.contains(s.replaceAll("the-", "")))
@@ -372,8 +372,8 @@ object NCGeoManager extends NCService {
// +=====================================+
for (
- res ← NCExtResourceManager.getDirContent(GEO, CASE_SENSITIVE_DIR, (name: String) ⇒ name.endsWith("yaml"));
- s ← extractSynonyms(res.content, res.resource, ignoreCase = false)
+ res ← NCExternalConfigManager.getDirContent(GEO, CASE_SENSITIVE_DIR, (name: String) ⇒ name.endsWith("yaml"));
+ s ← extractSynonyms(res.content, res.fileName, ignoreCase = false)
) {
def toLc(opt: Option[String]): Option[String] =
opt match {
@@ -426,7 +426,7 @@ object NCGeoManager extends NCService {
*/
def mkTopCities(res: String): immutable.Set[NCTopGeoCity] =
U.extractYamlString(
- NCExtResourceManager.getContent(GEO, res), res, ignoreCase = true, new TypeReference[List[YamlTopCity]] {}
+ NCExternalConfigManager.getContent(GEO, res), res, ignoreCase = true, new TypeReference[List[YamlTopCity]] {}
).
map(city ⇒
cntrs.find(_.name == city.country) match {
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpNerEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpNerEnricher.scala
index 559992a..ccfe257 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpNerEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpNerEnricher.scala
@@ -22,8 +22,8 @@ import opennlp.tools.namefind.{NameFinderME, TokenNameFinderModel}
import org.apache.ignite.IgniteCache
import org.apache.nlpcraft.common.nlp.core.opennlp.NCOpenNlpTokenizer
import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote}
-import org.apache.nlpcraft.common.resources.NCExtResourceManager
-import org.apache.nlpcraft.common.resources.NCResourceType.OPENNLP
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigManager
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigType.OPENNLP
import org.apache.nlpcraft.common.{NCService, U}
import org.apache.nlpcraft.server.ignite.NCIgniteHelpers._
import org.apache.nlpcraft.server.ignite.NCIgniteInstance
@@ -46,7 +46,7 @@ object NCOpenNlpNerEnricher extends NCService with NCNlpNerEnricher with NCIgnit
def add(typ: String, res: String): Unit = {
val f =
- managed(NCExtResourceManager.getStream(OPENNLP, res, span)) acquireAndGet { in ⇒
+ managed(NCExternalConfigManager.getStream(OPENNLP, res, span)) acquireAndGet { in ⇒
new NameFinderME(new TokenNameFinderModel(in))
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpParser.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpParser.scala
index 362d37e..bd94850 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpParser.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpParser.scala
@@ -23,8 +23,8 @@ import opennlp.tools.postag.{POSModel, POSTagger, POSTaggerME}
import org.apache.ignite.IgniteCache
import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.nlp.core.opennlp.NCOpenNlpTokenizer
-import org.apache.nlpcraft.common.resources.NCExtResourceManager
-import org.apache.nlpcraft.common.resources.NCResourceType.OPENNLP
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigManager
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigType.OPENNLP
import org.apache.nlpcraft.common.{NCService, U}
import org.apache.nlpcraft.server.ignite.NCIgniteHelpers._
import org.apache.nlpcraft.server.ignite.NCIgniteInstance
@@ -47,13 +47,13 @@ object NCOpenNlpParser extends NCService with NCNlpParser with NCIgniteInstance
U.executeParallel(
() ⇒ {
tagger =
- managed(NCExtResourceManager.getStream(OPENNLP, "en-pos-maxent.bin", span)) acquireAndGet { in ⇒
+ managed(NCExternalConfigManager.getStream(OPENNLP, "en-pos-maxent.bin", span)) acquireAndGet { in ⇒
new POSTaggerME(new POSModel(in))
}
},
() ⇒ {
lemmatizer =
- managed(NCExtResourceManager.getStream(OPENNLP, "en-lemmatizer.dict", span)) acquireAndGet { in ⇒
+ managed(NCExternalConfigManager.getStream(OPENNLP, "en-lemmatizer.dict", span)) acquireAndGet { in ⇒
new DictionaryLemmatizer(in)
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/geo/NCGeoEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/geo/NCGeoEnricher.scala
index dffee05..471518b 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/geo/NCGeoEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/geo/NCGeoEnricher.scala
@@ -24,8 +24,8 @@ import io.opencensus.trace.Span
import org.apache.nlpcraft.common._
import org.apache.nlpcraft.common.nlp._
import org.apache.nlpcraft.common.nlp.pos.NCPennTreebank
-import org.apache.nlpcraft.common.resources.NCExtResourceManager
-import org.apache.nlpcraft.common.resources.NCResourceType.GEO
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigManager
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigType.GEO
import org.apache.nlpcraft.server.geo.NCGeoLocationKind._
import org.apache.nlpcraft.server.geo._
import org.apache.nlpcraft.server.nlp.enrichers.NCServerEnricher
@@ -92,7 +92,7 @@ object NCGeoEnricher extends NCServerEnricher {
// Note that 'ignore case' parameter set as false because DLGeoLocationKind definition (CITY ect)
commons =
- NCExtResourceManager.getDirContent(
+ NCExternalConfigManager.getDirContent(
GEO,
EXCEPTIONS_PATH,
(name: String) ⇒ name.endsWith("yaml")
@@ -100,7 +100,7 @@ object NCGeoEnricher extends NCServerEnricher {
flatMap(p ⇒
U.extractYamlString(
p.content,
- p.resource,
+ p.fileName,
ignoreCase = false,
new TypeReference[immutable.Map[String, immutable.Set[String]]] {}
)
@@ -111,7 +111,7 @@ object NCGeoEnricher extends NCServerEnricher {
def readCities(res: String): List[TopCity] =
U.extractYamlString(
- NCExtResourceManager.getContent(GEO, res), res, ignoreCase = true, new TypeReference[List[TopCity]] {}
+ NCExternalConfigManager.getContent(GEO, res), res, ignoreCase = true, new TypeReference[List[TopCity]] {}
)
topUsa = readCities(US_TOP_PATH).map(city ⇒ glue(city.name, city.region)).toSet
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/spell/NCSpellCheckManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/spell/NCSpellCheckManager.scala
index 8f192a6..9cbe995 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/spell/NCSpellCheckManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/spell/NCSpellCheckManager.scala
@@ -19,9 +19,9 @@ package org.apache.nlpcraft.server.nlp.spell
import com.fasterxml.jackson.core.`type`.TypeReference
import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.resources.NCExtResourceManager
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigManager
import org.apache.nlpcraft.common.{NCService, U}
-import org.apache.nlpcraft.common.resources.NCResourceType.SPELL
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigType.SPELL
import scala.collection._
@@ -46,7 +46,7 @@ object NCSpellCheckManager extends NCService {
override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ ⇒
dict = U.extractYamlString(
- NCExtResourceManager.getContent(SPELL, RESOURCE),
+ NCExternalConfigManager.getContent(SPELL, RESOURCE),
RESOURCE,
ignoreCase = true,
new TypeReference[Map[String, String]] {}