You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2021/01/20 14:36:20 UTC
[incubator-nlpcraft] branch NLPCRAFT-111 created (now 2f1a98e)
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a change to branch NLPCRAFT-111
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
at 2f1a98e WIP.
This branch includes the following new commits:
new ea0045f WIP.
new 2f1a98e WIP.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-nlpcraft] 01/02: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-111
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit ea0045f9a25d9c9e618e3bfb384eb1ffd8def98a
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Wed Jan 20 17:35:02 2021 +0300
WIP.
---
nlpcraft/src/main/resources/nlpcraft.conf | 4 +
.../common/extcfg/NCExternalConfigManager.scala | 1 +
.../org/apache/nlpcraft/common/util/NCUtils.scala | 7 +-
.../org/apache/nlpcraft/probe/NCProbeBoot.scala | 14 +--
.../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 2 +-
.../org/apache/nlpcraft/server/NCServer.scala | 117 +++++++++++----------
.../server/nlp/core/NCNlpServerManager.scala | 3 +-
.../nlp/core/opennlp/NCOpenNlpNerEnricher.scala | 3 +-
.../server/nlp/core/opennlp/NCOpenNlpParser.scala | 3 +-
.../nlp/enrichers/NCServerEnrichmentManager.scala | 3 +-
.../server/nlp/enrichers/date/NCDateEnricher.scala | 4 +-
.../nlpcraft/server/probe/NCProbeManager.scala | 17 ++-
.../nlpcraft/server/query/NCQueryManager.scala | 5 +-
.../nlpcraft/server/rest/NCBasicRestApi.scala | 4 +-
.../server/sugsyn/NCSuggestSynonymManager.scala | 6 +-
15 files changed, 101 insertions(+), 92 deletions(-)
diff --git a/nlpcraft/src/main/resources/nlpcraft.conf b/nlpcraft/src/main/resources/nlpcraft.conf
index 1b1fa9d..b67ad76 100644
--- a/nlpcraft/src/main/resources/nlpcraft.conf
+++ b/nlpcraft/src/main/resources/nlpcraft.conf
@@ -207,6 +207,8 @@ nlpcraft {
# 'ctxword' server endpoint URL.
ctxword.url="http://localhost:5000"
+
+ # pools =
}
# Basic NLP toolkit to use on both server and probes. Possible values:
@@ -324,5 +326,7 @@ nlpcraft {
# Maximum execution result size in bytes. Default value is 1M.
# When exceeded the request will be automatically rejected.
resultMaxSizeBytes = 1048576
+
+ # pools =
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigManager.scala
index 773bfe7..f34eb81 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/extcfg/NCExternalConfigManager.scala
@@ -31,6 +31,7 @@ import org.apache.nlpcraft.common.{NCE, NCService, U}
import resource.managed
import scala.collection.JavaConverters._
import scala.io.Source
+import scala.concurrent.ExecutionContext.Implicits.global
/**
* External configuration manager.
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
index c6953b5..09cf8f7 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
@@ -57,7 +57,6 @@ import java.net.http.HttpRequest
import java.net.http.HttpResponse
import scala.collection.JavaConverters._
import scala.collection._
-import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.concurrent.{Await, ExecutionContext, Future}
import scala.io.{BufferedSource, Source}
@@ -931,7 +930,7 @@ object NCUtils extends LazyLogging {
def asFuture[T](
body: Unit ⇒ T,
onFailure: Throwable ⇒ Unit = _ ⇒ Unit,
- onSuccess: T ⇒ Unit = (_: T) ⇒ ())(implicit ec: ExecutionContext = global): Future[T] = {
+ onSuccess: T ⇒ Unit = (_: T) ⇒ ())(implicit ec: ExecutionContext): Future[T] = {
val fut = Future {
body(())
}(ec)
@@ -1149,7 +1148,7 @@ object NCUtils extends LazyLogging {
*
* @param es Executor service.
*/
- def shutdownPools(es: ExecutorService): Unit =
+ def shutdownPool(es: ExecutorService): Unit =
if (es != null) {
es.shutdown()
@@ -1815,7 +1814,7 @@ object NCUtils extends LazyLogging {
* @param bodies
* @param ec
*/
- def executeParallel(bodies: (() ⇒ Any)*)(implicit ec: ExecutionContext = global): Unit = {
+ def executeParallel(bodies: (() ⇒ Any)*)(implicit ec: ExecutionContext): Unit = {
bodies.map(body ⇒ {
Future {
body()
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
index f47eb2c..6ffbd36 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
@@ -17,22 +17,20 @@
package org.apache.nlpcraft.probe
-import java.io._
-import java.util.concurrent.CompletableFuture
-
import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory}
import com.typesafe.scalalogging.LazyLogging
import io.opencensus.trace.Span
import org.apache.commons.lang3.SystemUtils
import org.apache.nlpcraft.common.ascii.NCAsciiTable
import org.apache.nlpcraft.common.config.NCConfigurable
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigManager
import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.nlp.dict.NCDictionaryManager
import org.apache.nlpcraft.common.nlp.numeric.NCNumericManager
import org.apache.nlpcraft.common.opencensus.NCOpenCensusTrace
-import org.apache.nlpcraft.common.extcfg.NCExternalConfigManager
import org.apache.nlpcraft.common.version.NCVersion
-import org.apache.nlpcraft.common.{NCE, NCException, NCService, U}
+import org.apache.nlpcraft.common.{NCE, NCException, NCService, U, _}
+import org.apache.nlpcraft.model.tools.cmdline.NCCliProbeBeacon
import org.apache.nlpcraft.probe.mgrs.cmd.NCCommandManager
import org.apache.nlpcraft.probe.mgrs.conn.NCConnectionManager
import org.apache.nlpcraft.probe.mgrs.conversation.NCConversationManager
@@ -49,10 +47,11 @@ import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.sort.NCSortEnricher
import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.stopword.NCStopWordEnricher
import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.suspicious.NCSuspiciousNounsEnricher
import org.apache.nlpcraft.probe.mgrs.nlp.validate.NCValidateManager
-import org.apache.nlpcraft.common._
-import org.apache.nlpcraft.model.tools.cmdline.NCCliProbeBeacon
+import org.apache.nlpcraft.probe.mgrs.pool.NCProbePoolManager
import resource.managed
+import java.io._
+import java.util.concurrent.CompletableFuture
import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.compat.Platform.currentTime
@@ -494,6 +493,7 @@ private [probe] object NCProbeBoot extends LazyLogging with NCOpenCensusTrace {
"jarFolder" → cfg.jarsFolder
)
+ startedMgrs += NCProbePoolManager.start(span)
startedMgrs += NCExternalConfigManager.start(span)
startedMgrs += NCNlpCoreManager.start(span)
startedMgrs += NCNumericManager.start(span)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
index 7502cf5..1cd6fa7 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
@@ -113,7 +113,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats {
embeddedCbs.clear()
}
- U.shutdownPools(pool)
+ U.shutdownPool(pool)
executor = null
pool = null
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
index df113ce..dc0ec78 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
@@ -17,9 +17,6 @@
package org.apache.nlpcraft.server
-import java.io.{File, FileInputStream, FileOutputStream, IOException, ObjectInputStream, ObjectOutputStream}
-import java.util.concurrent.CountDownLatch
-
import com.typesafe.config.Config
import com.typesafe.scalalogging.LazyLogging
import org.apache.commons.lang3.SystemUtils
@@ -44,6 +41,7 @@ import org.apache.nlpcraft.server.nlp.enrichers.NCServerEnrichmentManager
import org.apache.nlpcraft.server.nlp.preproc.NCPreProcessManager
import org.apache.nlpcraft.server.nlp.spell.NCSpellCheckManager
import org.apache.nlpcraft.server.nlp.wordnet.NCWordNetManager
+import org.apache.nlpcraft.server.pool.NCServerPoolManager
import org.apache.nlpcraft.server.probe.NCProbeManager
import org.apache.nlpcraft.server.proclog.NCProcessLogManager
import org.apache.nlpcraft.server.query.NCQueryManager
@@ -54,45 +52,48 @@ import org.apache.nlpcraft.server.tx.NCTxManager
import org.apache.nlpcraft.server.user.NCUserManager
import resource.managed
+import java.io._
+import java.util.concurrent.CountDownLatch
import scala.collection.mutable
import scala.compat.Platform.currentTime
import scala.util.control.Exception.{catching, ignoring}
/**
- * NLPCraft server app.
- */
+ * NLPCraft server app.
+ */
object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCensusTrace {
private final val BEACON_PATH = ".nlpcraft/server_beacon"
private val startedMgrs = mutable.Buffer.empty[NCService]
/**
- * Prints ASCII-logo.
- */
+ * Prints ASCII-logo.
+ */
private def asciiLogo() {
val ver = NCVersion.getCurrent
logger.info(
U.NL +
- U.asciiLogo() +
- s"${U.NL}" +
- s"Server${U.NL}" +
- s"Version: ${bo(ver.version)}${U.NL}" +
- s"${NCVersion.copyright}${U.NL}"
+ U.asciiLogo() +
+ s"${U.NL}" +
+ s"Server${U.NL}" +
+ s"Version: ${bo(ver.version)}${U.NL}" +
+ s"${NCVersion.copyright}${U.NL}"
)
}
/**
- * Starts all managers.
- */
+ * Starts all managers.
+ */
private def startManagers(): Unit = {
val ver = NCVersion.getCurrent
// Lifecycle manager has to be started outside of the tracing span.
NCServerLifecycleManager.start()
NCServerLifecycleManager.beforeStart()
-
+
startScopedSpan("startManagers", "relVer" → ver.version, "relDate" → ver.date) { span ⇒
+ startedMgrs += NCServerPoolManager.start(span)
startedMgrs += NCExternalConfigManager.start(span)
startedMgrs += NCWordNetManager.start(span)
startedMgrs += NCDictionaryManager.start(span)
@@ -113,19 +114,19 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
startedMgrs += NCFeedbackManager.start(span)
startedMgrs += NCQueryManager.start(span)
startedMgrs += NCRestManager.start(span)
-
+
// Lifecycle callback.
NCServerLifecycleManager.afterStart()
}
}
-
+
/**
- * Stops all managers.
- */
+ * Stops all managers.
+ */
private def stopManagers(): Unit = {
// Lifecycle callback.
NCServerLifecycleManager.beforeStop()
-
+
startScopedSpan("stopManagers") { span ⇒
startedMgrs.reverse.foreach(p ⇒
try
@@ -135,38 +136,38 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
}
)
}
-
+
// Lifecycle callback.
NCServerLifecycleManager.afterStop()
NCServerLifecycleManager.stop()
}
-
+
/**
- * Acks server start.
- */
+ * Acks server start.
+ */
protected def ackStart() {
val dur = s"[${U.format((currentTime - executionStart) / 1000.0, 2)}s]"
-
+
val tbl = NCAsciiTable()
-
+
tbl.margin(top = 1, bottom = 1)
-
+
tbl += s"Server started ${b(dur)}"
-
+
tbl.info(logger)
}
/**
- *
- * @return
- */
+ *
+ * @return
+ */
private def setSysProps(): Unit = {
System.setProperty("java.net.preferIPv4Stack", "true")
}
/**
- *
- */
+ *
+ */
private def start(): Unit = {
NCAnsi.ackStatus()
@@ -177,25 +178,25 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
args.find(_.startsWith("-config=")) match {
case Some(s) ⇒
val fileName = s.substring("-config=".length)
-
+
val f = new java.io.File(fileName)
if (!(f.exists && f.canRead && f.isFile))
throw new NCE(s"Specified server configuration file does not exist or cannot be read: $fileName")
Some(fileName)
-
+
case None ⇒
Some("nlpcraft.conf") // Default to 'nlpcraft.conf'.
},
None, // No defaults.
(cfg: Config) ⇒ cfg.hasPath("nlpcraft.server")
)
-
+
asciiLogo()
val lifecycle = new CountDownLatch(1)
-
+
catching(classOf[Throwable]) either startManagers() match {
case Left(e) ⇒ // Exception.
U.prettyError(logger, "Failed to start server:", e)
@@ -203,7 +204,7 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
stopManagers()
System.exit(1)
-
+
case _ ⇒ // Managers started OK.
// Store beacon file once all managers started OK.
storeBeacon()
@@ -235,7 +236,7 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
/**
*
*/
- def save() = {
+ def save(): Unit = {
final object Config extends NCConfigurable {
final private val pre = "nlpcraft.server"
@@ -243,21 +244,21 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
lazy val restHost = getString(s"$pre.rest.host")
lazy val restApi = getString(s"$pre.rest.apiImpl")
lazy val restPort = getInt(s"$pre.rest.port")
- lazy val upLink = getString(s"$pre.probe.links.upLink")
- lazy val downLink = getString(s"$pre.probe.links.downLink")
- lazy val dbUrl = getString(s"$pre.database.jdbc.url")
- lazy val dbDriver = getString(s"$pre.database.jdbc.driver")
- lazy val dbPoolMin = getInt(s"$pre.database.c3p0.pool.minSize")
- lazy val dbPoolMax = getInt(s"$pre.database.c3p0.pool.maxSize")
- lazy val dbPoolInit = getInt(s"$pre.database.c3p0.pool.initSize")
- lazy val dbPoolInc = getInt(s"$pre.database.c3p0.pool.acquireIncrement")
- lazy val dbInit = getBool(s"$pre.database.igniteDbInitialize")
- lazy val tokProviders = getString(s"$pre.tokenProviders")
- lazy val acsToksScanMins = getInt(s"$pre.user.timeoutScannerFreqMins")
- lazy val acsToksExpireMins = getInt(s"$pre.user.accessTokenExpireTimeoutMins")
- lazy val nlpEngine = getString("nlpcraft.nlpEngine")
- lazy val extCfgUrl = getString("nlpcraft.extConfig.extUrl")
- lazy val extCfgCheckMd5 = getBool("nlpcraft.extConfig.checkMd5")
+ lazy val upLink = getString(s"$pre.probe.links.upLink")
+ lazy val downLink = getString(s"$pre.probe.links.downLink")
+ lazy val dbUrl = getString(s"$pre.database.jdbc.url")
+ lazy val dbDriver = getString(s"$pre.database.jdbc.driver")
+ lazy val dbPoolMin = getInt(s"$pre.database.c3p0.pool.minSize")
+ lazy val dbPoolMax = getInt(s"$pre.database.c3p0.pool.maxSize")
+ lazy val dbPoolInit = getInt(s"$pre.database.c3p0.pool.initSize")
+ lazy val dbPoolInc = getInt(s"$pre.database.c3p0.pool.acquireIncrement")
+ lazy val dbInit = getBool(s"$pre.database.igniteDbInitialize")
+ lazy val tokProviders = getString(s"$pre.tokenProviders")
+ lazy val acsToksScanMins = getInt(s"$pre.user.timeoutScannerFreqMins")
+ lazy val acsToksExpireMins = getInt(s"$pre.user.accessTokenExpireTimeoutMins")
+ lazy val nlpEngine = getString("nlpcraft.nlpEngine")
+ lazy val extCfgUrl = getString("nlpcraft.extConfig.extUrl")
+ lazy val extCfgCheckMd5 = getBool("nlpcraft.extConfig.checkMd5")
lazy val restEndpoint = s"${Config.restHost}:${Config.restPort}/api/v1"
}
@@ -327,7 +328,9 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
if (path.exists())
catching(classOf[IOException]) either {
- managed(new ObjectInputStream(new FileInputStream(path))) acquireAndGet { _.readObject() }
+ managed(new ObjectInputStream(new FileInputStream(path))) acquireAndGet {
+ _.readObject()
+ }
} match {
case Left(e) ⇒
logger.trace(s"Failed to read existing server beacon: ${path.getAbsolutePath}", e)
@@ -350,7 +353,7 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
}
}
else
- // No existing beacon file detected.
+ // No existing beacon file detected.
save()
}
@@ -358,7 +361,7 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
new Thread() {
override def run(): Unit = U.gaScreenView("server")
}
- .start()
+ .start()
NCIgniteRunner.runWith(
args.find(_.startsWith("-igniteConfig=")) match {
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/NCNlpServerManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/NCNlpServerManager.scala
index fb12deb..0e496c8 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/NCNlpServerManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/NCNlpServerManager.scala
@@ -21,13 +21,14 @@ import io.opencensus.trace.Span
import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.{NCService, _}
+import org.apache.nlpcraft.server.pool.NCServerPoolContext
import scala.collection.Seq
/**
* Server NLP manager.
*/
-object NCNlpServerManager extends NCService {
+object NCNlpServerManager extends NCService with NCServerPoolContext {
@volatile private var parser: NCNlpParser = _
@volatile private var ners: Map[String, NCNlpNerEnricher] = _
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpNerEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpNerEnricher.scala
index f2dc816..d2e923f 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpNerEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpNerEnricher.scala
@@ -28,6 +28,7 @@ import org.apache.nlpcraft.common.{NCService, U}
import org.apache.nlpcraft.server.ignite.NCIgniteHelpers._
import org.apache.nlpcraft.server.ignite.NCIgniteInstance
import org.apache.nlpcraft.server.nlp.core.NCNlpNerEnricher
+import org.apache.nlpcraft.server.pool.NCServerPoolContext
import resource.managed
import scala.util.control.Exception.catching
@@ -35,7 +36,7 @@ import scala.util.control.Exception.catching
/**
* OpenNLP NER enricher.
*/
-object NCOpenNlpNerEnricher extends NCService with NCNlpNerEnricher with NCIgniteInstance {
+object NCOpenNlpNerEnricher extends NCService with NCNlpNerEnricher with NCIgniteInstance with NCServerPoolContext {
@volatile private var nerFinders: Map[NameFinderME, String] = _
@volatile private var cache: IgniteCache[String, Array[String]] = _
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpParser.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpParser.scala
index 62b8374..a219115 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpParser.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpParser.scala
@@ -29,6 +29,7 @@ import org.apache.nlpcraft.common.{NCService, U}
import org.apache.nlpcraft.server.ignite.NCIgniteHelpers._
import org.apache.nlpcraft.server.ignite.NCIgniteInstance
import org.apache.nlpcraft.server.nlp.core.{NCNlpParser, NCNlpWord}
+import org.apache.nlpcraft.server.pool.NCServerPoolContext
import resource.managed
import scala.util.control.Exception.catching
@@ -36,7 +37,7 @@ import scala.util.control.Exception.catching
/**
* OpenNLP parser implementation.
*/
-object NCOpenNlpParser extends NCService with NCNlpParser with NCIgniteInstance {
+object NCOpenNlpParser extends NCService with NCNlpParser with NCIgniteInstance with NCServerPoolContext{
@volatile private var tagger: POSTagger = _
@volatile private var lemmatizer: DictionaryLemmatizer = _
@volatile private var cache: IgniteCache[String, Array[String]] = _
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/NCServerEnrichmentManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/NCServerEnrichmentManager.scala
index 86b7bee..f7b6cd8 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/NCServerEnrichmentManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/NCServerEnrichmentManager.scala
@@ -34,6 +34,7 @@ import org.apache.nlpcraft.server.nlp.enrichers.numeric.NCNumericEnricher
import org.apache.nlpcraft.server.nlp.enrichers.quote.NCQuoteEnricher
import org.apache.nlpcraft.server.nlp.enrichers.stopword.NCStopWordEnricher
import org.apache.nlpcraft.server.nlp.preproc.NCPreProcessManager
+import org.apache.nlpcraft.server.pool.NCServerPoolContext
import scala.collection.Seq
import scala.util.control.Exception.catching
@@ -41,7 +42,7 @@ import scala.util.control.Exception.catching
/**
* Server enrichment pipeline manager.
*/
-object NCServerEnrichmentManager extends NCService with NCIgniteInstance {
+object NCServerEnrichmentManager extends NCService with NCIgniteInstance with NCServerPoolContext {
private object Config extends NCConfigurable {
def isBuiltInEnrichers: Boolean = getStringList("nlpcraft.server.tokenProviders").contains("nlpcraft")
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/date/NCDateEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/date/NCDateEnricher.scala
index 44b8a49..7c9f32a 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/date/NCDateEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/date/NCDateEnricher.scala
@@ -19,7 +19,6 @@ package org.apache.nlpcraft.server.nlp.enrichers.date
import java.util
import java.util.{Calendar ⇒ C}
-
import io.opencensus.trace.Span
import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.common.nlp.{NCNlpSentence ⇒ Sentence}
@@ -29,6 +28,7 @@ import org.apache.nlpcraft.common.{NCService, _}
import org.apache.nlpcraft.server.nlp.enrichers.NCServerEnricher
import org.apache.nlpcraft.server.nlp.enrichers.date.NCDateConstants._
import org.apache.nlpcraft.server.nlp.enrichers.date.NCDateFormatType._
+import org.apache.nlpcraft.server.pool.NCServerPoolContext
import scala.collection.JavaConverters._
import scala.collection.immutable.Iterable
@@ -38,7 +38,7 @@ import scala.collection.mutable.{LinkedHashMap ⇒ LHM}
/**
* Date enricher.
*/
-object NCDateEnricher extends NCServerEnricher {
+object NCDateEnricher extends NCServerEnricher with NCServerPoolContext {
private object Config extends NCConfigurable {
def style: NCDateFormatType = getObject("nlpcraft.server.datesFormatStyle", NCDateFormatType.withName)
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index 8e210ac..c439148 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -17,13 +17,6 @@
package org.apache.nlpcraft.server.probe
-import java.io._
-import java.net.{InetSocketAddress, ServerSocket, Socket, SocketTimeoutException}
-import java.security.Key
-import java.util
-import java.util.Collections
-import java.util.concurrent.ConcurrentHashMap
-
import com.google.gson.Gson
import com.google.gson.reflect.TypeToken
import io.opencensus.trace.Span
@@ -36,6 +29,7 @@ import org.apache.nlpcraft.common.socket.NCSocket
import org.apache.nlpcraft.common.version.NCVersion
import org.apache.nlpcraft.common.{NCService, _}
import org.apache.nlpcraft.probe.mgrs.NCProbeMessage
+import org.apache.nlpcraft.probe.mgrs.pool.NCProbePoolContext
import org.apache.nlpcraft.server.company.NCCompanyManager
import org.apache.nlpcraft.server.mdo.{NCCompanyMdo, NCProbeMdo, NCProbeModelMdo, NCUserMdo}
import org.apache.nlpcraft.server.nlp.enrichers.NCServerEnrichmentManager
@@ -43,16 +37,21 @@ import org.apache.nlpcraft.server.proclog.NCProcessLogManager
import org.apache.nlpcraft.server.query.NCQueryManager
import org.apache.nlpcraft.server.sql.NCSql
+import java.io._
+import java.net.{InetSocketAddress, ServerSocket, Socket, SocketTimeoutException}
+import java.security.Key
+import java.util
+import java.util.Collections
+import java.util.concurrent.ConcurrentHashMap
import scala.collection.JavaConverters._
import scala.collection.{Map, mutable}
-import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.{Future, Promise}
import scala.util.{Failure, Success}
/**
* Probe manager.
*/
-object NCProbeManager extends NCService {
+object NCProbeManager extends NCService with NCProbePoolContext {
private final val GSON = new Gson()
private final val TYPE_MODEL_INFO_RESP = new TypeToken[util.HashMap[String, AnyRef]]() {}.getType
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala
index 9ed0bb8..5faa00a 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala
@@ -18,7 +18,6 @@
package org.apache.nlpcraft.server.query
import java.util.concurrent.ConcurrentHashMap
-
import io.opencensus.trace.Span
import org.apache.ignite.IgniteCache
import org.apache.ignite.events.{CacheEvent, EventType}
@@ -31,12 +30,12 @@ import org.apache.nlpcraft.server.ignite.NCIgniteHelpers._
import org.apache.nlpcraft.server.ignite.NCIgniteInstance
import org.apache.nlpcraft.server.mdo.NCQueryStateMdo
import org.apache.nlpcraft.server.nlp.enrichers.NCServerEnrichmentManager
+import org.apache.nlpcraft.server.pool.NCServerPoolContext
import org.apache.nlpcraft.server.probe.NCProbeManager
import org.apache.nlpcraft.server.proclog.NCProcessLogManager
import org.apache.nlpcraft.server.tx.NCTxManager
import org.apache.nlpcraft.server.user.NCUserManager
-import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.{Future, Promise}
import scala.util.{Failure, Success}
import scala.util.control.Exception._
@@ -44,7 +43,7 @@ import scala.util.control.Exception._
/**
* Query state machine.
*/
-object NCQueryManager extends NCService with NCIgniteInstance with NCOpenCensusServerStats {
+object NCQueryManager extends NCService with NCIgniteInstance with NCOpenCensusServerStats with NCServerPoolContext {
@volatile private var cache: IgniteCache[String/*Server request ID*/, NCQueryStateMdo] = _
// Promises cannot be used in cache.
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
index bc42891..1c78df6 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
@@ -36,6 +36,7 @@ import org.apache.nlpcraft.server.company.NCCompanyManager
import org.apache.nlpcraft.server.feedback.NCFeedbackManager
import org.apache.nlpcraft.server.mdo.{NCQueryStateMdo, NCUserMdo}
import org.apache.nlpcraft.server.opencensus.NCOpenCensusServerStats
+import org.apache.nlpcraft.server.pool.NCServerPoolContext
import org.apache.nlpcraft.server.probe.NCProbeManager
import org.apache.nlpcraft.server.query.NCQueryManager
import org.apache.nlpcraft.server.sugsyn.NCSuggestSynonymManager
@@ -44,13 +45,12 @@ import spray.json.DefaultJsonProtocol._
import spray.json.{JsObject, JsValue, RootJsonFormat}
import scala.collection.JavaConverters._
-import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
/**
* REST API default implementation.
*/
-class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace with NCOpenCensusServerStats {
+class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace with NCOpenCensusServerStats with NCServerPoolContext {
protected final val GSON = new Gson()
protected final val URL_VALIDATOR = new UrlValidator(Array("http", "https"), UrlValidator.ALLOW_LOCAL_URLS)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
index fc17201..3a1c22e 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
@@ -35,16 +35,16 @@ import scala.util.{Failure, Success}
import java.util
import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
import java.util.concurrent._
-
import org.apache.http.client.methods.HttpPost
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.nlpcraft.common.makro.NCMacroParser
+import org.apache.nlpcraft.server.pool.NCServerPoolContext
/**
* Synonym suggestion manager.
*/
-object NCSuggestSynonymManager extends NCService {
+object NCSuggestSynonymManager extends NCService with NCServerPoolContext {
// For context word server requests.
private final val MAX_LIMIT: Int = 10000
private final val BATCH_SIZE = 20
@@ -133,7 +133,7 @@ object NCSuggestSynonymManager extends NCService {
override def stop(parent: Span): Unit = startScopedSpan("stop", parent) { _ ⇒
ackStopping()
- U.shutdownPools(pool)
+ U.shutdownPool(pool)
pool = null
executor = null
[incubator-nlpcraft] 02/02: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-111
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 2f1a98e2f5cb143f9ac7b84952c73991391018ef
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Wed Jan 20 17:35:58 2021 +0300
WIP.
---
.../nlpcraft/common/pool/NCPoolManager.scala | 82 ++++++++++++++++++++++
.../apache/nlpcraft/common/pool/NcPoolFactory.java | 7 ++
.../probe/mgrs/pool/NCProbePoolContext.scala | 27 +++++++
.../probe/mgrs/pool/NCProbePoolManager.scala | 26 +++++++
.../nlpcraft/server/pool/NCServerPoolContext.scala | 27 +++++++
.../nlpcraft/server/pool/NCServerPoolManager.scala | 25 +++++++
6 files changed, 194 insertions(+)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/pool/NCPoolManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/pool/NCPoolManager.scala
new file mode 100644
index 0000000..19a9d09
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/pool/NCPoolManager.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.common.pool
+
+import io.opencensus.trace.Span
+import org.apache.nlpcraft.common.config.NCConfigurable
+import org.apache.nlpcraft.common.{NCService, U}
+
+import java.util.concurrent.{ConcurrentHashMap, ExecutorService}
+import scala.collection.JavaConverters._
+import scala.concurrent.ExecutionContext
+
+/**
+ *
+ * @param cfg
+ */
+abstract class NCPoolManager(cfg: String) extends NCService {
+ @volatile private var data: ConcurrentHashMap[String, Holder] = new ConcurrentHashMap
+
+ private case class Holder(context: ExecutionContext, pool: Option[ExecutorService])
+
+ private object Config extends NCConfigurable {
+ val factories: Map[String, NcPoolFactory] = {
+ val m: Option[Map[String, String]] = getMapOpt(cfg)
+
+ m.getOrElse(Map.empty).map(p ⇒ p._1 → U.mkObject(p._2))
+ }
+ }
+
+ def getContext(name: String): ExecutionContext =
+ data.computeIfAbsent(
+ name,
+ (_: String) ⇒
+ Config.factories.get(name) match {
+ case Some(f) ⇒
+ val p = f.mkExecutorService()
+
+ logger.info(s"Executor service created with factory '${f.getClass.getName}' for '$name'")
+
+ Holder(ExecutionContext.fromExecutor(p), Some(p))
+ case None ⇒
+ logger.info(s"System executor service used for '$name'")
+
+ Holder(scala.concurrent.ExecutionContext.Implicits.global, None)
+ }
+ ).context
+
+ override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ ⇒
+ ackStarting()
+
+ data = new ConcurrentHashMap
+
+ ackStarted()
+ }
+
+ override def stop(parent: Span): Unit = startScopedSpan("stop", parent) { _ ⇒
+ ackStopping()
+
+ data.values().asScala.flatMap(_.pool).foreach(U.shutdownPool)
+
+ data.clear()
+
+ data = null
+
+ ackStopped()
+ }
+}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/pool/NcPoolFactory.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/pool/NcPoolFactory.java
new file mode 100644
index 0000000..e5b99a0
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/pool/NcPoolFactory.java
@@ -0,0 +1,7 @@
+package org.apache.nlpcraft.common.pool;
+
+import java.util.concurrent.ExecutorService;
+
+public interface NcPoolFactory {
+ ExecutorService mkExecutorService();
+}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/pool/NCProbePoolContext.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/pool/NCProbePoolContext.scala
new file mode 100644
index 0000000..0cc2691
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/pool/NCProbePoolContext.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.probe.mgrs.pool
+
+/**
+ * Probe pool context trait.
+ */
+trait NCProbePoolContext {
+ def getName: String = getClass.getName
+
+ implicit def getContext = NCProbePoolManager.getContext(getName)
+}
\ No newline at end of file
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/pool/NCProbePoolManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/pool/NCProbePoolManager.scala
new file mode 100644
index 0000000..68fe2e5
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/pool/NCProbePoolManager.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.probe.mgrs.pool
+
+import org.apache.nlpcraft.common.pool.NCPoolManager
+
+/**
+ * Probe pool manager.
+ */
+object NCProbePoolManager extends NCPoolManager("nlpcraft.probe.pools")
+
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/pool/NCServerPoolContext.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/pool/NCServerPoolContext.scala
new file mode 100644
index 0000000..b0800c8
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/pool/NCServerPoolContext.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.server.pool
+
+/**
+ * Server pool context trait.
+ */
+trait NCServerPoolContext {
+ def getName: String = getClass.getName
+
+ implicit def getContext = NCServerPoolManager.getContext(getName)
+}
\ No newline at end of file
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/pool/NCServerPoolManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/pool/NCServerPoolManager.scala
new file mode 100644
index 0000000..2e0f0e9
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/pool/NCServerPoolManager.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.server.pool
+
+import org.apache.nlpcraft.common.pool.NCPoolManager
+
+/**
+ * Server pool manager.
+ */
+object NCServerPoolManager extends NCPoolManager("nlpcraft.server.pools")
\ No newline at end of file