You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by ar...@apache.org on 2020/09/12 00:17:38 UTC
[incubator-nlpcraft] branch NLPCRAFT-41-config updated: WIP.
This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a commit to branch NLPCRAFT-41-config
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-41-config by this push:
new f96ed1a WIP.
f96ed1a is described below
commit f96ed1a479a939fb079255606bf7f8b8df333fea
Author: Aaron Radzinski <ar...@datalingvo.com>
AuthorDate: Fri Sep 11 17:04:01 2020 -0700
WIP.
---
.../org/apache/nlpcraft/model/NCModelView.java | 40 +++++++++++++---------
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 2 +-
...sationDescriptor.scala => NCConversation.scala} | 9 ++---
.../mgrs/conversation/NCConversationManager.scala | 19 +++++-----
.../probe/mgrs/deploy/NCDeployManager.scala | 27 +++++++++------
.../mgrs/dialogflow/NCDialogFlowManager.scala | 4 +--
.../nlpcraft/probe/mgrs/model/NCModelManager.scala | 11 ++++--
.../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 2 +-
8 files changed, 68 insertions(+), 46 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelView.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelView.java
index 8fdee78..87e9b07 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelView.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelView.java
@@ -39,33 +39,39 @@ public interface NCModelView extends NCMetadata {
int DFLT_JIGGLE_FACTOR = 2;
// TODO: comments
- int DFLT_SUSP_MANY_SYNONYMS = 1000;
- boolean DFLT_SUSP_MANY_SYNONYMS_ERROR = false;
+ int DFLT_MAX_SYNONYMS_THRESHOLD = 1000;
+
+ boolean DFLT_MAX_SYNONYMS_THRESHOLD_ERROR = false;
+
+ long DFLT_CONV_USAGE_TIMEOUT_MS = Duration.ofMinutes(60).toMillis();
+
+ long DFLT_CONV_UPDATE_TIMEOUT_MS = Duration.ofMinutes(5).toMillis();
- Duration DFLT_CONV_USAGE_TIMEOUT = Duration.ofMinutes(60);
- Duration DFLT_CONV_UPDATE_TIMEOUT = Duration.ofMinutes(5);
int DFLT_CONV_MAX_DEPTH = 3;
- Duration DFLT_DIALOG_TIMEOUT = Duration.ofMinutes(60);
+ long DFLT_DIALOG_TIMEOUT_MS = Duration.ofMinutes(60).toMillis();
// TODO: comments, review and move to the end of file.
- default int getSuspManySynonyms() {
- return DFLT_SUSP_MANY_SYNONYMS;
- }
- default boolean isSuspManySynonymsError() {
- return DFLT_SUSP_MANY_SYNONYMS_ERROR;
+ default int getMaxSynonymsThreshold() { return DFLT_MAX_SYNONYMS_THRESHOLD; }
+
+ default boolean isMaxSynonymsThresholdError() {
+ return DFLT_MAX_SYNONYMS_THRESHOLD_ERROR;
}
- default Duration getConvUsageTimeout() {
- return DFLT_CONV_USAGE_TIMEOUT;
+
+ default long getConversationTimeout() {
+ return DFLT_CONV_USAGE_TIMEOUT_MS;
}
- default Duration getConvUpdateTimeout() {
- return DFLT_CONV_UPDATE_TIMEOUT;
+
+ default long getConversationStmThreshold() {
+ return DFLT_CONV_UPDATE_TIMEOUT_MS;
}
- default int getConvMaxDepth() {
+
+ default int getConversationMaxDepth() {
return DFLT_CONV_MAX_DEPTH;
}
- default Duration getDialogTimeout() {
- return DFLT_DIALOG_TIMEOUT;
+
+ default long getDialogTimeout() {
+ return DFLT_DIALOG_TIMEOUT_MS;
}
/**
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
index 05b99b2..05c4367 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
@@ -108,7 +108,7 @@ object NCCommandManager extends NCService {
case "S2P_MODEL_INFO" ⇒
val mdlId = msg.data[String]("mdlId")
- val mdlData = NCModelManager.getModelData(mdlId).getOrElse(throw new NCE(s"Model not found: $mdlId"))
+ val mdlData = NCModelManager.getModelData(mdlId)
val macros = mdlData.model.getMacros.asInstanceOf[Serializable]
val syns = mdlData.model.getElements.asScala.map(p ⇒ p.getId → p.getSynonyms).toMap.asJava.asInstanceOf[Serializable]
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversationDescriptor.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversation.scala
similarity index 96%
rename from nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversationDescriptor.scala
rename to nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversation.scala
index 9e9f6bc..121e90e 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversationDescriptor.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversation.scala
@@ -31,9 +31,9 @@ import scala.collection.JavaConverters._
import scala.collection.mutable
/**
- * Conversation as an ordered set of utterances.
+ * An active conversation as an ordered set of utterances for the specific user and data model.
*/
-case class NCConversationDescriptor(
+case class NCConversation(
usrId: Long,
mdlId: String,
updateTimeoutMs: Long,
@@ -91,6 +91,7 @@ case class NCConversationDescriptor(
}
else if (attempt > maxDepth) {
stm.clear()
+
logger.info(s"Conversation is reset after too many unsuccessful requests [" +
s"usrId=$usrId, " +
s"mdlId=$mdlId" +
@@ -219,7 +220,7 @@ case class NCConversationDescriptor(
item.holders --= hs
logger.info(
- "Conversation tokens overridden by the \"group rule\" [" +
+ "Conversation tokens are overridden by the \"group rule\" [" +
s"usrId=$usrId, " +
s"mdlId=$mdlId, " +
s"srvReqId=$srvReqId, " +
@@ -255,7 +256,7 @@ case class NCConversationDescriptor(
tok.getServerRequestId
))
- logger.info(s"Conversation tokens [usrId=$usrId, mdlId=$mdlId]:\n${tbl.toString()}")
+ logger.info(s"Conversation tokens [mdlId=$mdlId, usrId=$usrId]:\n${tbl.toString()}")
}
/**
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversationManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversationManager.scala
index aea99a4..d27797e 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversationManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversationManager.scala
@@ -32,14 +32,14 @@ import scala.collection.mutable.ArrayBuffer
*/
object NCConversationManager extends NCService {
case class Key(userId: Long, mdlId: String)
- case class Value(conv: NCConversationDescriptor, var tstamp: Long = 0)
+ case class Value(conv: NCConversation, var tstamp: Long = 0)
private object Config extends NCConfigurable {
def periodMs: Long = getInt(s"nlpcraft.probe.conversation.check.period.secs") * 1000
def check(): Unit =
if (periodMs <= 0)
- abortWith(s"Value of 'nlpcraft.probe.conversation.check.period.secs' must be positive")
+ abortWith(s"Value of 'nlpcraft.probe.conversation.check.period.secs' must be positive.")
}
Config.check()
@@ -53,7 +53,7 @@ object NCConversationManager extends NCService {
gc.scheduleWithFixedDelay(() ⇒ clearForTimeout(), Config.periodMs, Config.periodMs, TimeUnit.MILLISECONDS)
- logger.info(s"Conversation manager GC started [checkPeriodMs=${Config.periodMs}]")
+ logger.info(s"Conversation manager GC started [checkPeriodMs=${Config.periodMs}]") // TODO
super.start()
}
@@ -61,7 +61,7 @@ object NCConversationManager extends NCService {
override def stop(parent: Span = null): Unit = startScopedSpan("stop", parent) { _ ⇒
U.shutdownPools(gc)
- logger.info("Conversation manager GC stopped")
+ logger.info("Conversation manager GC stopped.")
super.stop()
}
@@ -76,10 +76,11 @@ object NCConversationManager extends NCService {
val delKeys = ArrayBuffer.empty[Key]
for ((key, value) ← convs)
- NCModelManager.getModelData(key.mdlId) match {
+ NCModelManager.getModelDataOpt(key.mdlId) match {
case Some(data) ⇒
- if (value.tstamp < System.currentTimeMillis() -data.model.getConvUsageTimeout.toMillis)
+ if (value.tstamp < System.currentTimeMillis() - data.model.getConversationTimeout)
delKeys += key
+
case None ⇒ delKeys += key
}
@@ -98,14 +99,14 @@ object NCConversationManager extends NCService {
* @param mdlId Model ID.
* @return New or existing conversation.
*/
- def getConversation(usrId: Long, mdlId: String, parent: Span = null): NCConversationDescriptor =
+ def getConversation(usrId: Long, mdlId: String, parent: Span = null): NCConversation =
startScopedSpan("getConversation", parent, "usrId" → usrId, "modelId" → mdlId) { _ ⇒
- val mdl = NCModelManager.getModelData(mdlId).getOrElse(throw new NCE(s"Model not found: $mdlId")).model
+ val mdl = NCModelManager.getModelData(mdlId).model
convs.synchronized {
val v = convs.getOrElseUpdate(
Key(usrId, mdlId),
- Value(NCConversationDescriptor(usrId, mdlId, mdl.getConvUpdateTimeout.toMillis, mdl.getConvMaxDepth))
+ Value(NCConversation(usrId, mdlId, mdl.getConversationStmThreshold, mdl.getConversationMaxDepth))
)
v.tstamp = U.nowUtcMs()
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
index 071e308..15fd089 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
@@ -796,16 +796,23 @@ object NCDeployManager extends NCService with DecorateAsScala {
if (value == null)
throw new NCE(s"$name is not provided [modeId=$mdlId]")
- checkNotNull(mdl.getConvUsageTimeout, "Conversation usage timeout")
- checkNotNull(mdl.getConvUpdateTimeout, "Conversation update timeout")
- checkNotNull(mdl.getDialogTimeout, "Dialog timeout")
+ checkLong(mdl.getConversationTimeout, "Conversation timeout")
+ checkLong(mdl.getConversationStmThreshold, "Conversation STM threshold")
+ checkLong(mdl.getDialogTimeout, "Dialog timeout")
@throws[NCE]
- def checkInt(v: Int, name: String, min: Int = 0, max: Int = Integer.MAX_VALUE): Unit =
+ def checkInt(v: Int, name: String, min: Int = 0, max: Int = Int.MaxValue): Unit =
if (v < min)
- throw new NCE(s"Invalid model configuration value '$name' [value=$v, min=$min, modelId=$mdlId]")
+ throw new NCE(s"Invalid 'int' model configuration value '$name' [value=$v, min=$min, modelId=$mdlId]")
else if (v > max)
- throw new NCE(s"Invalid model configuration value '$name' [value=$v, max=$min, modelId=$mdlId]")
+ throw new NCE(s"Invalid 'int' model configuration value '$name' [value=$v, max=$min, modelId=$mdlId]")
+
+ @throws[NCE]
+ def checkLong(v: Long, name: String, min: Long = 0L, max: Long = Long.MaxValue): Unit =
+ if (v < min)
+ throw new NCE(s"Invalid 'long' model configuration value '$name' [value=$v, min=$min, modelId=$mdlId]")
+ else if (v > max)
+ throw new NCE(s"Invalid 'long' model configuration value '$name' [value=$v, max=$min, modelId=$mdlId]")
checkInt(mdl.getMaxUnknownWords, "maxUnknownWords")
checkInt(mdl.getMaxFreeWords, "maxFreeWords")
@@ -816,8 +823,8 @@ object NCDeployManager extends NCService with DecorateAsScala {
checkInt(mdl.getMaxTokens, "maxTokens", max = 100)
checkInt(mdl.getMaxWords, "maxWords", min = 1, max = 100)
checkInt(mdl.getJiggleFactor, "jiggleFactor", max = 4)
- checkInt(mdl.getSuspManySynonyms, "suspManySynonyms", min = 1)
- checkInt(mdl.getConvMaxDepth, "convMaxDepth", min = 1)
+ checkInt(mdl.getMaxSynonymsThreshold, "suspManySynonyms", min = 1)
+ checkInt(mdl.getConversationMaxDepth, "convMaxDepth", min = 1)
@throws[NCE]
def checkCollection(name: String, col: Any): Unit =
@@ -936,12 +943,12 @@ object NCDeployManager extends NCService with DecorateAsScala {
if (size == 0)
logger.warn(s"Element '$elemId' doesn't have synonyms [modelId=$mdlId]")
- else if (size > mdl.getSuspManySynonyms) {
+ else if (size > mdl.getMaxSynonymsThreshold) {
val msg =
s"Element '$elemId' has too many ($size) synonyms. " +
s"Make sure this is truly necessary [modelId=$mdlId]"
- if (mdl.isSuspManySynonymsError)
+ if (mdl.isMaxSynonymsThresholdError)
throw new NCE(msg)
else
logger.warn(msg)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/dialogflow/NCDialogFlowManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/dialogflow/NCDialogFlowManager.scala
index b18a57e..6944e6d 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/dialogflow/NCDialogFlowManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/dialogflow/NCDialogFlowManager.scala
@@ -110,9 +110,9 @@ object NCDialogFlowManager extends NCService {
val delKeys = ArrayBuffer.empty[Key]
for ((key, values) ← flow)
- NCModelManager.getModelData(key.mdlId) match {
+ NCModelManager.getModelDataOpt(key.mdlId) match {
case Some(data) ⇒
- val ms = System.currentTimeMillis() -data.model.getDialogTimeout.toMillis
+ val ms = System.currentTimeMillis() -data.model.getDialogTimeout
values --= values.filter(_.tstamp < ms)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
index f239ac0..be6e8d6 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
@@ -113,8 +113,15 @@ object NCModelManager extends NCService with DecorateAsScala {
*
* @param mdlId Model ID.
*/
- def getModelData(mdlId: String, parent: Span = null): Option[NCModelData] =
- startScopedSpan("getModel", parent, "modelId" → mdlId) { _ ⇒
+ def getModelDataOpt(mdlId: String, parent: Span = null): Option[NCModelData] =
+ startScopedSpan("getModelOpt", parent, "modelId" → mdlId) { _ ⇒
mux.synchronized { data.get(mdlId) }
}
+
+ /**
+ *
+ * @param mdlId Model ID.
+ */
+ def getModelData(mdlId: String, parent: Span = null): NCModelData =
+ getModelDataOpt(mdlId, parent).getOrElse(throw new NCE(s"Model not found: $mdlId"))
}
\ No newline at end of file
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
index f1ff8c1..f30e729 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
@@ -315,7 +315,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats {
logger.info(s"REJECT response $msgName sent [srvReqId=$srvReqId, response=${errMsg.get}]")
}
- val mdlData = NCModelManager.getModelData(mdlId, span).getOrElse(throw new NCE(s"Model not found: $mdlId"))
+ val mdlData = NCModelManager.getModelData(mdlId, span)
var errData: Option[(String, Int)] = None