You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2022/02/18 09:37:26 UTC
[incubator-nlpcraft] branch NLPCRAFT-477 updated (9c34ab1 -> 613a751)
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a change to branch NLPCRAFT-477
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
from 9c34ab1 WIP.
new 24a831f WIP.
new 613a751 WIP.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../scala/org/apache/nlpcraft/NCConversation.java | 2 +-
...sationHolder.scala => NCConversationData.scala} | 23 +++--------
.../conversation/NCConversationManager.scala | 7 ++--
.../internal/dialogflow/NCDialogFlowManager.scala | 5 ++-
.../nlpcraft/internal/impl/NCModelClientImpl.scala | 44 +++++++++++++---------
.../internal/impl/NCModelPipelineManager.scala | 11 +++---
.../conversation/NCConversationManagerSpec.scala | 12 +++---
.../dialogflow/NCDialogFlowManagerSpec.scala | 3 +-
8 files changed, 51 insertions(+), 56 deletions(-)
rename nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/{NCConversationHolder.scala => NCConversationData.scala} (92%)
[incubator-nlpcraft] 01/02: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-477
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 24a831fdc07336760b0684531c8e3f86bf5073b2
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Fri Feb 18 12:13:11 2022 +0300
WIP.
---
.../nlpcraft/internal/impl/NCModelClientImpl.scala | 44 ++++++++++++----------
.../internal/impl/NCModelPipelineManager.scala | 11 +++---
2 files changed, 30 insertions(+), 25 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/impl/NCModelClientImpl.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/impl/NCModelClientImpl.scala
index 2666a8b..6990e5a 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/impl/NCModelClientImpl.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/impl/NCModelClientImpl.scala
@@ -56,10 +56,12 @@ class NCModelClientImpl(mdl: NCModel) extends LazyLogging:
verify()
private val intents = NCModelScanner.scan(mdl)
- private val pipelineMgr = new NCModelPipelineManager(mdl.getConfig, mdl.getPipeline)
private val convMgr = NCConversationManager(mdl.getConfig)
- private val dialogMgr = NCDialogFlowManager(mdl.getConfig)
- private val intentsMgr = NCIntentsManager(dialogMgr, intents.map(p => p.intent -> p.function).toMap)
+ private val dlgMgr = NCDialogFlowManager(mdl.getConfig)
+ private val plMgr = new NCModelPipelineManager(mdl.getConfig, mdl.getPipeline)
+ private val intentsMgr = NCIntentsManager(dlgMgr, intents.map(p => p.intent -> p.function).toMap)
+
+ init()
/**
*
@@ -82,10 +84,19 @@ class NCModelClientImpl(mdl: NCModel) extends LazyLogging:
/**
*
+ */
+ private def init(): Unit =
+ convMgr.start()
+ dlgMgr.start()
+ plMgr.start()
+
+
+ /**
+ *
* @param data
* @return
*/
- private def ask0(data: NCPipelineVariants): NCResult =
+ private def ask0(data: NCPipelineData): NCResult =
val userId = data.request.getUserId
val convHldr = convMgr.getConversation(userId)
val allEnts = data.variants.flatMap(_.getEntities.asScala)
@@ -94,9 +105,9 @@ class NCModelClientImpl(mdl: NCModel) extends LazyLogging:
new NCConversation:
override val getSession: NCPropertyMap = convHldr.getUserData
override val getStm: JList[NCEntity] = convHldr.getEntities
- override val getDialogFlow: JList[NCDialogFlowItem] = dialogMgr.getDialogFlow(userId).asJava
+ override val getDialogFlow: JList[NCDialogFlowItem] = dlgMgr.getDialogFlow(userId).asJava
override def clearStm(filter: Predicate[NCEntity]): Unit = convHldr.clearEntities(filter)
- override def clearDialog(filter: Predicate[String]): Unit = dialogMgr.clearForPredicate(userId, (s: String) => filter.test(s))
+ override def clearDialog(filter: Predicate[String]): Unit = dlgMgr.clearForPredicate(userId, (s: String) => filter.test(s))
val ctx: NCContext =
new NCContext:
@@ -118,10 +129,9 @@ class NCModelClientImpl(mdl: NCModel) extends LazyLogging:
*/
def ask(txt: String, data: JMap[String, AnyRef], usrId: String): CompletableFuture[NCResult] =
val fut = new CompletableFuture[NCResult]
- val check = () => if fut.isCancelled then
- E(s"Asynchronous ask is interrupted [txt=$txt, usrId=$usrId]")
+ val check = () => if fut.isCancelled then E(s"Asynchronous ask is interrupted [txt=$txt, usrId=$usrId]")
- fut.completeAsync(() => ask0(pipelineMgr.prepare(txt, data, usrId, Option(check))))
+ fut.completeAsync(() => ask0(plMgr.prepare(txt, data, usrId, Option(check))))
/**
*
@@ -130,25 +140,21 @@ class NCModelClientImpl(mdl: NCModel) extends LazyLogging:
* @param usrId
* @return
*/
- def askSync(txt: String, data: JMap[String, AnyRef], usrId: String): NCResult =
- ask0(pipelineMgr.prepare(txt, data, usrId))
+ def askSync(txt: String, data: JMap[String, AnyRef], usrId: String): NCResult = ask0(plMgr.prepare(txt, data, usrId))
/**
*
* @param usrId
*/
- def clearConversation(usrId: String): Unit = convMgr.getConversation(usrId)
+ def clearConversation(usrId: String): Unit = convMgr.getConversation(usrId).clearEntities(_ => true)
/**
*
* @param usrId
*/
- def clearDialog(usrId: String): Unit = dialogMgr.clear(usrId)
+ def clearDialog(usrId: String): Unit = dlgMgr.clear(usrId)
- /**
- *
- */
def close(): Unit =
- if pipelineMgr != null then pipelineMgr.close()
- if convMgr != null then convMgr.close()
- if dialogMgr != null then dialogMgr.close()
+ plMgr.close()
+ dlgMgr.close()
+ convMgr.close()
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/impl/NCModelPipelineManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/impl/NCModelPipelineManager.scala
index d2758c9..1a43827 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/impl/NCModelPipelineManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/impl/NCModelPipelineManager.scala
@@ -42,7 +42,7 @@ import scala.jdk.OptionConverters.*
* @param tokens
* @param checkCancel
*/
-case class NCPipelineVariants(request: NCRequest, variants: Seq[NCVariant], tokens: JList[NCToken], checkCancel: Option[() => Unit])
+case class NCPipelineData(request: NCRequest, variants: Seq[NCVariant], tokens: JList[NCToken], checkCancel: Option[() => Unit])
/**
*
@@ -62,9 +62,6 @@ class NCModelPipelineManager(cfg: NCModelConfig, pipeline: NCModelPipeline) exte
private val allSrvs: Seq[NCLifecycle] =
tokEnrichers ++ entEnrichers ++ entParsers ++ tokVals ++ entVals ++ varFilterOpt.toSeq
- processServices(_.onStart(cfg), "started")
-
-
/**
*
* @param act
@@ -94,7 +91,7 @@ class NCModelPipelineManager(cfg: NCModelConfig, pipeline: NCModelPipeline) exte
* @param checkCancel
* @return
*/
- def prepare(txt: String, data: JMap[String, AnyRef], usrId: String, checkCancel: Option[() => Unit] = None): NCPipelineVariants =
+ def prepare(txt: String, data: JMap[String, AnyRef], usrId: String, checkCancel: Option[() => Unit] = None): NCPipelineData =
require(txt != null && usrId != null)
/**
@@ -166,7 +163,9 @@ class NCModelPipelineManager(cfg: NCModelConfig, pipeline: NCModelPipeline) exte
check()
variants = varFilterOpt.get.filter(req, cfg, variants)
- NCPipelineVariants(req, variants.asScala.toSeq, toks, checkCancel)
+ NCPipelineData(req, variants.asScala.toSeq, toks, checkCancel)
+
+ def start(): Unit = processServices(_.onStart(cfg), "started")
/**
*
*/
[incubator-nlpcraft] 02/02: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-477
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 613a751e7777f46470240c14353f018b7641be67
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Fri Feb 18 12:36:00 2022 +0300
WIP.
---
.../scala/org/apache/nlpcraft/NCConversation.java | 2 +-
...sationHolder.scala => NCConversationData.scala} | 23 ++++++----------------
.../conversation/NCConversationManager.scala | 7 +++----
.../internal/dialogflow/NCDialogFlowManager.scala | 5 +++--
.../nlpcraft/internal/impl/NCModelClientImpl.scala | 12 ++++++-----
.../conversation/NCConversationManagerSpec.scala | 12 +++++------
.../dialogflow/NCDialogFlowManagerSpec.scala | 3 ++-
7 files changed, 27 insertions(+), 37 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCConversation.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCConversation.java
index e2dcfb5..7856601 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCConversation.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCConversation.java
@@ -60,5 +60,5 @@ public interface NCConversation {
*
* @param filter Dialog flow filter.
*/
- void clearDialog(Predicate<String> filter); // TODO: was NCDialogFlowItem
+ void clearDialog(Predicate<NCDialogFlowItem> filter);
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationHolder.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationData.scala
similarity index 92%
rename from nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationHolder.scala
rename to nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationData.scala
index 6f5f13c..f9d97ff 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationHolder.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationData.scala
@@ -32,7 +32,7 @@ import scala.jdk.CollectionConverters.*
/**
* An active conversation is an ordered set of utterances for the specific user and data model.
*/
-case class NCConversationHolder(
+case class NCConversationData(
usrId: String,
mdlId: String,
timeoutMs: Long,
@@ -111,7 +111,7 @@ case class NCConversationHolder(
*
* @param p Java-side predicate.
*/
- def clearEntities(p: Predicate[NCEntity]): Unit =
+ def clear(p: Predicate[NCEntity]): Unit =
stm.synchronized {
for (item <- stm) item.holders --= item.holders.filter(h => p.test(h.entity))
squeezeEntities()
@@ -121,16 +121,6 @@ case class NCConversationHolder(
logger.trace(s"STM is cleared [usrId=$usrId, mdlId=$mdlId]")
/**
- * Clears all entities from this conversation satisfying given predicate.
- *
- * @param p Scala-side predicate.
- */
- def clearEntities(p: NCEntity => Boolean): Unit =
- clearEntities(new Predicate[NCEntity]:
- override def test(t: NCEntity): Boolean = p(t)
- )
-
- /**
*
* @param ents
*/
@@ -206,16 +196,15 @@ case class NCConversationHolder(
*
* @return
*/
- def getEntities: util.List[NCEntity] =
+ def getEntities: Seq[NCEntity] =
+ // TODO: copy?
stm.synchronized {
val reqIds = ctx.map(_.getRequestId).distinct.zipWithIndex.toMap
- val ents = ctx.groupBy(_.getRequestId).toSeq.sortBy(p => reqIds(p._1)).reverse.flatMap(_._2)
-
- new util.ArrayList[NCEntity](ents.asJava)
+ ctx.groupBy(_.getRequestId).toSeq.sortBy(p => reqIds(p._1)).reverse.flatMap(_._2)
}
/**
- *
+ * TODO: thread safe?
*/
val getUserData: NCPropertyMap = data
}
\ No newline at end of file
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationManager.scala
index 77574da..1b1d052 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/conversation/NCConversationManager.scala
@@ -28,22 +28,21 @@ import scala.jdk.CollectionConverters.*
* Conversation manager.
*/
class NCConversationManager(cfg: NCModelConfig) extends LazyLogging:
- case class Value(conv: NCConversationHolder, var tstamp: Long = 0)
+ case class Value(conv: NCConversationData, var tstamp: Long = 0)
private final val convs: mutable.Map[String, Value] = mutable.HashMap.empty[String, Value]
@volatile private var gc: Thread = _
-
/**
* Gets conversation for given user ID.
*
* @param usrId User ID.
* @return New or existing conversation.
*/
- def getConversation(usrId: String): NCConversationHolder =
+ def getConversation(usrId: String): NCConversationData =
convs.synchronized {
val v = convs.getOrElseUpdate(
usrId,
- Value(NCConversationHolder(usrId, cfg.getId, cfg.getConversationTimeout, cfg.getConversationDepth))
+ Value(NCConversationData(usrId, cfg.getId, cfg.getConversationTimeout, cfg.getConversationDepth))
)
v.tstamp = NCUtils.nowUtcMs()
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/dialogflow/NCDialogFlowManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/dialogflow/NCDialogFlowManager.scala
index 1026e3d..adac2f1 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/dialogflow/NCDialogFlowManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/dialogflow/NCDialogFlowManager.scala
@@ -117,6 +117,7 @@ class NCDialogFlowManager(cfg: NCModelConfig) extends LazyLogging:
* @return Dialog flow.
*/
def getDialogFlow(usrId: String): Seq[NCDialogFlowItem] =
+ // TODO: copy?
flow.synchronized { flow.get(usrId) } match
case Some(buf) => buf.toSeq
case None => Seq.empty
@@ -166,8 +167,8 @@ class NCDialogFlowManager(cfg: NCModelConfig) extends LazyLogging:
* @param pred Intent ID predicate.
* @param parent Parent span, if any.
*/
- def clearForPredicate(usrId: String, pred: String => Boolean): Unit =
+ def clear(usrId: String, pred: NCDialogFlowItem => Boolean): Unit =
flow.synchronized {
- flow(usrId) = flow(usrId).filterNot(v => pred(v.getIntentMatch.getIntentId))
+ flow(usrId) = flow(usrId).filterNot(pred)
flow.notifyAll()
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/impl/NCModelClientImpl.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/impl/NCModelClientImpl.scala
index 6990e5a..facdabf 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/impl/NCModelClientImpl.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/impl/NCModelClientImpl.scala
@@ -90,7 +90,6 @@ class NCModelClientImpl(mdl: NCModel) extends LazyLogging:
dlgMgr.start()
plMgr.start()
-
/**
*
* @param data
@@ -104,10 +103,10 @@ class NCModelClientImpl(mdl: NCModel) extends LazyLogging:
val conv: NCConversation =
new NCConversation:
override val getSession: NCPropertyMap = convHldr.getUserData
- override val getStm: JList[NCEntity] = convHldr.getEntities
+ override val getStm: JList[NCEntity] = convHldr.getEntities.asJava
override val getDialogFlow: JList[NCDialogFlowItem] = dlgMgr.getDialogFlow(userId).asJava
- override def clearStm(filter: Predicate[NCEntity]): Unit = convHldr.clearEntities(filter)
- override def clearDialog(filter: Predicate[String]): Unit = dlgMgr.clearForPredicate(userId, (s: String) => filter.test(s))
+ override def clearStm(filter: Predicate[NCEntity]): Unit = convHldr.clear(filter)
+ override def clearDialog(filter: Predicate[NCDialogFlowItem]): Unit = dlgMgr.clear(userId, (s: NCDialogFlowItem) => filter.test(s))
val ctx: NCContext =
new NCContext:
@@ -146,7 +145,7 @@ class NCModelClientImpl(mdl: NCModel) extends LazyLogging:
*
* @param usrId
*/
- def clearConversation(usrId: String): Unit = convMgr.getConversation(usrId).clearEntities(_ => true)
+ def clearConversation(usrId: String): Unit = convMgr.getConversation(usrId).clear(_ => true)
/**
*
@@ -154,6 +153,9 @@ class NCModelClientImpl(mdl: NCModel) extends LazyLogging:
*/
def clearDialog(usrId: String): Unit = dlgMgr.clear(usrId)
+ /**
+ *
+ */
def close(): Unit =
plMgr.close()
dlgMgr.close()
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/conversation/NCConversationManagerSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/conversation/NCConversationManagerSpec.scala
index 4a3de41..c402a99 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/conversation/NCConversationManagerSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/conversation/NCConversationManagerSpec.scala
@@ -39,7 +39,7 @@ class NCConversationManagerSpec:
val conv = mgr.getConversation("user1")
def checkSize(size: Int): Unit =
- require(conv.getEntities.size() == size, s"Unexpected entities size: ${conv.getEntities.size()}, expected: $size")
+ require(conv.getEntities.sizeIs == size, s"Unexpected entities size: ${conv.getEntities.size}, expected: $size")
// Initial empty.
checkSize(0)
@@ -53,11 +53,9 @@ class NCConversationManagerSpec:
checkSize(2)
// Partially cleared.
- conv.clearEntities(new Predicate[NCEntity]:
- override def test(t: NCEntity): Boolean = t.getId == "e1"
- )
+ conv.clear(_.getId == "e1")
checkSize(1)
- require(conv.getEntities.get(0).getId == "e2")
+ require(conv.getEntities.head.getId == "e2")
@Test
def testTimeout(): Unit =
@@ -68,11 +66,11 @@ class NCConversationManagerSpec:
val reqId = "req1"
// TODO: Drop method and use saved conversation instead - error is thrown
- def getConversation: NCConversationHolder = mgr.getConversation("user1")
+ def getConversation: NCConversationData = mgr.getConversation("user1")
def checkSize(size: Int): Unit =
val conv = getConversation
- require(conv.getEntities.size() == size, s"Unexpected entities size: ${conv.getEntities.size()}, expected: $size")
+ require(conv.getEntities.sizeIs == size, s"Unexpected entities size: ${conv.getEntities.size}, expected: $size")
// Initial empty.
checkSize(0)
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/dialogflow/NCDialogFlowManagerSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/dialogflow/NCDialogFlowManagerSpec.scala
index 01d2ef7..b8d0736 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/dialogflow/NCDialogFlowManagerSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/internal/dialogflow/NCDialogFlowManagerSpec.scala
@@ -23,6 +23,7 @@ import org.apache.nlpcraft.nlp.util.NCTestRequest
import org.junit.jupiter.api.*
import java.util
+import java.util.function.Predicate
/**
*
@@ -99,7 +100,7 @@ class NCDialogFlowManagerSpec:
mgr.clear(usrId = "user1")
check("user1" -> 0, "user2" -> 2, "user3" -> 1, "user4" -> 0)
- mgr.clearForPredicate(usrId = "user2", _ == "i21")
+ mgr.clear(usrId = "user2", _.getIntentMatch.getIntentId == "i21")
check("user1" -> 0, "user2" -> 1, "user3" -> 1, "user4" -> 0)
mgr.clear(usrId = "user2")