You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by ar...@apache.org on 2020/10/19 23:15:37 UTC
[incubator-nlpcraft] branch NLPCRAFT-158 updated: Fix for
NLPCRAFT-158 + misc. fixes.
This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a commit to branch NLPCRAFT-158
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-158 by this push:
new d7f1310 Fix for NLPCRAFT-158 + misc. fixes.
d7f1310 is described below
commit d7f1310466e1fdbf11e8eae61d6f34859750a0c1
Author: Aaron Radzinski <ar...@datalingvo.com>
AuthorDate: Mon Oct 19 16:15:27 2020 -0700
Fix for NLPCRAFT-158 + misc. fixes.
---
.../org/apache/nlpcraft/common/util/NCUtils.scala | 18 +-
.../nlpcraft/model/tools/cmdline/NCCli.scala | 184 +++++++++++++++++----
.../probe/mgrs/conversation/NCConversation.scala | 39 ++---
.../mgrs/dialogflow/NCDialogFlowManager.scala | 15 +-
.../nlpcraft/server/probe/NCProbeManager.scala | 30 ++--
5 files changed, 219 insertions(+), 67 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
index 15a5c38..9d0349a 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
@@ -1269,12 +1269,23 @@ object NCUtils extends LazyLogging {
Option(scala.util.Try { Jsoup.connect(url).get() } getOrElse null)
/**
- * Records GA screen view event. Ignores any errors.
+ * Records anonymous GA screen view event. Ignores any errors.
*
* @param cd Content description for GA measurement protocol.
*/
def gaScreenView(cd: String): Unit =
- try
+ try {
+ val anonym = NetworkInterface.getByInetAddress(InetAddress.getLocalHost) match {
+ case null ⇒ 555
+ case nif ⇒
+ val addr = nif.getHardwareAddress
+
+ if (addr == null)
+ 555
+ else
+ addr.mkString(",").hashCode
+ }
+
HttpClient.newHttpClient.send(
HttpRequest.newBuilder()
.uri(
@@ -1285,7 +1296,7 @@ object NCUtils extends LazyLogging {
s"v=1&" +
s"t=screenview&" +
s"tid=UA-180663034-1&" + // 'nlpcraft.apache.org' web property.
- s"cid=555&" + // Hide any user information (anonymous user).
+ s"cid=$anonym&" + // Hide any user information (anonymous user).
s"aip=&" + // Hide user IP (anonymization).
s"an=nlpcraft&" +
s"av=${NCVersion.getCurrent.version}&" +
@@ -1296,6 +1307,7 @@ object NCUtils extends LazyLogging {
.build(),
HttpResponse.BodyHandlers.ofString()
)
+ }
catch {
case _: Exception ⇒ () // Ignore.
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCli.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCli.scala
index a39434d..9b98b82 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCli.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCli.scala
@@ -156,7 +156,7 @@ object NCCli extends App {
case class InvalidJsonParameter(cmd: Command, param: String)
extends IllegalArgumentException(
s"Invalid JSON parameter $C${"'" + param + "'"}$RST, " +
- s"type $C'help --cmd=${cmd.name}'$RST to get help."
+ s"type $C'help --cmd=${cmd.name}'$RST to get help."
)
case class HttpError(httpCode: Int)
extends IllegalStateException(s"REST error (HTTP ${c(httpCode)}).")
@@ -691,6 +691,7 @@ object NCCli extends App {
usage = Seq(
s"$PROMPT $SCRIPT_NAME call --path=ask/sync",
" --acsTok=qwerty123456",
+ " --txt=\"User request\"",
" --mdlId=my.model.id",
" --data='{\"data1\": true, \"data2\": 123, \"data3\": \"some text\"}'",
" --enableLog=false"
@@ -701,6 +702,58 @@ object NCCli extends App {
)
),
Command(
+ name = "ask",
+ group = "2. REST Commands",
+ synopsis = s"Wrapper for REST ${c("/ask/sync")} call.",
+ desc = Some(
+ s"Requires user to be already signed in. This command ${bo("only makes sense in the REPL mode")} as " +
+ s"it requires user to be signed in. REPL session keeps the currently active access " +
+ s"token after user signed in. For command line mode, use ${c("'rest'")} command with " +
+ s"corresponding parameters."
+ ),
+ body = cmdAsk,
+ params = Seq(
+ Parameter(
+ id = "mdlId",
+ names = Seq("--mdlId"),
+ value = Some("model.id"),
+ desc =
+ s"ID of the data model to send the request to. " +
+ s"In REPL mode, hit ${rv(" Tab ")} to see auto-suggestion for possible model IDs."
+ ),
+ Parameter(
+ id = "txt",
+ names = Seq("--txt"),
+ value = Some("txt"),
+ desc =
+ s"Text of the question."
+ ),
+ Parameter(
+ id = "data",
+ names = Seq("--data"),
+ value = Some("'{}'"),
+ optional = true,
+ desc = s"Additional JSON data with maximum JSON length of 512000 bytes. Default is ${c("'null'")}."
+ ),
+ Parameter(
+ id = "enableLog",
+ names = Seq("--enableLog"),
+ value = Some("true|false"),
+ optional = true,
+ desc = s"Flag to enable detailed processing log to be returned with the result. Default is ${c("'false'")}."
+ )
+ ),
+ examples = Seq(
+ Example(
+ usage = Seq(
+ s"""> ask --txt="User request" --mdlId=my.model.id"""
+ ),
+ desc =
+ s"Issues ${y("'ask/sync'")} REST call with given text and model ID."
+ )
+ )
+ ),
+ Command(
name = "tail-server",
group = "1. Server Commands",
synopsis = s"Shows last N lines from the local REST server log.",
@@ -1050,6 +1103,7 @@ object NCCli extends App {
private final val HELP_CMD = CMDS.find(_.name == "help").get
private final val REST_CMD = CMDS.find(_.name == "rest").get
private final val CALL_CMD = CMDS.find(_.name == "call").get
+ private final val ASK_CMD = CMDS.find(_.name == "ask").get
private final val STOP_SRV_CMD = CMDS.find(_.name == "stop-server").get
private final val START_SRV_CMD = CMDS.find(_.name == "start-server").get
@@ -1636,6 +1690,9 @@ object NCCli extends App {
lines
}
+ def helpHelp(): Unit =
+ logln(s"\nType ${c("help --cmd=xxx")} to get help for ${c("xxx")} command.")
+
if (args.isEmpty) { // Default - show abbreviated help.
if (!repl)
header()
@@ -1653,6 +1710,8 @@ object NCCli extends App {
logln(s"\n$B$grp:$RST\n${tbl.toString}")
})
+
+ helpHelp()
}
else if (args.size == 1 && args.head.parameter.id == "all") { // Show a full format help for all commands.
if (!repl)
@@ -1668,6 +1727,8 @@ object NCCli extends App {
)
logln(tbl.toString)
+
+ helpHelp()
}
else { // Help for individual commands.
var err = false
@@ -1881,23 +1942,62 @@ object NCCli extends App {
* @param args Arguments, if any, for this command.
* @param repl Whether or not executing from REPL.
*/
- private def cmdSignIn(cmd: Command, args: Seq[Argument], repl: Boolean): Unit = {
- if (state.accessToken.nonEmpty)
- error("Already signed in.")
+ private def cmdSignIn(cmd: Command, args: Seq[Argument], repl: Boolean): Unit =
+ state.accessToken match {
+ case None ⇒
+ val email = args.find(_.parameter.id == "email").flatMap(_.value).getOrElse("admin@admin.com")
+ val passwd = args.find(_.parameter.id == "passwd").flatMap(_.value).getOrElse("admin")
+
+ httpRest(
+ cmd,
+ "signin",
+ s"""
+ |{
+ | "email": ${jsonQuote(email)},
+ | "passwd": ${jsonQuote(passwd)}
+ |}
+ |""".stripMargin
+ )
+
+ case Some(_) ⇒ error(s"Already signed in. See ${c("'signout'")} command.")
+ }
+
+ /**
+ *
+ * @param cmd Command descriptor.
+ * @param args Arguments, if any, for this command.
+ * @param repl Whether or not executing from REPL.
+ */
+ private def cmdSignOut(cmd: Command, args: Seq[Argument], repl: Boolean): Unit =
+ state.accessToken match {
+ case Some(acsTok) ⇒
+ httpRest(
+ cmd,
+ "signout",
+ s"""
+ |{"acsTok": ${jsonQuote(acsTok)}}
+ |""".stripMargin
+ )
+
+ case None ⇒ error(s"Not signed in. See ${c("'signin'")} command.")
+ }
+
+ /**
+ * Quotes given string in double quotes unless it is already quoted as such.
+ *
+ * @param s
+ * @return
+ */
+ private def jsonQuote(s: String): String = {
+ if (s == null)
+ null
else {
- val email = args.find(_.parameter.id == "email").flatMap(_.value).getOrElse("admin@admin.com")
- val passwd = args.find(_.parameter.id == "passwd").flatMap(_.value).getOrElse("admin")
-
- httpRest(
- cmd,
- "signin",
- s"""
- |{
- |"email": "$email",
- |"passwd": "$passwd"
- |}
- |""".stripMargin
- )
+ val ss = s.trim()
+
+ if (ss.startsWith("\"") && ss.endsWith("\""))
+ ss
+ else
+ s""""$ss""""
}
}
@@ -1907,18 +2007,30 @@ object NCCli extends App {
* @param args Arguments, if any, for this command.
* @param repl Whether or not executing from REPL.
*/
- private def cmdSignOut(cmd: Command, args: Seq[Argument], repl: Boolean): Unit = {
- if (state.accessToken.isEmpty)
- error("Not signed in.")
- else
- httpRest(
- cmd,
- "signout",
- s"""
- |{"acsTok": "${state.accessToken.getOrElse("")}"}
- |""".stripMargin
- )
- }
+ private def cmdAsk(cmd: Command, args: Seq[Argument], repl: Boolean): Unit =
+ state.accessToken match {
+ case Some(acsTok) ⇒
+ val mdlId = args.find(_.parameter.id == "mdlId").flatMap(_.value).getOrElse(throw MissingParameter(cmd, "mdlId"))
+ val txt = args.find(_.parameter.id == "txt").flatMap(_.value).getOrElse(throw MissingParameter(cmd, "txt"))
+ val data = args.find(_.parameter.id == "data").flatMap(_.value).orNull
+ val enableLog = args.find(_.parameter.id == "enableLog").flatMap(_.value).getOrElse(false)
+
+ httpRest(
+ cmd,
+ "ask/sync",
+ s"""
+ |{
+ | "acsTok": ${jsonQuote(acsTok)},
+ | "mdlId": ${jsonQuote(mdlId)},
+ | "txt": ${jsonQuote(txt)},
+ | "data": ${jsonQuote(data)},
+ | "enableLog": $enableLog
+ |}
+ |""".stripMargin
+ )
+
+ case None ⇒ error(s"Not signed in. See ${c("'signin'")} command.")
+ }
/**
*
@@ -2127,6 +2239,20 @@ object NCCli extends App {
)
}
+ // For 'ask' - add additional auto-completion/suggestion candidates.
+ if (cmd == ASK_CMD.name)
+ candidates.addAll(
+ state.probes.flatMap(_.models.toList).map(mdl ⇒ {
+ mkCandidate(
+ disp = s"--mdlId=${mdl.id}",
+ grp = MANDATORY_GRP,
+ desc = null,
+ completed = true
+ )
+ })
+ .asJava
+ )
+
// For 'call' - add additional auto-completion/suggestion candidates.
if (cmd == CALL_CMD.name) {
val pathParam = CALL_CMD.findParameterById("path")
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversation.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversation.scala
index e68f2bb..51f2236 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversation.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conversation/NCConversation.scala
@@ -89,7 +89,7 @@ case class NCConversation(
if (now - lastUpdateTstamp > timeoutMs) {
stm.clear()
- logger.info(s"Conversation is reset by timeout [" +
+ logger.trace(s"Conversation is reset by timeout [" +
s"usrId=$usrId, " +
s"mdlId=$mdlId" +
s"]")
@@ -97,7 +97,7 @@ case class NCConversation(
else if (depth > maxDepth) {
stm.clear()
- logger.info(s"Conversation is reset after reaching max depth [" +
+ logger.trace(s"Conversation is reset after reaching max depth [" +
s"usrId=$usrId, " +
s"mdlId=$mdlId" +
s"]")
@@ -114,7 +114,7 @@ case class NCConversation(
if (delHs.nonEmpty) {
item.holders --= delHs
- logger.info(
+ logger.trace(
s"Conversation overridden tokens removed [" +
s"usrId=$usrId, " +
s"mdlId=$mdlId, " +
@@ -142,21 +142,22 @@ case class NCConversation(
* @param p Java-side predicate.
* @param parent Optional parent span.
*/
- def clearTokens(p: Predicate[NCToken], parent: Span = null): Unit = startScopedSpan("clearTokens", parent) { _ ⇒
- stm.synchronized {
- for (item ← stm)
- item.holders --= item.holders.filter(h ⇒ p.test(h.token))
+ def clearTokens(p: Predicate[NCToken], parent: Span = null): Unit =
+ startScopedSpan("clearTokens", parent) { _ ⇒
+ stm.synchronized {
+ for (item ← stm)
+ item.holders --= item.holders.filter(h ⇒ p.test(h.token))
- squeezeTokens()
+ squeezeTokens()
- ctx = ctx.asScala.filter(tok ⇒ !p.test(tok)).asJava
- }
+ ctx = ctx.asScala.filter(tok ⇒ !p.test(tok)).asJava
+ }
- logger.info(s"Conversation is cleared using token predicate [" +
- s"usrId=$usrId, " +
- s"mdlId=$mdlId" +
- s"]")
- }
+ logger.trace(s"Conversation is cleared [" +
+ s"usrId=$usrId, " +
+ s"mdlId=$mdlId" +
+ s"]")
+ }
/**
* Clears all tokens from this conversation satisfying given predicate.
@@ -200,7 +201,7 @@ case class NCConversation(
lastUpdateTstamp
)
- logger.info(
+ logger.trace(
s"Added new tokens to the conversation [" +
s"usrId=$usrId, " +
s"mdlId=$mdlId, " +
@@ -227,7 +228,7 @@ case class NCConversation(
case Some(_) ⇒
item.holders --= hs
- logger.info(
+ logger.trace(
"Conversation tokens are overridden [" +
s"usrId=$usrId, " +
s"mdlId=$mdlId, " +
@@ -255,7 +256,7 @@ case class NCConversation(
require(Thread.holdsLock(stm))
if (ctx.isEmpty)
- logger.info(s"Conversation context is empty for [" +
+ logger.trace(s"Conversation context is empty for [" +
s"mdlId=$mdlId, " +
s"usrId=$usrId" +
s"]")
@@ -270,7 +271,7 @@ case class NCConversation(
tok.getServerRequestId
))
- logger.info(s"Conversation tokens [" +
+ logger.trace(s"Conversation tokens [" +
s"mdlId=$mdlId, " +
s"usrId=$usrId" +
s"]:\n${tbl.toString()}")
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/dialogflow/NCDialogFlowManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/dialogflow/NCDialogFlowManager.scala
index 1705ffb..17d7eba 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/dialogflow/NCDialogFlowManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/dialogflow/NCDialogFlowManager.scala
@@ -136,6 +136,7 @@ object NCDialogFlowManager extends NCService {
// https://github.com/scala/bug/issues/10151
// Scala bug workaround.
()
+
case None ⇒ delKeys += key
}
@@ -152,7 +153,7 @@ object NCDialogFlowManager extends NCService {
}
/**
- * Clears dialog for given user and model IDs.
+ * Clears dialog history for given user and model IDs.
*
* @param usrId User ID.
* @param mdlId Model ID.
@@ -165,10 +166,15 @@ object NCDialogFlowManager extends NCService {
flow.notifyAll()
}
+
+ logger.trace(s"Dialog history is cleared [" +
+ s"usrId=$usrId, " +
+ s"mdlId=$mdlId" +
+ s"]")
}
/**
- * Clears dialog for given use, model IDs and predicate.
+ * Clears dialog history for given user, model IDs and predicate.
*
* @param usrId User ID.
* @param mdlId Model ID.
@@ -184,5 +190,10 @@ object NCDialogFlowManager extends NCService {
flow.notifyAll()
}
+
+ logger.trace(s"Dialog history is cleared [" +
+ s"usrId=$usrId, " +
+ s"mdlId=$mdlId" +
+ s"]")
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index 354739a..1733695 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -765,16 +765,24 @@ object NCProbeManager extends NCService {
}
}
}
-
+
+ /**
+ *
+ * @param modelId
+ * @return
+ */
+ private def getProbesForModelId(modelId: String): Iterable[ProbeHolder] =
+ probes.synchronized {
+ probes.values.filter(_.probe.models.exists(mdl ⇒ mdl.id == modelId))
+ }
+
/**
*
- * @param modelId
+ * @param mdlId
* @return
*/
- private def getProbeForModelId(modelId: String): Option[ProbeHolder] = {
- val candidates = probes.synchronized {
- probes.values.filter(_.probe.models.exists(_.id == modelId)).toSeq
- }
+ private def getProbeForModelId(mdlId: String): Option[ProbeHolder] = {
+ val candidates = getProbesForModelId(mdlId).toList
val sz = candidates.size
@@ -975,10 +983,7 @@ object NCProbeManager extends NCService {
)
// Send to all probes.
- probes.synchronized {
- probes.values
- }
- .map(_.probeKey).foreach(sendToProbe(_, msg, span))
+ getProbesForModelId(mdlId).map(_.probeKey).foreach(sendToProbe(_, msg, span))
}
/**
@@ -996,10 +1001,7 @@ object NCProbeManager extends NCService {
)
// Send to all probes.
- probes.synchronized {
- probes.values
- }
- .map(_.probeKey).foreach(sendToProbe(_, msg, span))
+ getProbesForModelId(mdlId).map(_.probeKey).foreach(sendToProbe(_, msg, span))
}
/**