You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2021/07/22 11:20:49 UTC
[incubator-nlpcraft] branch NLPCRAFT-371 updated: WIP.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-371
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-371 by this push:
new 60f2786 WIP.
60f2786 is described below
commit 60f2786af7e2e200d7d0b8de5e8757a2bc80876a
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Thu Jul 22 14:20:33 2021 +0300
WIP.
---
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 113 ++++++++++++++++-----
.../probe/mgrs/conn/NCConnectionManager.scala | 4 +-
.../nlpcraft/server/mdo/NCProbeModelMdo.scala | 3 +-
.../nlpcraft/server/probe/NCProbeManager.scala | 106 ++++++++++++++++---
.../nlpcraft/server/rest/NCBasicRestApi.scala | 50 ++++-----
.../server/sugsyn/NCSuggestSynonymManager.scala | 2 +-
.../nlpcraft/server/rest/NCRestModelSpec.scala | 33 ++++--
7 files changed, 232 insertions(+), 79 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
index 1f96350..2c6acca 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
@@ -17,11 +17,10 @@
package org.apache.nlpcraft.probe.mgrs.cmd
-import java.io.{Serializable => JSerializable}
import com.google.gson.Gson
import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.{NCService, _}
import org.apache.nlpcraft.common.nlp.NCNlpSentence
+import org.apache.nlpcraft.common.{NCService, _}
import org.apache.nlpcraft.model.NCToken
import org.apache.nlpcraft.probe.mgrs.NCProbeMessage
import org.apache.nlpcraft.probe.mgrs.conn.NCConnectionManager
@@ -30,9 +29,9 @@ import org.apache.nlpcraft.probe.mgrs.dialogflow.NCDialogFlowManager
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnrichmentManager
+import java.io.{Serializable => JSerializable}
import java.util
-import java.util.{List => JList}
-
+import java.util.{Collections, List => JList}
import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, MapHasAsScala, SeqHasAsJava, SetHasAsScala}
/**
@@ -63,6 +62,28 @@ object NCCommandManager extends NCService {
/**
*
+ * @param mkMsg
+ * @param mkErrorMsg
+ */
+ private def send0(mkMsg: () => NCProbeMessage, mkErrorMsg: Throwable => NCProbeMessage, parent: Span = null): Unit = {
+ val msgOpt: Option[NCProbeMessage] =
+ try
+ Some(mkMsg())
+ catch {
+ case e: Throwable =>
+ NCConnectionManager.send(mkErrorMsg(e), parent)
+
+ None
+ }
+
+ msgOpt match {
+ case Some(msg) => NCConnectionManager.send(msg, parent)
+ case None => // No-op.
+ }
+ }
+
+ /**
+ *
* @param msg Server message to process.
* @param parent Optional parent span.
*/
@@ -108,26 +129,72 @@ object NCCommandManager extends NCService {
)
case "S2P_MODEL_INFO" =>
- val mdlId = msg.data[String]("mdlId")
-
- val mdlData = NCModelManager.getModel(mdlId)
-
- val macros: util.Map[String, String] = mdlData.model.getMacros
- val syns: util.Map[String, util.List[String]] = mdlData.model.getElements.asScala.map(p => p.getId -> p.getSynonyms).toMap.asJava
- val samples: util.Map[String, util.List[util.List[String]]] = mdlData.samples.map(p => p._1 -> p._2.map(_.asJava).asJava).toMap.asJava
-
- NCConnectionManager.send(
- NCProbeMessage(
- "P2S_MODEL_INFO",
- "reqGuid" -> msg.getGuid,
- "resp" -> GSON.toJson(
- Map(
- "macros" -> macros.asInstanceOf[JSerializable],
- "synonyms" -> syns.asInstanceOf[JSerializable],
- "samples" -> samples.asInstanceOf[JSerializable]
- ).asJava
+ send0(
+ mkMsg = () => {
+ val mdlId = msg.data[String]("mdlId")
+
+ val mdlData = NCModelManager.getModel(mdlId)
+
+ val macros: util.Map[String, String] = mdlData.model.getMacros
+ val syns: util.Map[String, util.List[String]] =
+ mdlData.model.getElements.asScala.map(p => p.getId -> p.getSynonyms).toMap.asJava
+ val samples: util.Map[String, util.List[util.List[String]]] =
+ mdlData.samples.map(p => p._1 -> p._2.map(_.asJava).asJava).toMap.asJava
+
+ NCProbeMessage(
+ "P2S_MODEL_INFO",
+ "reqGuid" -> msg.getGuid,
+ "resp" -> GSON.toJson(
+ Map(
+ "macros" -> macros.asInstanceOf[JSerializable],
+ "synonyms" -> syns.asInstanceOf[JSerializable],
+ "samples" -> samples.asInstanceOf[JSerializable]
+ ).asJava
+ )
+ )
+ },
+ mkErrorMsg = e =>
+ NCProbeMessage(
+ "P2S_MODEL_INFO",
+ "reqGuid" -> msg.getGuid,
+ "error" -> e.getLocalizedMessage
+ ),
+ span
+ )
+ case "S2P_MODEL_ELEMENT_INFO" =>
+ send0(
+ mkMsg = () => {
+ val mdlId = msg.data[String]("mdlId")
+ val elemId = msg.data[String]("elemId")
+
+ val elm = NCModelManager.
+ getModel(mdlId).
+ model.getElements.asScala.find(_.getId == elemId).
+ getOrElse(throw new NCE(s"Element not found in model: $elemId"))
+
+ val vals: util.Map[String, JList[String]] =
+ if (elm.getValues != null)
+ elm.getValues.asScala.map(e => e.getName -> e.getSynonyms).toMap.asJava
+ else
+ Collections.emptyMap()
+
+ NCProbeMessage(
+ "P2S_MODEL_ELEMENT_INFO",
+ "reqGuid" -> msg.getGuid,
+ "resp" -> GSON.toJson(
+ Map(
+ "synonyms" -> elm.getSynonyms.asInstanceOf[JSerializable],
+ "values" -> vals.asInstanceOf[JSerializable]
+ ).asJava
+ )
)
- ),
+ },
+ mkErrorMsg = e =>
+ NCProbeMessage(
+ "P2S_MODEL_ELEMENT_INFO",
+ "reqGuid" -> msg.getGuid,
+ "error" -> e.getLocalizedMessage
+ ),
span
)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
index 362e930..d2a4619 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
@@ -34,6 +34,7 @@ import java.util
import java.util.concurrent.CountDownLatch
import java.util.{Properties, TimeZone}
import scala.collection.mutable
+import scala.jdk.CollectionConverters.{SetHasAsJava, SetHasAsScala}
/**
* Probe down/up link connection manager.
@@ -221,7 +222,8 @@ object NCConnectionManager extends NCService {
mdl.getId,
mdl.getName,
mdl.getVersion,
- new util.HashSet[String](mdl.getEnabledBuiltInTokens)
+ new util.HashSet[String](mdl.getEnabledBuiltInTokens),
+ new util.HashSet[String](mdl.getElements.asScala.map(_.getId).asJava)
)
})
), cryptoKey)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala
index 16edd61..1b6001b 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala
@@ -27,7 +27,8 @@ case class NCProbeModelMdo(
@NCMdoField id: String,
@NCMdoField name: String,
@NCMdoField version: String,
- @NCMdoField enabledBuiltInTokens: Set[String]
+ @NCMdoField enabledBuiltInTokens: Set[String],
+ @NCMdoField elementIds: Set[String]
) extends NCAnnotatedMdo[NCProbeModelMdo] {
override def hashCode(): Int = s"$id$name".hashCode()
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index f572b9f..a6c4ce9 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -54,6 +54,7 @@ import scala.util.{Failure, Success}
object NCProbeManager extends NCService {
private final val GSON = new Gson()
private final val TYPE_MODEL_INFO_RESP = new TypeToken[JavaMeta]() {}.getType
+ private final val TYPE_MODEL_ELEMENT_INFO_RESP = new TypeToken[JavaMeta]() {}.getType
// Type safe and eager configuration container.
private object Config extends NCConfigurable {
@@ -157,6 +158,7 @@ object NCProbeManager extends NCService {
@volatile private var pending: mutable.Map[ProbeKey, ProbeHolder] = _
@volatile private var modelsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
+ @volatile private var modelElemsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
/**
*
@@ -179,6 +181,7 @@ object NCProbeManager extends NCService {
)
modelsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
+ modelElemsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
dnSrv = startServer("Downlink", dnHost, dnPort, downLinkHandler)
upSrv = startServer("Uplink", upHost, upPort, upLinkHandler)
@@ -216,6 +219,7 @@ object NCProbeManager extends NCService {
U.stopThread(upSrv)
modelsInfo = null
+ modelElemsInfo = null
ackStopped()
}
@@ -613,6 +617,7 @@ object NCProbeManager extends NCService {
String,
String,
String,
+ java.util.Set[String],
java.util.Set[String]
)]]("PROBE_MODELS").
map {
@@ -620,18 +625,21 @@ object NCProbeManager extends NCService {
mdlId,
mdlName,
mdlVer,
- enabledBuiltInToks
+ enabledBuiltInToks,
+ elemIds
) =>
require(mdlId != null)
require(mdlName != null)
require(mdlVer != null)
require(enabledBuiltInToks != null)
+ require(elemIds != null)
NCProbeModelMdo(
id = mdlId,
name = mdlName,
version = mdlVer,
- enabledBuiltInTokens = enabledBuiltInToks.asScala.toSet
+ enabledBuiltInTokens = enabledBuiltInToks.asScala.toSet,
+ elementIds = elemIds.asScala.toSet
)
}.toSet
@@ -711,9 +719,23 @@ object NCProbeManager extends NCService {
val p = modelsInfo.remove(probeMsg.data[String]("reqGuid"))
if (p != null)
- p.success(GSON.fromJson(probeMsg.data[String]("resp"), TYPE_MODEL_INFO_RESP))
+ probeMsg.dataOpt[String]("resp") match {
+ case Some(resp) => p.success(GSON.fromJson(resp, TYPE_MODEL_INFO_RESP))
+ case None => p.failure(new NCE(probeMsg.data[String]("error")))
+ }
+ else
+ logger.warn(s"Message ignored: $probeMsg")
+ case "P2S_MODEL_ELEMENT_INFO" =>
+ val p = modelElemsInfo.remove(probeMsg.data[String]("reqGuid"))
+
+ if (p != null)
+ probeMsg.dataOpt[String]("resp") match {
+ case Some(resp) => p.success(GSON.fromJson(resp, TYPE_MODEL_ELEMENT_INFO_RESP))
+ case None => p.failure(new NCE(probeMsg.data[String]("error")))
+ }
else
logger.warn(s"Message ignored: $probeMsg")
+
case "P2S_ASK_RESULT" =>
val srvReqId = probeMsg.data[String]("srvReqId")
@@ -966,6 +988,27 @@ object NCProbeManager extends NCService {
}
/**
+ * Checks whether or not a data probe exists for given model element.
+ *
+ * @param compId Company ID for authentication purpose.
+ * @param mdlId Model ID.
+ * @param elemId Element ID.
+ * @param parent Optional parent span.
+ * @return
+ */
+ def existsForModelElement(compId: Long, mdlId: String, elemId: String, parent: Span = null): Boolean =
+ startScopedSpan(
+ "existsForModelElement", parent, "compId" -> compId, "mdlId" -> mdlId, "elemId" -> elemId
+ ) { _ =>
+ val authTok = getCompany(compId).authToken
+
+ probes.synchronized {
+ probes.filter(_._1.probeToken == authTok).values.
+ exists(_.probe.models.exists(p => p.id == mdlId && p.elementIds.contains(elemId)))
+ }
+ }
+
+ /**
*
* @param usrId User ID.
* @param mdlId Model ID.
@@ -1016,24 +1059,57 @@ object NCProbeManager extends NCService {
/**
*
* @param mdlId
+ * @param msg
+ * @param holder
* @param parent
- * @return
*/
- def getModelInfo(mdlId: String, parent: Span = null): Future[JavaMeta] =
- startScopedSpan("getModelInfo", parent, "mdlId" -> mdlId) { _ =>
- getProbeForModelId(mdlId) match {
- case Some(probe) =>
- val msg = NCProbeMessage("S2P_MODEL_INFO", "mdlId" -> mdlId)
+ private def processModelDataRequest(
+ mdlId: String, msg: NCProbeMessage, holder: ConcurrentHashMap[String, Promise[JavaMeta]], parent: Span = null
+ ): Future[JavaMeta] = {
+ val p = Promise[JavaMeta]()
- val p = Promise[JavaMeta]()
+ getProbeForModelId(mdlId) match {
+ case Some(probe) =>
+ holder.put(msg.getGuid, p)
- modelsInfo.put(msg.getGuid, p)
+ sendToProbe(probe.probeKey, msg, parent)
+ case None =>
+ p.failure(new NCE(s"Probe not found for model: '$mdlId''"))
+ }
- sendToProbe(probe.probeKey, msg, parent)
+ p.future
+ }
- p.future
+ /**
+ *
+ * @param mdlId
+ * @param parent
+ * @return
+ */
+ def getModelInfo(mdlId: String, parent: Span = null): Future[JavaMeta] =
+ startScopedSpan("getModelInfo", parent, "mdlId" -> mdlId) { _ =>
+ processModelDataRequest(
+ mdlId,
+ NCProbeMessage("S2P_MODEL_INFO", "mdlId" -> mdlId),
+ modelsInfo,
+ parent
+ )
+ }
- case None => throw new NCE(s"Probe not found for model: '$mdlId''")
- }
+ /**
+ *
+ * @param mdlId
+ * @param elemId
+ * @param parent
+ * @return
+ */
+ def getElementInfo(mdlId: String, elemId: String, parent: Span = null): Future[JavaMeta] =
+ startScopedSpan("getModelInfo", parent, "mdlId" -> mdlId, "elemId" -> elemId) { _ =>
+ processModelDataRequest(
+ mdlId,
+ NCProbeMessage("S2P_MODEL_ELEMENT_INFO", "mdlId" -> mdlId, "elemId" -> elemId),
+ modelElemsInfo,
+ parent
+ )
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
index 70368f1..d94f7d1 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
@@ -36,7 +36,6 @@ import org.apache.nlpcraft.common.pool.NCThreadPoolManager
import org.apache.nlpcraft.common.util.NCUtils.{jsonToJavaMap, uncompress}
import org.apache.nlpcraft.common.{JavaMeta, NCE, U}
import org.apache.nlpcraft.model.NCModelView
-import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import org.apache.nlpcraft.server.apicodes.NCApiStatusCode.{API_OK, _}
import org.apache.nlpcraft.server.company.NCCompanyManager
import org.apache.nlpcraft.server.feedback.NCFeedbackManager
@@ -89,6 +88,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
case class InvalidExternalUserId(usrExtId: String) extends InvalidArguments(s"External user ID is invalid or unknown: $usrExtId")
case class InvalidUserId(id: Long) extends InvalidArguments(s"User ID is invalid or unknown: $id")
case class InvalidModelId(id: String) extends InvalidArguments(s"Unknown model ID: $id")
+ case class InvalidModelOrElementId(mdlId: String, elemId: String) extends InvalidArguments(s"Unknown model ID: $mdlId or element ID: $elemId")
case class AskReqHolder(
usrId: Long,
@@ -834,20 +834,6 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
implicit val reqFmt: RootJsonFormat[Req$Model$Syn] = jsonFormat3(Req$Model$Syn)
- case class Res$Model$Value(
- name: String,
- synonyms: Seq[String]
- )
-
- case class Res$Model$Element(
- status: String,
- synonyms: Seq[String],
- values: Seq[Res$Model$Value]
- )
-
- implicit val resValFmt: RootJsonFormat[Res$Model$Value] = jsonFormat2(Res$Model$Value)
- implicit val resFmt: RootJsonFormat[Res$Model$Element] = jsonFormat3(Res$Model$Element)
-
entity(as[Req$Model$Syn]) { req =>
startScopedSpan(
"model$syns",
@@ -858,24 +844,26 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
val admUsr = authenticateAsAdmin(req.acsTok)
- checkModelId(req.mdlId, admUsr.companyId)
+ if (!NCProbeManager.existsForModelElement(admUsr.companyId, req.mdlId, req.elemId))
+ throw InvalidModelOrElementId(req.mdlId, req.elemId)
- val elm =
- NCModelManager.
- getModel(req.mdlId, span).
- model.
- getElements.asScala.find(_.getId == req.elemId).getOrElse(throw InvalidModelId(req.elemId))
+ val fut = NCProbeManager.getElementInfo(req.mdlId, req.elemId, span)
- complete {
- Res$Model$Element(
- API_OK,
- elm.getSynonyms.asScala.toSeq,
- if (elm.getValues != null)
- elm.getValues.asScala.map(p => Res$Model$Value(p.getName, p.getSynonyms.asScala.toSeq)).toSeq
- else
- Seq.empty
- )
- }
+ successWithJs(
+ fut.collect {
+ // We have to use Jackson (not spray) here to serialize 'result' field.
+ case res =>
+ require(res.containsKey("synonyms") && res.containsKey("values"))
+
+ toJs(
+ Map(
+ "status" -> API_OK.toString,
+ "synonyms" -> res.get("synonyms"),
+ "values" -> res.get("values")
+ )
+ )
+ }
+ )
}
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
index 76af8a0..919fc76 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
@@ -143,7 +143,7 @@ object NCSuggestSynonymManager extends NCService {
val promise = Promise[NCSuggestSynonymResult]()
- NCProbeManager.getModelInfo(mdlId, parent).onComplete {
+ NCProbeManager.getModelInfo(mdlId = mdlId, parent = parent).onComplete {
case Success(m) =>
try {
require(
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
index 86fb1af..6cdc0af 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
@@ -54,25 +54,44 @@ class NCRestModelSpec extends NCRestSpec {
assertTrue(scores.forall(s => s >= 0.5 && s <= 1))
})
)
+
+ postError("model/sugsyn", 400, "NC_INVALID_FIELD", "mdlId" -> "UNKNOWN")
+ postError("model/sugsyn", 400, "NC_INVALID_FIELD", "mdlId" -> "rest.test.model", "minScore" -> 2)
+ postError("model/sugsyn", 400, "NC_ERROR")
}
@Test
def testSyns(): Unit = {
- def extract(data: JList[java.util.Map[String, Object]]): Seq[Double] =
- data.asScala.map(_.get("score").asInstanceOf[Number].doubleValue()).toSeq
-
// Note that checked values are valid for current configuration of `RestTestModel` model.
post("model/syns", "mdlId" -> "rest.test.model", "elemId" -> "x")(
("$.status", (status: String) => assertEquals("API_OK", status)),
- ("$.synonyms", (data: ResponseList) => {
- println("data="+data)
+ ("$.synonyms", (syns: ResponseList) => {
+ println("synonyms="+syns)
+
+ assertTrue(!syns.isEmpty)
+ }),
+ ("$.values", (vals: java.util.Map[Object, Object]) => {
+ println("values="+vals)
+
+ assertTrue(vals.isEmpty)
})
)
post("model/syns", "mdlId" -> "rest.test.model", "elemId" -> "valElem")(
("$.status", (status: String) => assertEquals("API_OK", status)),
- ("$.synonyms", (data: ResponseList) => {
- println("data="+data)
+ ("$.synonyms", (syns: ResponseList) => {
+ println("synonyms="+syns)
+
+ assertTrue(!syns.isEmpty)
+ }),
+ ("$.values", (vals: java.util.Map[Object, Object]) => {
+ println("values="+vals)
+
+ assertTrue(!vals.isEmpty)
})
)
+
+ postError("model/syns", 400, "NC_INVALID_FIELD", "mdlId" -> "UNKNOWN", "elemId" -> "UNKNOWN")
+ postError("model/syns", 400, "NC_INVALID_FIELD", "mdlId" -> "rest.test.model", "elemId" -> "UNKNOWN")
+ postError("model/syns", 400, "NC_ERROR", "mdlId" -> "rest.test.model")
}
}