You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2021/07/25 08:53:28 UTC
[incubator-nlpcraft] branch NLPCRAFT-379 created (now 7af50af)
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a change to branch NLPCRAFT-379
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
at 7af50af WIP.
This branch includes the following new commits:
new 7af50af WIP.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-nlpcraft] 01/01: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-379
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 7af50af47614b71ffd7754eaca9d9accfbc9882b
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Sun Jul 25 11:53:12 2021 +0300
WIP.
---
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 127 ++++++++++++++++++---
.../nlpcraft/server/probe/NCProbeManager.scala | 76 ++++++------
.../nlpcraft/server/rest/NCBasicRestApi.scala | 46 ++++++++
.../server/sugsyn/NCSuggestSynonymManager.scala | 2 +-
.../nlpcraft/server/rest/NCRestModelSpec.scala | 11 ++
5 files changed, 213 insertions(+), 49 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
index f63a088..26d29c1 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
@@ -17,11 +17,12 @@
package org.apache.nlpcraft.probe.mgrs.cmd
-import com.google.gson.Gson
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
import io.opencensus.trace.Span
import org.apache.nlpcraft.common.nlp.NCNlpSentence
import org.apache.nlpcraft.common.{NCService, _}
-import org.apache.nlpcraft.model.NCToken
+import org.apache.nlpcraft.model.{NCCustomParser, NCElement, NCModelView, NCToken, NCValue, NCValueLoader}
import org.apache.nlpcraft.probe.mgrs.NCProbeMessage
import org.apache.nlpcraft.probe.mgrs.conn.NCConnectionManager
import org.apache.nlpcraft.probe.mgrs.conversation.NCConversationManager
@@ -30,21 +31,18 @@ import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnrichmentManager
import java.io.{Serializable => JSerializable}
-import java.util
-import java.util.{Collections, List => JList}
-import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, MapHasAsScala, SeqHasAsJava, SetHasAsScala}
+import java.util.{Collections, Optional, List => JList}
+import java.{lang, util}
+import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, MapHasAsScala, SeqHasAsJava, SetHasAsJava, SetHasAsScala}
/**
* Probe commands processor.
*/
object NCCommandManager extends NCService {
- private final val GSON = new Gson()
+ private final val JS_MAPPER = new ObjectMapper()
+
+ JS_MAPPER.registerModule(DefaultScalaModule)
- /**
- * Starts this service.
- *
- * @param parent Optional parent span.
- */
override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ =>
ackStarting()
ackStarted()
@@ -128,7 +126,7 @@ object NCCommandManager extends NCService {
span
)
- case "S2P_MODEL_INFO" =>
+ case "S2P_MODEL_SYNS_INFO" =>
send0(
mkMsg = () => {
val mdlId = msg.data[String]("mdlId")
@@ -142,9 +140,9 @@ object NCCommandManager extends NCService {
mdlData.samples.map(p => p._1 -> p._2.map(_.asJava).asJava).toMap.asJava
NCProbeMessage(
- "P2S_MODEL_INFO",
+ "P2S_MODEL_SYNS_INFO",
"reqGuid" -> msg.getGuid,
- "resp" -> GSON.toJson(
+ "resp" -> JS_MAPPER.writeValueAsString(
Map(
"macros" -> macros.asInstanceOf[JSerializable],
"synonyms" -> syns.asInstanceOf[JSerializable],
@@ -155,7 +153,7 @@ object NCCommandManager extends NCService {
},
mkErrorMsg = e =>
NCProbeMessage(
- "P2S_MODEL_INFO",
+ "P2S_MODEL_SYNS_INFO",
"reqGuid" -> msg.getGuid,
"error" -> e.getLocalizedMessage
),
@@ -182,7 +180,7 @@ object NCCommandManager extends NCService {
NCProbeMessage(
"P2S_MODEL_ELEMENT_INFO",
"reqGuid" -> msg.getGuid,
- "resp" -> GSON.toJson(
+ "resp" -> JS_MAPPER.writeValueAsString(
Map(
"synonyms" -> elm.getSynonyms.asInstanceOf[JSerializable],
"values" -> vals.asInstanceOf[JSerializable],
@@ -200,6 +198,103 @@ object NCCommandManager extends NCService {
span
)
+ case "S2P_MODEL_INFO" =>
+ send0(
+ mkMsg = () => {
+ val mdlId = msg.data[String]("mdlId")
+
+ val mdl = NCModelManager.getModel(mdlId).model
+
+ val convertedMdl =
+ new NCModelView {
+ // As is.
+ override def getId: String = mdl.getId
+ override def getName: String = mdl.getName
+ override def getVersion: String = mdl.getVersion
+ override def getDescription: String = mdl.getDescription
+ override def getOrigin: String = mdl.getOrigin
+ override def getMaxUnknownWords: Int = mdl.getMaxUnknownWords
+ override def getMaxFreeWords: Int = mdl.getMaxFreeWords
+ override def getMaxSuspiciousWords: Int = mdl.getMaxSuspiciousWords
+ override def getMinWords: Int = mdl.getMinWords
+ override def getMaxWords: Int = mdl.getMaxWords
+ override def getMinTokens: Int = mdl.getMinTokens
+ override def getMaxTokens: Int = mdl.getMaxTokens
+ override def getMinNonStopwords: Int = mdl.getMinNonStopwords
+ override def isNonEnglishAllowed: Boolean = mdl.isNonEnglishAllowed
+ override def isNotLatinCharsetAllowed: Boolean = mdl.isNotLatinCharsetAllowed
+ override def isSwearWordsAllowed: Boolean = mdl.isSwearWordsAllowed
+ override def isNoNounsAllowed: Boolean = mdl.isNoNounsAllowed
+ override def isPermutateSynonyms: Boolean = mdl.isPermutateSynonyms
+ override def isDupSynonymsAllowed: Boolean = mdl.isDupSynonymsAllowed
+ override def getMaxTotalSynonyms: Int = mdl.getMaxTotalSynonyms
+ override def isNoUserTokensAllowed: Boolean = mdl.isNoUserTokensAllowed
+ override def isSparse: Boolean = mdl.isSparse
+ override def getMetadata: util.Map[String, AnyRef] = mdl.getMetadata
+ override def getAdditionalStopWords: util.Set[String] =
+ mdl.getAdditionalStopWords
+ override def getExcludedStopWords: util.Set[String] = mdl.getExcludedStopWords
+ override def getSuspiciousWords: util.Set[String] = mdl.getSuspiciousWords
+ override def getMacros: util.Map[String, String] = mdl.getMacros
+ override def getEnabledBuiltInTokens: util.Set[String] =
+ mdl.getEnabledBuiltInTokens
+ override def getAbstractTokens: util.Set[String] = mdl.getAbstractTokens
+ override def getMaxElementSynonyms: Int = mdl.getMaxElementSynonyms
+ override def isMaxSynonymsThresholdError: Boolean =
+ mdl.isMaxSynonymsThresholdError
+ override def getConversationTimeout: Long = mdl.getConversationTimeout
+ override def getConversationDepth: Int = mdl.getConversationDepth
+ override def getRestrictedCombinations: util.Map[String, util.Set[String]] =
+ mdl.getRestrictedCombinations
+
+ // Cleared.
+ override def getParsers: JList[NCCustomParser] = null
+ // Converted.
+ override def getElements: util.Set[NCElement] = mdl.getElements.asScala.map(e =>
+ new NCElement {
+ // As is.
+ override def getId: String = e.getId
+ override def getGroups: JList[String] = e.getGroups
+ override def getMetadata: util.Map[String, AnyRef] = e.getMetadata
+ override def getDescription: String = e.getDescription
+ override def getParentId: String = e.getParentId
+ override def getSynonyms: JList[String] = e.getSynonyms
+ override def isPermutateSynonyms: Optional[lang.Boolean] =
+ e.isPermutateSynonyms
+ override def isSparse: Optional[lang.Boolean] = e.isSparse
+
+ // Cleared.
+ override def getValueLoader: Optional[NCValueLoader] = null
+ // Converted.
+ override def getValues: JList[NCValue] =
+ if (e.getValues != null) {
+ e.getValues.asScala.map(v => new NCValue {
+ override def getName: String = v.getName
+ // Cleared.
+ override def getSynonyms: JList[String] = null
+ }).asJava
+ }
+ else
+ null
+ }
+ ).asJava
+ }
+
+ NCProbeMessage(
+ "P2S_MODEL_INFO",
+ "reqGuid" -> msg.getGuid,
+ "resp" -> JS_MAPPER.writeValueAsString(convertedMdl)
+ )
+ },
+ mkErrorMsg = e =>
+ NCProbeMessage(
+ "P2S_MODEL_SYNS_INFO",
+ "reqGuid" -> msg.getGuid,
+ "error" -> e.getLocalizedMessage
+ ),
+ span
+ )
+
case _ =>
logger.error(s"Received unknown server message (you need to update the probe): ${msg.getType}")
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index 04792f6..3c7c46c 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -25,7 +25,7 @@ import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.common.crypto.NCCipher
import org.apache.nlpcraft.common.makro.NCMacroParser
import org.apache.nlpcraft.common.nlp.NCNlpSentence
-import org.apache.nlpcraft.common.nlp.core.{NCNlpCoreManager, NCNlpPorterStemmer}
+import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.pool.NCThreadPoolManager
import org.apache.nlpcraft.common.socket.NCSocket
import org.apache.nlpcraft.common.version.NCVersion
@@ -54,8 +54,7 @@ import scala.util.{Failure, Success}
*/
object NCProbeManager extends NCService {
private final val GSON = new Gson()
- private final val TYPE_MODEL_INFO_RESP = new TypeToken[JavaMeta]() {}.getType
- private final val TYPE_MODEL_ELEMENT_INFO_RESP = new TypeToken[JavaMeta]() {}.getType
+ private final val TYPE_JAVA_META = new TypeToken[JavaMeta]() {}.getType
// Type safe and eager configuration container.
private object Config extends NCConfigurable {
@@ -67,7 +66,7 @@ object NCProbeManager extends NCService {
def reconnectTimeoutMs: Long = getLong(s"$pre.reconnectTimeoutMs")
def pingTimeoutMs: Long = getLong(s"$pre.pingTimeoutMs")
def soTimeoutMs: Int = getInt(s"$pre.soTimeoutMs")
-
+
/**
*
*/
@@ -158,8 +157,9 @@ object NCProbeManager extends NCService {
// All probes pending complete handshake keyed by probe key.
@volatile private var pending: mutable.Map[ProbeKey, ProbeHolder] = _
- @volatile private var modelsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
+ @volatile private var modelsSynsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
@volatile private var modelElmsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
+ @volatile private var modelsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _
/**
*
@@ -181,8 +181,9 @@ object NCProbeManager extends NCService {
"downlink" -> s"$dnHost:$dnPort"
)
- modelsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
+ modelsSynsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
modelElmsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
+ modelsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]()
dnSrv = startServer("Downlink", dnHost, dnPort, downLinkHandler)
upSrv = startServer("Uplink", upHost, upPort, upLinkHandler)
@@ -219,8 +220,9 @@ object NCProbeManager extends NCService {
U.stopThread(dnSrv)
U.stopThread(upSrv)
- modelsInfo = null
+ modelsSynsInfo = null
modelElmsInfo = null
+ modelsInfo = null
ackStopped()
}
@@ -691,6 +693,23 @@ object NCProbeManager extends NCService {
}
}
}
+
+ /**
+ *
+ * @param probeMsg
+ * @param m
+ */
+ private def processJavaMetaMessage(probeMsg: NCProbeMessage, m: ConcurrentHashMap[String, Promise[JavaMeta]]): Unit = {
+ val p = m.remove(probeMsg.data[String]("reqGuid"))
+
+ if (p != null)
+ probeMsg.dataOpt[String]("resp") match {
+ case Some(resp) => p.success(GSON.fromJson(resp, TYPE_JAVA_META))
+ case None => p.failure(new NCE(probeMsg.data[String]("error")))
+ }
+ else
+ logger.warn(s"Message ignored: $probeMsg")
+ }
/**
* Processes the messages received from the probe.
@@ -716,27 +735,9 @@ object NCProbeManager extends NCService {
typ match {
case "P2S_PING" => ()
- case "P2S_MODEL_INFO" =>
- val p = modelsInfo.remove(probeMsg.data[String]("reqGuid"))
-
- if (p != null)
- probeMsg.dataOpt[String]("resp") match {
- case Some(resp) => p.success(GSON.fromJson(resp, TYPE_MODEL_INFO_RESP))
- case None => p.failure(new NCE(probeMsg.data[String]("error")))
- }
- else
- logger.warn(s"Message ignored: $probeMsg")
-
- case "P2S_MODEL_ELEMENT_INFO" =>
- val p = modelElmsInfo.remove(probeMsg.data[String]("reqGuid"))
-
- if (p != null)
- probeMsg.dataOpt[String]("resp") match {
- case Some(resp) => p.success(GSON.fromJson(resp, TYPE_MODEL_ELEMENT_INFO_RESP))
- case None => p.failure(new NCE(probeMsg.data[String]("error")))
- }
- else
- logger.warn(s"Message ignored: $probeMsg")
+ case "P2S_MODEL_SYNS_INFO" => processJavaMetaMessage(probeMsg, modelsSynsInfo)
+ case "P2S_MODEL_ELEMENT_INFO" => processJavaMetaMessage(probeMsg, modelElmsInfo)
+ case "P2S_MODEL_INFO" => processJavaMetaMessage(probeMsg, modelsInfo)
case "P2S_ASK_RESULT" =>
val srvReqId = probeMsg.data[String]("srvReqId")
@@ -1089,12 +1090,12 @@ object NCProbeManager extends NCService {
* @param parent
* @return
*/
- def getModelInfo(mdlId: String, parent: Span = null): Future[JavaMeta] =
- startScopedSpan("getModelInfo", parent, "mdlId" -> mdlId) { _ =>
+ def getModelSynonymsInfo(mdlId: String, parent: Span = null): Future[JavaMeta] =
+ startScopedSpan("getModelSynonymsInfo", parent, "mdlId" -> mdlId) { _ =>
processModelDataRequest(
mdlId,
- NCProbeMessage("S2P_MODEL_INFO", "mdlId" -> mdlId),
- modelsInfo,
+ NCProbeMessage("S2P_MODEL_SYNS_INFO", "mdlId" -> mdlId),
+ modelsSynsInfo,
parent
)
}
@@ -1142,4 +1143,15 @@ object NCProbeManager extends NCService {
}
)
}
+
+ def getModelInfo(mdlId: String, parent: Span = null): Future[JavaMeta] =
+ startScopedSpan("getModelInfo", parent, "mdlId" -> mdlId) { _ =>
+ processModelDataRequest(
+ mdlId,
+ NCProbeMessage("S2P_MODEL_INFO", "mdlId" -> mdlId),
+ modelsInfo,
+ parent
+ )
+ }
+
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
index e74fb68..39b84e4 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
@@ -879,6 +879,51 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
}
}
+
+ /**
+ *
+ * @return
+ */
+ protected def $model$info(): Route = {
+ case class Req$Model$Info(
+ acsTok: String,
+ mdlId: String
+ )
+
+ implicit val reqFmt: RootJsonFormat[Req$Model$Info] = jsonFormat2(Req$Model$Info)
+
+ entity(as[Req$Model$Info]) { req =>
+ startScopedSpan(
+ "model$syns",
+ "acsTok" -> req.acsTok,
+ "mdlId" -> req.mdlId
+ ) { span =>
+ checkLength("acsTok" -> req.acsTok, "mdlId" -> req.mdlId)
+
+ val admUsr = authenticateAsAdmin(req.acsTok)
+ val compId = admUsr.companyId
+
+ if (!NCProbeManager.existsForModel(compId, req.mdlId))
+ throw InvalidModelId(req.mdlId)
+
+ val fut = NCProbeManager.getModelInfo(req.mdlId, span)
+
+ successWithJs(
+ fut.collect {
+ // We have to use Jackson (not spray) here to serialize 'result' field.
+ case res =>
+ toJs(
+ Map(
+ "status" -> API_OK.toString,
+ "model" -> res
+ )
+ )
+ }
+ )
+ }
+ }
+ }
+
/**
*
* @return
@@ -2040,6 +2085,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
path(API / "probe" / "all") { withMetric(M_PROBE_ALL_LATENCY_MS, $probe$All) } ~
path(API / "model" / "sugsyn") { withMetric(M_MODEL_SUGSYN_LATENCY_MS, $model$sugsyn) } ~
path(API / "model" / "syns") { withMetric(M_MODEL_SYNS_LATENCY_MS, $model$syns) } ~
+ path(API / "model" / "info") { withMetric(M_MODEL_SYNS_LATENCY_MS, $model$info) } ~
path(API / "ask") { withMetric(M_ASK_LATENCY_MS, $ask) } ~
path(API / "ask" / "sync") { withMetric(M_ASK_SYNC_LATENCY_MS, $ask$Sync) }
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
index fae55d0..d89ba98 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
@@ -143,7 +143,7 @@ object NCSuggestSynonymManager extends NCService {
val promise = Promise[NCSuggestSynonymResult]()
- NCProbeManager.getModelInfo(mdlId, parent).onComplete {
+ NCProbeManager.getModelSynonymsInfo(mdlId, parent).onComplete {
case Success(m) =>
try {
require(
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
index 0b8c03f..d7d22b8 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
@@ -111,4 +111,15 @@ class NCRestModelSpec2 extends NCRestSpec {
postError("model/syns", 400, "NC_INVALID_FIELD", "mdlId" -> "rest.test.model", "elmId" -> ("A" * 65))
postError("model/syns", 400, "NC_ERROR", "mdlId" -> "rest.test.model")
}
+
+ @Test
+ def testModelInfo(): Unit = {
+ post("model/info", "mdlId" -> "rest.test.model")(
+ ("$.status", (status: String) => assertEquals("API_OK", status)),
+ ("$.model", (data: java.util.Map[Object, Object]) => assertTrue(!data.isEmpty))
+ )
+
+ postError("model/info", 400, "NC_INVALID_FIELD", "mdlId" -> "UNKNOWN")
+ postError("model/info", 400, "NC_ERROR")
+ }
}