You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2020/08/30 19:06:48 UTC
[incubator-nlpcraft] 01/01: WIP.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-41
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit f3895f0b4ec6803dab38dcf52a41b9f2cee97b54
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Sun Aug 30 22:06:37 2020 +0300
WIP.
---
.../nlpcraft/common/inspections/NCInspection.scala | 79 ++--
.../NCInspectionParameter.scala | 8 +-
.../NCInspectionResult.scala | 8 +-
.../common/inspections/impl/NCInspectionImpl.scala | 50 +++
.../NCInspectionParameterImpl.scala} | 18 +-
.../inspections/impl/NCInspectionResultImpl.scala | 56 +++
.../common/inspections2/NCInspection.scala | 65 ---
.../org/apache/nlpcraft/probe/NCProbeBoot.scala | 6 +-
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 50 ++-
.../probe/mgrs/deploy/NCDeployManager.scala | 2 -
...tionManager.scala => NCInspectionManager.scala} | 39 +-
.../mgrs/inspections/inspectors}/NCInspector.scala | 16 +-
.../inspectors/NCInspectorIntents.scala | 63 +--
.../inspections/inspectors/NCInspectorMacros.scala | 36 +-
.../inspectors/NCInspectorSynonyms.scala | 57 ++-
.../NCInspectorSynonymsSuggestions.scala | 44 --
.../mgrs/inspections2/NCInspectionManager.scala | 50 ---
.../nlpcraft/probe/mgrs/model/NCModelManager.scala | 61 ++-
.../org/apache/nlpcraft/server/NCServer.scala | 8 +-
.../server/inspections/NCInspectionManager.scala | 106 +++++
.../inspections/NCServerInspectorManager.scala | 94 -----
.../inspections/inspectors/NCInspector.scala | 10 +
.../inspectors/NCInspectorSuggestions.scala | 443 +++++++++++++++++++++
.../NCInspectorSynonymsSuggestions.scala | 420 -------------------
.../server/inspections2/NCInspectionManager.scala | 57 ---
.../nlpcraft/server/probe/NCProbeManager.scala | 118 ++++--
.../nlpcraft/server/rest/NCBasicRestApi.scala | 82 ++--
27 files changed, 1066 insertions(+), 980 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
index e44d31b..61c1965 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
@@ -17,40 +17,47 @@
package org.apache.nlpcraft.common.inspections
-import scala.collection.JavaConverters._
-
/**
- * Note that 'suggestions' and 'data' must be simple type or java collections to be transfer between server and probe as JSON
- */
-case class NCInspection(
- errors: java.util.List[String],
- warnings: java.util.List[String],
- suggestions: java.util.List[AnyRef],
-
- // Information for next inspection layer.
- data: AnyRef = None
-)
-object NCInspection {
- def apply(
- errors: Option[Seq[String]],
- warnings: Option[Seq[String]],
- suggestions: Option[Seq[AnyRef]],
- data: Option[AnyRef]
- ): NCInspection = {
- def convert[T](optSeq: Option[Seq[T]]): java.util.List[T] = optSeq.getOrElse(Seq.empty).asJava
-
- NCInspection(
- errors = convert(errors),
- warnings = convert(warnings),
- suggestions = convert(suggestions),
- data = data.orNull
- )
- }
-
- def apply(errors: Option[Seq[String]], warnings: Option[Seq[String]], suggestions: Option[Seq[AnyRef]]): NCInspection =
- apply(errors, warnings, suggestions, None)
-
- def apply(data: AnyRef): NCInspection = apply(None, None, None, Some(data))
-
- def apply(): NCInspection = NCInspection(None, None, None, None)
-}
\ No newline at end of file
+ * Inspection descriptor.
+ */
+trait NCInspection {
+ /**
+ * Globally unique, internal inspection ID.
+ *
+ * @return
+ */
+ def id(): String
+
+ /**
+ * User-visible name of the inspection.
+ *
+ * @return
+ */
+ def name(): String
+
+ /**
+ * Short, one-line, description.
+ *
+ * @return
+ */
+ def synopsis(): String
+
+ /**
+ *
+ * @return
+ */
+ def parameters(): java.util.List[NCInspectionParameter]
+
+ /**
+ * Full description of this inspection additionally to the synopsis.
+ *
+ * @return
+ */
+ def description(): String
+
+ /**
+ *
+ * @return
+ */
+ def isServerSide: Boolean
+}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections2/NCInspectionParameter.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionParameter.scala
similarity index 88%
rename from nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections2/NCInspectionParameter.scala
rename to nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionParameter.scala
index 9eea5ec..27b43c1 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections2/NCInspectionParameter.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionParameter.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.nlpcraft.common.inspections2
+package org.apache.nlpcraft.common.inspections
/**
* Parameter descriptor for the inspection.
@@ -37,13 +37,13 @@ trait NCInspectionParameter {
*
* @return
*/
- def value(): Option[String]
+ def value(): String
/**
*
* @return
*/
- def valueType(): Option[String]
+ def valueType(): String
/**
*
@@ -55,5 +55,5 @@ trait NCInspectionParameter {
*
* @return
*/
- def description(): Option[String]
+ def description(): String
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections2/NCInspectionResult.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionResult.scala
similarity index 89%
rename from nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections2/NCInspectionResult.scala
rename to nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionResult.scala
index 086345b..5118a0c 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections2/NCInspectionResult.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionResult.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.nlpcraft.common.inspections2
+package org.apache.nlpcraft.common.inspections
/**
*
@@ -24,12 +24,12 @@ trait NCInspectionResult {
/**
*
*/
- def inspectionId();
+ def inspectionId(): String
/**
*
*/
- def modelId();
+ def modelId(): String
/**
*
@@ -65,5 +65,5 @@ trait NCInspectionResult {
*
* @return
*/
- def suggestions(): java.util.List[String]
+ def suggestions(): java.util.List[AnyRef]
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionImpl.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionImpl.scala
new file mode 100644
index 0000000..e5ee15b
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionImpl.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.common.inspections.impl
+
+import org.apache.nlpcraft.common.inspections.{NCInspection, NCInspectionParameter}
+
+import scala.collection.JavaConverters.seqAsJavaListConverter
+
+case class NCInspectionImpl(
+ id: String,
+ name: String,
+ synopsis: String,
+ parameters: java.util.List[NCInspectionParameter],
+ description: String,
+ isServerSide: Boolean
+) extends NCInspection
+
+object NCInspectionImpl {
+ def apply(
+ id: String,
+ name: String,
+ synopsis: String,
+ parameters: Seq[NCInspectionParameter],
+ description: String,
+ isServerSide: Boolean,
+ ) : NCInspectionImpl =
+ NCInspectionImpl(
+ id,
+ name,
+ synopsis,
+ parameters = parameters.asJava,
+ description,
+ isServerSide
+ )
+}
\ No newline at end of file
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionType.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionParameterImpl.scala
similarity index 72%
rename from nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionType.scala
rename to nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionParameterImpl.scala
index 595ba7a..cf8f5f2 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionType.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionParameterImpl.scala
@@ -15,13 +15,15 @@
* limitations under the License.
*/
-package org.apache.nlpcraft.common.inspections
+package org.apache.nlpcraft.common.inspections.impl
-object NCInspectionType extends Enumeration {
- type NCInspectionType = Value
+import org.apache.nlpcraft.common.inspections.NCInspectionParameter
- val SUGGEST_SYNONYMS: Value = Value
- val INSPECTION_MACROS: Value = Value
- val INSPECTION_SYNONYMS: Value = Value
- val INSPECTION_INTENTS: Value = Value
-}
+case class NCInspectionParameterImpl(
+ id: String,
+ name: String,
+ value: String,
+ valueType: String,
+ synopsis: String,
+ description: String
+) extends NCInspectionParameter
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionResultImpl.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionResultImpl.scala
new file mode 100644
index 0000000..5be8bb7
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionResultImpl.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.common.inspections.impl
+
+import org.apache.nlpcraft.common.inspections.NCInspectionResult
+
+import scala.collection.JavaConverters.seqAsJavaListConverter
+
+case class NCInspectionResultImpl(
+ inspectionId: String,
+ modelId: String,
+ inspectionArguments: String ,
+ durationMs: Long,
+ timestamp: Long,
+ errors: java.util.List[String],
+ warnings: java.util.List[String],
+ suggestions: java.util.List[AnyRef]
+) extends NCInspectionResult
+
+object NCInspectionResultImpl {
+ def apply(
+ inspectionId: String,
+ modelId: String,
+ inspectionArguments: Option[String],
+ durationMs: Long,
+ timestamp: Long,
+ errors: Seq[String],
+ warnings: Seq[String],
+ suggestions: Seq[AnyRef]
+ ): NCInspectionResultImpl =
+ new NCInspectionResultImpl(
+ inspectionId,
+ modelId,
+ inspectionArguments.orNull,
+ durationMs,
+ timestamp,
+ errors.asJava,
+ warnings.asJava,
+ suggestions.asJava
+ )
+}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections2/NCInspection.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections2/NCInspection.scala
deleted file mode 100644
index 74bd02a..0000000
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections2/NCInspection.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.common.inspections2
-
-import io.opencensus.trace.Span
-
-/**
- * Inspection descriptor.
- */
-trait NCInspection {
- /**
- * Globally unique, internal inspection ID.
- *
- * @return
- */
- def id(): String
-
- /**
- * User-visible name of the inspection.
- *
- * @return
- */
- def name(): String
-
- /**
- * Short, one-line, description.
- *
- * @return
- */
- def synopsis(): String
-
- /**
- *
- * @return
- */
- def parameters(): Seq[NCInspectionParameter]
-
- /**
- * Full description of this inspection additionally to the synopsis.
- *
- * @return
- */
- def description(): Option[String]
-
- /**
- *
- * @return
- */
- def isServerSide: Boolean
-}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
index ea55a1b..dad67ab 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
@@ -36,7 +36,7 @@ import org.apache.nlpcraft.probe.mgrs.conn.NCConnectionManager
import org.apache.nlpcraft.probe.mgrs.conversation.NCConversationManager
import org.apache.nlpcraft.probe.mgrs.deploy.NCDeployManager
import org.apache.nlpcraft.probe.mgrs.dialogflow.NCDialogFlowManager
-import org.apache.nlpcraft.probe.mgrs.inspections.NCProbeInspectionManager
+import org.apache.nlpcraft.probe.mgrs.inspections.NCInspectionManager
import org.apache.nlpcraft.probe.mgrs.lifecycle.NCLifecycleManager
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnrichmentManager
@@ -423,7 +423,7 @@ private [probe] object NCProbeBoot extends LazyLogging with NCOpenCensusTrace {
NCNlpCoreManager.start(span)
NCNumericManager.start(span)
NCDeployManager.start(span)
- NCProbeInspectionManager.start(span)
+ NCInspectionManager.start(span)
NCModelManager.start(span)
NCCommandManager.start(span)
NCDictionaryManager.start(span)
@@ -463,7 +463,7 @@ private [probe] object NCProbeBoot extends LazyLogging with NCOpenCensusTrace {
NCDictionaryManager.stop(span)
NCCommandManager.stop(span)
NCModelManager.stop(span)
- NCProbeInspectionManager.stop(span)
+ NCInspectionManager.stop(span)
NCDeployManager.stop(span)
NCNumericManager.stop(span)
NCNlpCoreManager.stop(span)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
index 5f08876..180f531 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
@@ -18,19 +18,18 @@
package org.apache.nlpcraft.probe.mgrs.cmd
import java.io.Serializable
-import java.util
import com.google.gson.Gson
import io.opencensus.trace.Span
import org.apache.nlpcraft.common.NCService
-import org.apache.nlpcraft.common.inspections.NCInspectionType
import org.apache.nlpcraft.common.nlp.NCNlpSentence
import org.apache.nlpcraft.model.NCToken
import org.apache.nlpcraft.probe.mgrs.NCProbeMessage
import org.apache.nlpcraft.probe.mgrs.conn.NCConnectionManager
import org.apache.nlpcraft.probe.mgrs.conversation.NCConversationManager
import org.apache.nlpcraft.probe.mgrs.dialogflow.NCDialogFlowManager
-import org.apache.nlpcraft.probe.mgrs.inspections.NCProbeInspectionManager
+import org.apache.nlpcraft.probe.mgrs.inspections.NCInspectionManager
+import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnrichmentManager
import scala.collection.JavaConverters._
@@ -93,25 +92,36 @@ object NCCommandManager extends NCService {
span
)
- case "S2P_MODEL_INSPECTION" ⇒
- val res =
- NCProbeInspectionManager.inspect(
- mdlId = msg.data[String]("mdlId"),
- types =
- msg.data[java.util.List[String]]("types").
- asScala.
- map(p ⇒ NCInspectionType.withName(p.toUpperCase)),
- span
- )
+ case "S2P_PROBE_INSPECTION" ⇒
+ NCInspectionManager.inspect(
+ mdlId = msg.data[String]("mdlId"),
+ inspId = msg.data[String]("inspId"),
+ args = msg.data[String]("args"),
+ span
+ ).collect {
+ case res ⇒
+ NCConnectionManager.send(
+ NCProbeMessage(
+ "P2S_PROBE_INSPECTION",
+ "reqGuid" → msg.getGuid,
+ "resp" → GSON.toJson(res)
+ ),
+ span
+ )
+
+ }(scala.concurrent.ExecutionContext.Implicits.global)
+
+ case "S2P_MODEL_INFO" ⇒
+ val res = NCModelManager.getModelTransferData(msg.data[String]("mdlId"))
NCConnectionManager.send(
- NCProbeMessage(
- "P2S_MODEL_INSPECTION",
- "reqGuid" → msg.getGuid,
- "resp" → GSON.toJson(res.map(p ⇒ p._1.toString → p._2).asJava)
- ),
- span
- )
+ NCProbeMessage(
+ "P2S_MODEL_INFO",
+ "reqGuid" → msg.getGuid,
+ "resp" → GSON.toJson(res)
+ ),
+ span
+ )
case _ ⇒
logger.error(s"Received unknown server message (you need to update the probe): ${msg.getType}")
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
index 466b51f..e696bde 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
@@ -23,12 +23,10 @@ import java.util.jar.{JarInputStream ⇒ JIS}
import io.opencensus.trace.Span
import org.apache.nlpcraft.common._
import org.apache.nlpcraft.common.config.NCConfigurable
-import org.apache.nlpcraft.common.inspections.NCInspectionType
import org.apache.nlpcraft.model._
import org.apache.nlpcraft.model.factories.basic.NCBasicModelFactory
import org.apache.nlpcraft.model.impl.NCModelWrapper
import org.apache.nlpcraft.model.intent.impl.{NCIntentScanner, NCIntentSolver}
-import org.apache.nlpcraft.probe.mgrs.inspections.NCProbeInspectionManager
import resource.managed
import scala.collection.JavaConverters._
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCProbeInspectionManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
similarity index 60%
rename from nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCProbeInspectionManager.scala
rename to nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
index 91963fe..2b65771 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCProbeInspectionManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
@@ -18,24 +18,24 @@
package org.apache.nlpcraft.probe.mgrs.inspections
import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.inspections.NCInspectionType._
-import org.apache.nlpcraft.common.inspections.{NCInspection, NCInspectionType}
+import org.apache.nlpcraft.common.inspections.NCInspectionResult
import org.apache.nlpcraft.common.{NCE, NCService}
import org.apache.nlpcraft.model.opencensus.stats.NCOpenCensusModelStats
-import org.apache.nlpcraft.probe.mgrs.inspections.inspectors._
+import org.apache.nlpcraft.probe.mgrs.inspections.inspectors.{NCInspectorIntents, NCInspectorMacros, NCInspectorSynonyms}
-// TODO:
-object NCProbeInspectionManager extends NCService with NCOpenCensusModelStats {
+import scala.concurrent.Future
+
+/**
+ *
+ */
+object NCInspectionManager extends NCService with NCOpenCensusModelStats {
private final val INSPECTORS =
Map(
- SUGGEST_SYNONYMS → NCInspectorSynonymsSuggestions,
- INSPECTION_MACROS → NCInspectorMacros,
- INSPECTION_SYNONYMS → NCInspectorSynonyms,
- INSPECTION_INTENTS → NCInspectorIntents
+ "macros" → NCInspectorMacros,
+ "synonyms" → NCInspectorSynonyms,
+ "intents" → NCInspectorIntents
)
- require(NCInspectionType.values.forall(INSPECTORS.contains))
-
override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ ⇒
INSPECTORS.values.foreach(_.start())
@@ -48,9 +48,18 @@ object NCProbeInspectionManager extends NCService with NCOpenCensusModelStats {
INSPECTORS.values.foreach(_.stop())
}
- @throws[NCE]
- def inspect(mdlId: String, types: Seq[NCInspectionType], parent: Span = null): Map[NCInspectionType, NCInspection] =
- startScopedSpan("inspect", parent, "modelId" → mdlId, "types" → types.map(_.toString)) { _ ⇒
- types.map(t ⇒ t → INSPECTORS(t).inspect(mdlId, parent = parent)).toMap
+ /**
+ *
+ * @param mdlId Model ID.
+ * @param inspId Inspection ID.
+ * @param args Inspection arguments.
+ * @param parent Optional parent trace span.
+ */
+ def inspect(mdlId: String, inspId: String, args: String, parent: Span = null): Future[NCInspectionResult] =
+ startScopedSpan("inspect", parent, "modelId" → mdlId, "inspectionId" → inspId) { _ ⇒
+ INSPECTORS.get(inspId) match {
+ case Some(inspector) ⇒ inspector.inspect(mdlId, inspId, args)
+ case None ⇒ throw new NCE(s"Unsupported inspection: $inspId")
+ }
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspector.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
similarity index 73%
rename from nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspector.scala
rename to nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
index a0d5310..09adf31 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspector.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
@@ -15,17 +15,13 @@
* limitations under the License.
*/
-package org.apache.nlpcraft.common.inspections
+package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
import io.opencensus.trace.Span
+import org.apache.nlpcraft.common.inspections.NCInspectionResult
-// TODO:
-trait NCInspector {
- /**
- *
- * @param mdlId
- * @param prevLayerInspection
- * @param parent
- */
- def inspect(mdlId: String, prevLayerInspection: Option[NCInspection] = None, parent: Span = null): NCInspection
+import scala.concurrent.Future
+
+private[inspections] trait NCInspector {
+ def inspect(mdlId: String, inspId: String, args: String, parent: Span = null): Future[NCInspectionResult]
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
index a973c21..1feab43 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
@@ -18,7 +18,8 @@
package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.inspections.{NCInspection, NCInspector}
+import org.apache.nlpcraft.common.inspections.NCInspectionResult
+import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
import org.apache.nlpcraft.common.makro.NCMacroParser
import org.apache.nlpcraft.common.nlp.core.NCNlpPorterStemmer
import org.apache.nlpcraft.common.{NCE, NCService}
@@ -27,42 +28,56 @@ import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import scala.collection.JavaConverters._
import scala.collection._
+import scala.concurrent.Future
// TODO:
object NCInspectorIntents extends NCService with NCInspector {
private final val SEPARATORS = Seq('?', ',', '.', '-', '!')
- override def inspect(mdlId: String, prevLayerInspection: Option[NCInspection], parent: Span = null): NCInspection =
+ override def inspect(mdlId: String, inspId: String, args: String, parent: Span = null): Future[NCInspectionResult] =
startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
- val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new NCE(s"Model not found: '$mdlId'")).model
+ Future {
+ val now = System.currentTimeMillis()
- val res = NCIntentScanner.scanIntentsSamples(mdl.proxy)
+ val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new NCE(s"Model not found: '$mdlId'")).model
- val warns = mutable.ArrayBuffer.empty[String] ++ res.warnings
+ val res = NCIntentScanner.scanIntentsSamples(mdl.proxy)
- val parser = new NCMacroParser
+ val warns = mutable.ArrayBuffer.empty[String] ++ res.warnings
- mdl.getMacros.asScala.foreach { case (name, str) ⇒ parser.addMacro(name, str) }
+ val parser = new NCMacroParser
- val allSyns: Set[Seq[String]] =
- mdl.getElements.
- asScala.
- flatMap(_.getSynonyms.asScala.flatMap(parser.expand)).
- map(NCNlpPorterStemmer.stem).map(_.split(" ").toSeq).
- toSet
+ mdl.getMacros.asScala.foreach { case (name, str) ⇒ parser.addMacro(name, str) }
- res.samples.
- flatMap { case (_, samples) ⇒ samples.map(_.toLowerCase) }.
- // Note that we don't use system tokenizer, because ContextWordServer doesn't have this tokenizer.
- // We just split examples words with spaces. Also we divide SEPARATORS as separated words.
- map(s ⇒ s → SEPARATORS.foldLeft(s)((s, ch) ⇒ s.replaceAll(s"\\$ch", s" $ch "))).
- foreach { case (s, sNorm) ⇒
- val seq: Seq[String] = sNorm.split(" ").map(NCNlpPorterStemmer.stem)
+ val allSyns: Set[Seq[String]] =
+ mdl.getElements.
+ asScala.
+ flatMap(_.getSynonyms.asScala.flatMap(parser.expand)).
+ map(NCNlpPorterStemmer.stem).map(_.split(" ").toSeq).
+ toSet
- if (!allSyns.exists(_.intersect(seq).nonEmpty))
- warns += s"Sample: '$s' doesn't contain synonyms"
- }
+ res.samples.
+ flatMap { case (_, samples) ⇒ samples.map(_.toLowerCase) }.
+ // Note that we don't use system tokenizer, because ContextWordServer doesn't have this tokenizer.
+ // We just split examples words with spaces. Also we divide SEPARATORS as separated words.
+ map(s ⇒ s → SEPARATORS.foldLeft(s)((s, ch) ⇒ s.replaceAll(s"\\$ch", s" $ch "))).
+ foreach { case (s, sNorm) ⇒
+ val seq: Seq[String] = sNorm.split(" ").map(NCNlpPorterStemmer.stem)
- NCInspection(errors = None, warnings = if (warns.isEmpty) None else Some(warns), suggestions = None)
+ if (!allSyns.exists(_.intersect(seq).nonEmpty))
+ warns += s"Sample: '$s' doesn't contain synonyms"
+ }
+
+ NCInspectionResultImpl(
+ inspectionId = inspId,
+ modelId = mdlId,
+ inspectionArguments = None,
+ durationMs = System.currentTimeMillis() - now,
+ timestamp = now,
+ errors = Seq.empty,
+ warnings = warns,
+ suggestions = Seq.empty
+ )
+ }(scala.concurrent.ExecutionContext.Implicits.global)
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
index ba2c8bc..eb09059 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
@@ -18,26 +18,42 @@
package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.inspections.{NCInspection, NCInspector}
+import org.apache.nlpcraft.common.inspections.NCInspectionResult
+import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
import org.apache.nlpcraft.common.{NCE, NCService}
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import scala.collection.JavaConverters._
+import scala.collection.Seq
+import scala.concurrent.Future
// TODO:
object NCInspectorMacros extends NCService with NCInspector {
- override def inspect(mdlId: String, prevLayerInspection: Option[NCInspection], parent: Span = null): NCInspection =
+ override def inspect(mdlId: String, inspId: String, args: String, parent: Span = null): Future[NCInspectionResult] =
startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
- val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new NCE(s"Model not found: '$mdlId'")).model
+ Future {
+ val now = System.currentTimeMillis()
- val syns = mdl.getElements.asScala.flatMap(_.getSynonyms.asScala)
+ val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new NCE(s"Model not found: '$mdlId'")).model
- val warns =
- mdl.getMacros.asScala.keys.
- // TODO: is it valid check? (simple contains)
- flatMap(m ⇒ if (syns.exists(_.contains(m))) None else Some(s"Macro is not used: $m")).
- toSeq
+ val syns = mdl.getElements.asScala.flatMap(_.getSynonyms.asScala)
- NCInspection(errors = None, warnings = if (warns.isEmpty) None else Some(warns), suggestions = None)
+ val warns =
+ mdl.getMacros.asScala.keys.
+ // TODO: is it valid check? (simple contains)
+ flatMap(m ⇒ if (syns.exists(_.contains(m))) None else Some(s"Macro is not used: $m")).
+ toSeq
+
+ NCInspectionResultImpl(
+ inspectionId = inspId,
+ modelId = mdlId,
+ inspectionArguments = None,
+ durationMs = System.currentTimeMillis() - now,
+ timestamp = now,
+ errors = Seq.empty,
+ warnings = warns,
+ suggestions = Seq.empty
+ )
+ }(scala.concurrent.ExecutionContext.Implicits.global)
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
index bee9648..112a0dd 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
@@ -18,47 +18,62 @@
package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.inspections.{NCInspection, NCInspector}
+import org.apache.nlpcraft.common.inspections.NCInspectionResult
+import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
import org.apache.nlpcraft.common.makro.NCMacroParser
import org.apache.nlpcraft.common.{NCE, NCService}
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
import scala.collection.JavaConverters._
-import scala.collection.mutable
+import scala.collection.{Seq, mutable}
+import scala.concurrent.Future
// TODO:
object NCInspectorSynonyms extends NCService with NCInspector {
private final val TOO_MANY_SYNS = 10000
- override def inspect(mdlId: String, prevLayerInspection: Option[NCInspection], parent: Span = null): NCInspection =
+ override def inspect(mdlId: String, inspId: String, args: String, parent: Span = null): Future[NCInspectionResult] =
startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
- val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new NCE(s"Model not found: '$mdlId'")).model
+ Future {
+ val now = System.currentTimeMillis()
- val warns = mutable.ArrayBuffer.empty[String]
+ val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new NCE(s"Model not found: '$mdlId'")).model
- val parser = new NCMacroParser()
+ val warns = mutable.ArrayBuffer.empty[String]
- mdl.getMacros.asScala.foreach { case (name, str) ⇒ parser.addMacro(name, str) }
+ val parser = new NCMacroParser()
- val mdlSyns = mdl.getElements.asScala.map(p ⇒ p.getId → p.getSynonyms.asScala.flatMap(parser.expand))
+ mdl.getMacros.asScala.foreach { case (name, str) ⇒ parser.addMacro(name, str) }
- mdlSyns.foreach { case (elemId, syns) ⇒
- val size = syns.size
+ val mdlSyns = mdl.getElements.asScala.map(p ⇒ p.getId → p.getSynonyms.asScala.flatMap(parser.expand))
- if (size == 0)
- warns += s"Element: '$elemId' doesn't have synonyms"
- else if (size > TOO_MANY_SYNS)
- warns += s"Element: '$elemId' has too many synonyms: $size"
+ mdlSyns.foreach { case (elemId, syns) ⇒
+ val size = syns.size
- val others = mdlSyns.filter { case (othId, _) ⇒ othId != elemId}
+ if (size == 0)
+ warns += s"Element: '$elemId' doesn't have synonyms"
+ else if (size > TOO_MANY_SYNS)
+ warns += s"Element: '$elemId' has too many synonyms: $size"
- val intersects =
- others.filter { case (_, othSyns) ⇒ othSyns.intersect(syns).nonEmpty }.toMap.keys.mkString(",")
+ val others = mdlSyns.filter { case (othId, _) ⇒ othId != elemId }
- if (intersects.nonEmpty)
- warns += s"Element: '$elemId' has same synonyms with '$intersects'"
- }
+ val intersects =
+ others.filter { case (_, othSyns) ⇒ othSyns.intersect(syns).nonEmpty }.toMap.keys.mkString(",")
- NCInspection(errors = None, warnings = if (warns.isEmpty) None else Some(warns), suggestions = None)
+ if (intersects.nonEmpty)
+ warns += s"Element: '$elemId' has same synonyms with '$intersects'"
+ }
+
+ NCInspectionResultImpl(
+ inspectionId = inspId,
+ modelId = mdlId,
+ inspectionArguments = None,
+ durationMs = System.currentTimeMillis() - now,
+ timestamp = now,
+ errors = Seq.empty,
+ warnings = warns,
+ suggestions = Seq.empty
+ )
+ }(scala.concurrent.ExecutionContext.Implicits.global)
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonymsSuggestions.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonymsSuggestions.scala
deleted file mode 100644
index 76bddb4..0000000
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonymsSuggestions.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
-
-import java.util
-
-import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.inspections.{NCInspection, NCInspector}
-import org.apache.nlpcraft.common.{NCE, NCService}
-import org.apache.nlpcraft.model.intent.impl.NCIntentScanner
-import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
-
-import scala.collection.JavaConverters._
-
-// TODO:
-object NCInspectorSynonymsSuggestions extends NCService with NCInspector {
- override def inspect(mdlId: String, prevLayerInspection: Option[NCInspection], parent: Span = null): NCInspection =
- startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
- val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new NCE(s"Model not found: '$mdlId'")).model
-
- val data = new util.HashMap[String, Any]()
-
- data.put("macros", mdl.getMacros)
- data.put("elementsSynonyms", mdl.getElements.asScala.map(p ⇒ p.getId → p.getSynonyms).toMap.asJava)
- data.put("intentsSamples", NCIntentScanner.scanIntentsSamples(mdl.proxy).samples.map(p ⇒ p._1 → p._2.asJava).asJava)
-
- NCInspection(data = data)
- }
-}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections2/NCInspectionManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections2/NCInspectionManager.scala
deleted file mode 100644
index 8f8ff7a..0000000
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections2/NCInspectionManager.scala
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.probe.mgrs.inspections2
-
-import org.apache.nlpcraft.common.NCService
-import org.apache.nlpcraft.model.opencensus.stats.NCOpenCensusModelStats
-import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.inspections2.NCInspectionResult
-
-/**
- *
- */
-object NCInspectionManager extends NCService with NCOpenCensusModelStats {
- override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ ⇒
- // TODO
-
- super.start(parent)
- }
-
- override def stop(parent: Span): Unit = startScopedSpan("stop", parent) { _ ⇒
- super.stop()
-
- // TODO
- }
-
- /**
- *
- * @param mdlId Model ID.
- * @param inspId Inspection ID.
- * @param inspArgs Inspection arguments as JSON string.
- * @param parent Optional parent trace span.
- * @return
- */
- def inspect(mdlId: String, inspId: String, inspArgs: String, parent: Span = null): NCInspectionResult = ???
-}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
index e195c0a..1deb26e 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
@@ -17,25 +17,26 @@
package org.apache.nlpcraft.probe.mgrs.model
+import java.util
import java.util.regex.{Pattern, PatternSyntaxException}
import io.opencensus.trace.Span
import org.apache.nlpcraft.common._
-import org.apache.nlpcraft.common.util.NCUtils._
import org.apache.nlpcraft.common.ascii.NCAsciiTable
-import org.apache.nlpcraft.common.inspections.NCInspectionType
import org.apache.nlpcraft.common.makro.NCMacroParser
import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
+import org.apache.nlpcraft.common.util.NCUtils._
import org.apache.nlpcraft.model._
import org.apache.nlpcraft.model.impl.NCModelWrapper
import org.apache.nlpcraft.model.intent.impl.NCIntentScanner
import org.apache.nlpcraft.probe.mgrs.NCSynonymChunkKind._
import org.apache.nlpcraft.probe.mgrs.deploy._
-import org.apache.nlpcraft.probe.mgrs.inspections.NCProbeInspectionManager
+import org.apache.nlpcraft.probe.mgrs.inspections.NCInspectionManager
import org.apache.nlpcraft.probe.mgrs.{NCModelDecorator, NCSynonym, NCSynonymChunk}
-import collection.convert.ImplicitConversions._
+import scala.collection.JavaConverters._
import scala.collection.convert.DecorateAsScala
+import scala.collection.convert.ImplicitConversions._
import scala.collection.mutable
import scala.collection.mutable.ListBuffer
import scala.util.control.Exception._
@@ -52,6 +53,8 @@ object NCModelManager extends NCService with DecorateAsScala {
// Access mutex.
private final val mux = new Object()
+ private final val DFLT_INSPECTIONS = Seq("macros", "intents", "synonyms")
+
/**
*
* @param elementId Element ID.
@@ -108,18 +111,23 @@ object NCModelManager extends NCService with DecorateAsScala {
tbl.info(logger, Some(s"Models deployed: ${models.size}\n"))
- models.values.foreach(mdl ⇒ {
+ for (mdl ← models.values; insId ← DFLT_INSPECTIONS) {
val mdlId = mdl.model.getId
- val inspections = NCProbeInspectionManager.inspect(mdlId, NCInspectionType.values.toSeq)
+ NCInspectionManager.inspect(mdlId, insId, null, parent).collect {
+ case res ⇒
+ res.errors().asScala.foreach(
+ p ⇒ logger.error(s"Validation error [model=$mdlId, inspection=$insId, text=$p")
+ )
+ res.warnings().asScala.foreach(
+ p ⇒ logger.warn(s"Validation warning [model=$mdlId, inspection=$insId, text=$p")
+ )
+ res.suggestions().asScala.foreach(
+ p ⇒ logger.info(s"Validation suggestion [model=$mdlId, inspection=$insId, text=$p")
+ )
+ }(scala.concurrent.ExecutionContext.Implicits.global)
+ }
- inspections.foreach { case(t, i) ⇒
- i.errors.asScala.foreach(p ⇒ logger.error(s"Validation error [model=$mdlId, type=$t, text=$p"))
- i.warnings.asScala.foreach(p ⇒ logger.warn(s"Validation warning [model=$mdlId, type=$t, text=$p"))
- i.suggestions.asScala.foreach(p ⇒ logger.info(s"Validation suggestion [model=$mdlId, type=$t, text=$p"))
- }
- })
-
addTags(
span,
"deployedModels" → models.values.map(_.model.getId).mkString(",")
@@ -731,13 +739,32 @@ object NCModelManager extends NCService with DecorateAsScala {
/**
*
- * @param id Model ID.
+ * @param mdlId Model ID.
* @return
*/
- def getModel(id: String, parent: Span = null): Option[NCModelDecorator] =
- startScopedSpan("getModel", parent, "id" → id) { _ ⇒
+ def getModel(mdlId: String, parent: Span = null): Option[NCModelDecorator] =
+ startScopedSpan("getModel", parent, "modelId" → mdlId) { _ ⇒
mux.synchronized {
- models.get(id)
+ models.get(mdlId)
}
}
+
+ /**
+ * Gets model data which can be transfered between probe and server.
+ *
+ * @param mdlId Model ID.
+ */
+ def getModelTransferData(mdlId: String, parent: Span = null): java.util.Map[String, Any] =
+ startScopedSpan("getModel", parent, "mdlId" → mdlId) { _ ⇒
+ val mdl = mux.synchronized { models.get(mdlId) }.
+ getOrElse(throw new NCE(s"Model not found: '$mdlId'")).model
+
+ val data = new util.HashMap[String, Any]()
+
+ data.put("macros", mdl.getMacros)
+ data.put("elementsSynonyms", mdl.getElements.asScala.map(p ⇒ p.getId → p.getSynonyms).toMap.asJava)
+ data.put("intentsSamples", NCIntentScanner.scanIntentsSamples(mdl.proxy).samples.map(p ⇒ p._1 → p._2.asJava).asJava)
+
+ data
+ }
}
\ No newline at end of file
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
index 3277039..cb71f34 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
@@ -24,16 +24,17 @@ import com.typesafe.scalalogging.LazyLogging
import org.apache.nlpcraft.common._
import org.apache.nlpcraft.common.ascii.NCAsciiTable
import org.apache.nlpcraft.common.config.NCConfigurable
+import org.apache.nlpcraft.common.extcfg.NCExternalConfigManager
import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.nlp.dict.NCDictionaryManager
import org.apache.nlpcraft.common.nlp.numeric.NCNumericManager
import org.apache.nlpcraft.common.opencensus.NCOpenCensusTrace
-import org.apache.nlpcraft.common.extcfg.NCExternalConfigManager
import org.apache.nlpcraft.common.version._
import org.apache.nlpcraft.server.company.NCCompanyManager
import org.apache.nlpcraft.server.feedback.NCFeedbackManager
import org.apache.nlpcraft.server.geo.NCGeoManager
import org.apache.nlpcraft.server.ignite.{NCIgniteInstance, NCIgniteRunner}
+import org.apache.nlpcraft.server.inspections.NCInspectionManager
import org.apache.nlpcraft.server.lifecycle.NCServerLifecycleManager
import org.apache.nlpcraft.server.nlp.core.NCNlpServerManager
import org.apache.nlpcraft.server.nlp.enrichers.NCServerEnrichmentManager
@@ -45,7 +46,6 @@ import org.apache.nlpcraft.server.proclog.NCProcessLogManager
import org.apache.nlpcraft.server.query.NCQueryManager
import org.apache.nlpcraft.server.rest.NCRestManager
import org.apache.nlpcraft.server.sql.NCSqlManager
-import org.apache.nlpcraft.server.inspections.NCServerInspectorManager
import org.apache.nlpcraft.server.tx.NCTxManager
import org.apache.nlpcraft.server.user.NCUserManager
@@ -123,7 +123,7 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
},
() ⇒ {
NCProbeManager.start(span)
- NCServerInspectorManager.start(span)
+ NCInspectionManager.start(span)
},
() ⇒ NCFeedbackManager.start(span)
)
@@ -148,7 +148,7 @@ object NCServer extends App with NCIgniteInstance with LazyLogging with NCOpenCe
NCRestManager,
NCQueryManager,
NCFeedbackManager,
- NCServerInspectorManager,
+ NCInspectionManager,
NCProbeManager,
NCCompanyManager,
NCUserManager,
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCInspectionManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCInspectionManager.scala
new file mode 100644
index 0000000..5c46416
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCInspectionManager.scala
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.server.inspections
+
+import io.opencensus.trace.Span
+import org.apache.nlpcraft.common.NCService
+import org.apache.nlpcraft.common.inspections.impl.NCInspectionImpl
+import org.apache.nlpcraft.common.inspections.{NCInspection, NCInspectionResult}
+import org.apache.nlpcraft.server.inspections.inspectors.NCInspectorSuggestions
+import org.apache.nlpcraft.server.probe.NCProbeManager
+
+import scala.collection.Map
+import scala.concurrent.Future
+
+/**
+ *
+ */
+object NCInspectionManager extends NCService {
+ private final val INSPECTIONS: Seq[NCInspection] =
+ Seq(
+ NCInspectionImpl(
+ id = "macros",
+ name = "macros",
+ synopsis = "macros",
+ parameters = Seq.empty,
+ description = "macros",
+ isServerSide = false
+ ),
+ NCInspectionImpl(
+ id = "elements",
+ name = "elements",
+ synopsis = "elements",
+ parameters = Seq.empty,
+ description = "elements",
+ isServerSide = false
+ ),
+ NCInspectionImpl(
+ id = "synonyms",
+ name = "synonyms",
+ synopsis = "synonyms",
+ parameters = Seq.empty,
+ description = "synonyms",
+ isServerSide = false
+ ),
+ NCInspectionImpl(
+ id = "synonyms_suggestions",
+ name = "synonyms_suggestions",
+ synopsis = "synonyms_suggestions",
+ parameters = Seq.empty,
+ description = "synonyms_suggestions",
+ isServerSide = true
+ )
+ )
+
+ private final val SRV_INSPECTORS =
+ Map(
+ "suggestions" → NCInspectorSuggestions
+ )
+
+ override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ ⇒
+ SRV_INSPECTORS.values.foreach(_.start())
+
+ super.start(parent)
+ }
+
+ override def stop(parent: Span): Unit = startScopedSpan("stop", parent) { _ ⇒
+ super.stop()
+
+ SRV_INSPECTORS.values.foreach(_.stop(parent))
+ }
+
+ /**
+ *
+ * @param mdlId Model ID.
+ * @param inspId Inspection ID.
+ * @param args Inspection arguments .
+ * @param parent Optional parent trace span.
+ */
+ def inspect(mdlId: String, inspId: String, args: String, parent: Span = null): Future[NCInspectionResult] =
+ SRV_INSPECTORS.get(inspId) match {
+ case Some(inspector) ⇒ inspector.inspect(mdlId, inspId, args, parent)
+ case None ⇒ NCProbeManager.getProbeInspection(mdlId, inspId, args, parent)
+ }
+ /**
+ * Gets all supported server and probe inspections.
+ *
+ * @param parent
+ * @return
+ */
+ def getInspections(parent: Span = null): Seq[NCInspection] = INSPECTIONS
+}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCServerInspectorManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCServerInspectorManager.scala
deleted file mode 100644
index 4761039..0000000
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCServerInspectorManager.scala
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.server.inspections
-
-import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.inspections.NCInspection
-import org.apache.nlpcraft.common.inspections.NCInspectionType._
-import org.apache.nlpcraft.common.{NCE, NCService}
-import org.apache.nlpcraft.server.inspections.inspectors._
-import org.apache.nlpcraft.server.probe.NCProbeManager
-
-import scala.collection.JavaConverters._
-import scala.collection._
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.{Future, Promise}
-import scala.util.{Failure, Success}
-
-/**
- * TODO: + check all texts
- */
-object NCServerInspectorManager extends NCService {
- private final val INSPECTORS =
- Map(
- SUGGEST_SYNONYMS → NCInspectorSynonymsSuggestions
- )
-
- override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ ⇒
- INSPECTORS.values.foreach(_.start())
-
- super.start(parent)
- }
-
- override def stop(parent: Span): Unit = startScopedSpan("stop", parent) { _ ⇒
- super.stop()
-
- INSPECTORS.values.foreach(_.stop())
- }
-
- /**
- *
- * @param mdlId
- * @param types
- * @param parent
- */
- @throws[NCE]
- def inspect(mdlId: String, types: Seq[NCInspectionType], parent: Span = null): Future[Map[NCInspectionType, NCInspection]] =
- startScopedSpan("enhance", parent, "modelId" → mdlId, "types" → types.map(_.toString)) { _ ⇒
- val promise = Promise[Map[NCInspectionType, NCInspection]]()
-
- NCProbeManager.inspect(mdlId, types, parent).onComplete {
- case Success(probeRes) ⇒
- val srvRes =
- probeRes.map { case (typ, inspectionProbe) ⇒
- val inspectionSrv = INSPECTORS.get(typ) match {
- case Some(inspector) ⇒ inspector.inspect(mdlId, Some(inspectionProbe))
- case None ⇒ NCInspection()
- }
-
- def union[T](seq1: java.util.List[T], seq2: java.util.List[T]): Option[Seq[T]] = {
- val seq = seq1.asScala ++ seq2.asScala
-
- if (seq.isEmpty) None else Some(seq)
- }
-
- // Data - we don't need pass it on last step.
- typ → NCInspection(
- errors = union(inspectionProbe.errors, inspectionSrv.errors),
- warnings = union(inspectionProbe.warnings, inspectionSrv.warnings),
- suggestions = union(inspectionProbe.suggestions, inspectionSrv.suggestions),
- )
- }
-
- promise.success(srvRes)
- case Failure(err) ⇒ throw err
- }(global)
-
- promise.future
- }
-}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspector.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspector.scala
new file mode 100644
index 0000000..9e985d4
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspector.scala
@@ -0,0 +1,10 @@
+package org.apache.nlpcraft.server.inspections.inspectors
+
+import io.opencensus.trace.Span
+import org.apache.nlpcraft.common.inspections.NCInspectionResult
+
+import scala.concurrent.Future
+
+private[inspections] trait NCInspector {
+ def inspect(mdlId: String, inspId: String, args: String, parent: Span = null): Future[NCInspectionResult]
+}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSuggestions.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSuggestions.scala
new file mode 100644
index 0000000..f9a0259
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSuggestions.scala
@@ -0,0 +1,443 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.server.inspections.inspectors
+
+import java.util
+import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
+import java.util.concurrent.{ConcurrentHashMap, CopyOnWriteArrayList, CountDownLatch, TimeUnit}
+import java.util.{List => JList}
+
+import com.google.gson.Gson
+import com.google.gson.reflect.TypeToken
+import io.opencensus.trace.Span
+import org.apache.http.HttpResponse
+import org.apache.http.client.ResponseHandler
+import org.apache.http.client.methods.HttpPost
+import org.apache.http.entity.StringEntity
+import org.apache.http.impl.client.HttpClients
+import org.apache.http.util.EntityUtils
+import org.apache.nlpcraft.common.config.NCConfigurable
+import org.apache.nlpcraft.common.inspections.NCInspectionResult
+import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
+import org.apache.nlpcraft.common.makro.NCMacroParser
+import org.apache.nlpcraft.common.nlp.core.NCNlpPorterStemmer
+import org.apache.nlpcraft.common.util.NCUtils
+import org.apache.nlpcraft.common.{NCE, NCService}
+import org.apache.nlpcraft.server.probe.NCProbeManager
+
+import scala.collection.JavaConverters._
+import scala.collection.{Seq, mutable}
+import scala.concurrent.{Future, Promise}
+
+// TODO:
+object NCInspectorSuggestions extends NCService with NCInspector {
+ // For context word server requests.
+ private final val MAX_LIMIT: Int = 10000
+ private final val BATCH_SIZE = 20
+
+ // For warnings.
+ private final val MIN_CNT_INTENT = 5
+ private final val MIN_CNT_MODEL = 20
+
+ private object Config extends NCConfigurable {
+ val urlOpt: Option[String] = getStringOpt("nlpcraft.server.ctxword.url")
+ val suggestionsMinScore: Int = getInt("nlpcraft.server.ctxword.suggestions.minScore")
+
+ @throws[NCE]
+ def check(): Unit =
+ if (suggestionsMinScore < 0 || suggestionsMinScore > 1)
+ throw new NCE("Invalid 'nlpcraft.server.ctxword.suggestions.minScore' parameter value. It should be double value between 0 and 1, inclusive")
+ }
+
+ Config.check()
+
+ case class Suggestion(word: String, score: Double)
+ case class RequestData(sentence: String, example: String, elementId: String, index: Int)
+ case class RestRequestSentence(text: String, indexes: JList[Int])
+ case class RestRequest(sentences: JList[RestRequestSentence], limit: Int, min_score: Double)
+ case class Word(word: String, stem: String) {
+ require(!word.contains(" "), s"Word cannot contains spaces: $word")
+ require(
+ word.forall(ch ⇒
+ ch.isLetterOrDigit ||
+ ch == '\'' ||
+ SEPARATORS.contains(ch)
+ ),
+ s"Unsupported symbols: $word"
+ )
+ }
+
+ case class SuggestionResult(
+ synonym: String,
+ ctxWorldServerScore: Double,
+ suggestedCount: Int
+ )
+
+ private final val GSON = new Gson
+ private final val TYPE_RESP = new TypeToken[JList[JList[Suggestion]]]() {}.getType
+ private final val SEPARATORS = Seq('?', ',', '.', '-', '!')
+
+ private final val HANDLER: ResponseHandler[Seq[Seq[Suggestion]]] =
+ (resp: HttpResponse) ⇒ {
+ val code = resp.getStatusLine.getStatusCode
+ val e = resp.getEntity
+
+ val js = if (e != null) EntityUtils.toString(e) else null
+
+ if (js == null)
+ throw new RuntimeException(s"Unexpected empty response [code=$code]")
+
+ code match {
+ case 200 ⇒
+ val data: JList[JList[Suggestion]] = GSON.fromJson(js, TYPE_RESP)
+
+ data.asScala.map(p ⇒ if (p.isEmpty) Seq.empty else p.asScala.tail)
+
+ case 400 ⇒ throw new RuntimeException(js)
+ case _ ⇒ throw new RuntimeException(s"Unexpected response [code=$code, response=$js]")
+ }
+ }
+
+ private def split(s: String): Seq[String] = s.split(" ").toSeq.map(_.trim).filter(_.nonEmpty)
+ private def toStem(s: String): String = split(s).map(NCNlpPorterStemmer.stem).mkString(" ")
+ private def toStemWord(s: String): String = NCNlpPorterStemmer.stem(s)
+
+ /**
+ *
+ * @param seq1
+ * @param seq2
+ */
+ private def getAllSlices(seq1: Seq[String], seq2: Seq[String]): Seq[Int] = {
+ val seq = mutable.Buffer.empty[Int]
+
+ var i = seq1.indexOfSlice(seq2)
+
+ while (i >= 0) {
+ seq += i
+
+ i = seq1.indexOfSlice(seq2, i + 1)
+ }
+
+ seq
+ }
+
+ override def inspect(mdlId: String, inspId: String, args: String, parent: Span = null): Future[NCInspectionResult] =
+ startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
+ val now = System.currentTimeMillis()
+
+ val promise = Promise[NCInspectionResult]()
+
+ NCProbeManager.getModelInfo(mdlId, parent).collect {
+ case data ⇒
+ println("args=" + args)
+ println("data=" + data)
+
+ val macrosJ = data.get("macros").asInstanceOf[util.Map[String, String]]
+ val elementsSynonymsJ = data.get("elementsSynonyms").asInstanceOf[util.Map[String, util.List[String]]]
+ val intentsSamplesJ = data.get("intentsSamples").asInstanceOf[util.Map[String, util.List[String]]]
+
+ require(macrosJ != null)
+ require(elementsSynonymsJ != null)
+ require(intentsSamplesJ != null)
+
+ val macros = macrosJ.asScala
+ val elementsSynonyms = elementsSynonymsJ.asScala.map(p ⇒ p._1 → p._2.asScala)
+ val intentsSamples = intentsSamplesJ.asScala.map(p ⇒ p._1 → p._2.asScala)
+
+ def onError(err: String): Unit =
+ promise.success(
+ NCInspectionResultImpl(
+ inspectionId = inspId,
+ modelId = mdlId,
+ inspectionArguments = None,
+ durationMs = System.currentTimeMillis() - now,
+ timestamp = now,
+ errors = Seq(err),
+ warnings = Seq.empty,
+ suggestions = Seq.empty
+ )
+ )
+
+ if (intentsSamples.isEmpty)
+ onError(s"Missed intents samples for: '$mdlId'")
+ else {
+ val url = s"${Config.urlOpt.getOrElse(throw new NCE("Context word server is not configured"))}/suggestions"
+
+ val allSamplesCnt = intentsSamples.map { case (_, samples) ⇒ samples.size }.sum
+
+ val warns = mutable.ArrayBuffer.empty[String]
+
+ if (allSamplesCnt < MIN_CNT_MODEL)
+ warns +=
+ s"Model: '$mdlId' has too small intents samples count: $allSamplesCnt. " +
+ s"Potentially is can be not enough for suggestions service high quality work. " +
+ s"Try to increase their count at least to $MIN_CNT_MODEL."
+
+ else {
+ val ids =
+ intentsSamples.
+ filter { case (_, samples) ⇒ samples.size < MIN_CNT_INTENT }.
+ map { case (intentId, _) ⇒ intentId }
+
+ if (ids.nonEmpty)
+ warns +=
+ s"Models '$mdlId' has intents: [${ids.mkString(", ")}] with too small intents samples count." +
+ s"Potentially it can be not enough for suggestions service high quality work. " +
+ s"Try to increase their count at least to $MIN_CNT_INTENT."
+ }
+
+ val parser = new NCMacroParser()
+
+ macros.foreach { case (name, str) ⇒ parser.addMacro(name, str) }
+
+ // Note that we don't use system tokenizer, because ContextWordServer doesn't have this tokenizer.
+ // We just split examples words with spaces. Also we divide SEPARATORS as separated words.
+ val examples =
+ intentsSamples.
+ flatMap { case (_, samples) ⇒ samples }.
+ map(ex ⇒ SEPARATORS.foldLeft(ex)((s, ch) ⇒ s.replaceAll(s"\\$ch", s" $ch "))).
+ map(ex ⇒ {
+ val seq = ex.split(" ")
+
+ seq → seq.map(toStemWord)
+ }).
+ toMap
+
+ val elemSyns =
+ elementsSynonyms.map { case (elemId, syns) ⇒ elemId → syns.flatMap(parser.expand) }.
+ map { case (id, seq) ⇒ id → seq.map(txt ⇒ split(txt).map(p ⇒ Word(p, toStemWord(p)))) }
+
+ val allReqs =
+ elemSyns.map {
+ case (elemId, syns) ⇒
+ val normSyns: Seq[Seq[Word]] = syns.filter(_.size == 1)
+ val synsStems = normSyns.map(_.map(_.stem))
+ val synsWords = normSyns.map(_.map(_.word))
+
+ val reqs =
+ examples.flatMap { case (exampleWords, exampleStems) ⇒
+ val exampleIdxs = synsStems.flatMap(synStems ⇒ getAllSlices(exampleStems, synStems))
+
+ def mkRequestData(idx: Int, synStems: Seq[String], synStemsIdx: Int): RequestData = {
+ val fromIncl = idx
+ val toExcl = idx + synStems.length
+
+ RequestData(
+ sentence = exampleWords.zipWithIndex.flatMap {
+ case (exampleWord, i) ⇒
+ i match {
+ case x if x == fromIncl ⇒ synsWords(synStemsIdx)
+ case x if x > fromIncl && x < toExcl ⇒ Seq.empty
+ case _ ⇒ Seq(exampleWord)
+ }
+ }.mkString(" "),
+ example = exampleWords.mkString(" "),
+ elementId = elemId,
+ index = idx
+ )
+ }
+
+ (for (idx ← exampleIdxs; (synStems, i) ← synsStems.zipWithIndex)
+ yield mkRequestData(idx, synStems, i)).distinct
+ }
+
+ elemId → reqs.toSet
+ }.filter(_._2.nonEmpty)
+
+ val noExElems =
+ elementsSynonyms.
+ filter { case (elemId, syns) ⇒ syns.nonEmpty && !allReqs.contains(elemId) }.
+ map { case (elemId, _) ⇒ elemId }
+
+ if (noExElems.nonEmpty)
+ warns +=
+ "Some elements don't have synonyms in their intent samples, " +
+ s"so the service can't suggest any new synonyms for such elements: [${noExElems.mkString(", ")}]"
+
+ val allReqsCnt = allReqs.map(_._2.size).sum
+ val allSynsCnt = elemSyns.map(_._2.size).sum
+
+ logger.info(s"Data prepared. Request is going to execute on ContextWord Server " +
+ s"[examples=${examples.size}, " +
+ s"synonyms=$allSynsCnt, " +
+ s"requests=$allReqsCnt]"
+ )
+
+ if (allReqsCnt == 0)
+ onError(s"Suggestions cannot be prepared: '$mdlId'. Samples don't contain synonyms")
+ else {
+ val allSuggs = new ConcurrentHashMap[String, JList[Suggestion]]()
+ val cdl = new CountDownLatch(1)
+ val debugs = mutable.HashMap.empty[RequestData, Seq[Suggestion]]
+ val cnt = new AtomicInteger(0)
+
+ val client = HttpClients.createDefault
+ val err = new AtomicReference[Throwable]()
+
+ for ((elemId, reqs) ← allReqs; batch ← reqs.sliding(BATCH_SIZE, BATCH_SIZE).map(_.toSeq)) {
+ NCUtils.asFuture(
+ _ ⇒ {
+ val post = new HttpPost(url)
+
+ post.setHeader("Content-Type", "application/json")
+
+ post.setEntity(
+ new StringEntity(
+ GSON.toJson(
+ RestRequest(
+ sentences = batch.map(p ⇒ RestRequestSentence(p.sentence, Seq(p.index).asJava)).asJava,
+ // ContextWord server range is (0, 2), input range is (0, 1)
+ min_score = Config.suggestionsMinScore * 2,
+ // We set big limit value and in fact only minimal score is taken into account.
+ limit = MAX_LIMIT
+ )
+ ),
+ "UTF-8"
+ )
+ )
+
+ val resps: Seq[Seq[Suggestion]] =
+ try
+ client.execute(post, HANDLER)
+ finally
+ post.releaseConnection()
+
+ require(batch.size == resps.size, s"Batch: ${batch.size}, responses: ${resps.size}")
+
+ batch.zip(resps).foreach { case (req, resp) ⇒ debugs += req → resp }
+
+ val i = cnt.addAndGet(batch.size)
+
+ logger.debug(s"Executed: $i requests...")
+
+ allSuggs.
+ computeIfAbsent(elemId, (_: String) ⇒ new CopyOnWriteArrayList[Suggestion]()).
+ addAll(resps.flatten.asJava)
+
+ if (i == allReqsCnt)
+ cdl.countDown()
+ },
+ (e: Throwable) ⇒ {
+ err.compareAndSet(null, e)
+
+ cdl.countDown()
+ },
+ (_: Unit) ⇒ ()
+ )
+ }
+
+ cdl.await(Long.MaxValue, TimeUnit.MILLISECONDS)
+
+ if (err.get() != null)
+ throw new NCE("Error during work with ContextWord Server", err.get())
+
+ val allSynsStems = elemSyns.flatMap(_._2).flatten.map(_.stem).toSet
+
+ val nonEmptySuggs = allSuggs.asScala.map(p ⇒ p._1 → p._2.asScala).filter(_._2.nonEmpty)
+
+ val res = mutable.HashMap.empty[String, mutable.ArrayBuffer[SuggestionResult]]
+
+ nonEmptySuggs.
+ foreach { case (elemId, elemSuggs) ⇒
+ elemSuggs.
+ map(sugg ⇒ (sugg, toStem(sugg.word))).
+ groupBy { case (_, stem) ⇒ stem }.
+ // Drops already defined.
+ filter { case (stem, _) ⇒ !allSynsStems.contains(stem) }.
+ map { case (_, group) ⇒
+ val seq = group.map { case (sugg, _) ⇒ sugg }.sortBy(-_.score)
+
+ // Drops repeated.
+ (seq.head, seq.length)
+ }.
+ toSeq.
+ map { case (sugg, cnt) ⇒ (sugg, cnt, sugg.score * cnt / elemSuggs.size) }.
+ sortBy { case (_, _, sumFactor) ⇒ -sumFactor }.
+ zipWithIndex.
+ foreach { case ((sugg, cnt, _), _) ⇒
+ val seq =
+ res.get(elemId) match {
+ case Some(seq) ⇒ seq
+ case None ⇒
+ val buf = mutable.ArrayBuffer.empty[SuggestionResult]
+
+ res += elemId → buf
+
+ buf
+ }
+
+ seq += SuggestionResult(sugg.word, sugg.score, cnt)
+ }
+ }
+
+ logger.whenDebugEnabled({
+ logger.debug("Request information:")
+
+ var i = 1
+
+ debugs.groupBy(_._1.example).foreach { case (_, m) ⇒
+ m.toSeq.sortBy(_._1.sentence).foreach { case (req, suggs) ⇒
+ val s =
+ split(req.sentence).
+ zipWithIndex.map { case (w, i) ⇒ if (i == req.index) s"<<<$w>>>" else w }.
+ mkString(" ")
+
+ logger.debug(
+ s"$i. " +
+ s"Request=$s, " +
+ s"suggestions=[${suggs.map(_.word).mkString(", ")}], " +
+ s"element=${req.elementId}"
+ )
+
+ i = i + 1
+ }
+ }
+ })
+
+ val resJ: util.Map[String, JList[util.HashMap[String, Any]]] =
+ res.map { case (id, data) ⇒
+ id → data.map(d ⇒ {
+ val m = new util.HashMap[String, Any]()
+
+ m.put("synonym", d.synonym)
+ m.put("ctxWorldServerScore", d.ctxWorldServerScore)
+ m.put("suggestedCount", d.suggestedCount)
+
+ m
+ }).asJava
+ }.asJava
+
+ promise.success(
+ NCInspectionResultImpl(
+ inspectionId = inspId,
+ modelId = mdlId,
+ inspectionArguments = None,
+ durationMs = System.currentTimeMillis() - now,
+ timestamp = now,
+ errors = Seq.empty,
+ warnings = warns,
+ suggestions = Seq(resJ)
+ )
+ )
+ }
+ }
+ }(scala.concurrent.ExecutionContext.Implicits.global)
+
+ promise.future
+ }
+}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSynonymsSuggestions.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSynonymsSuggestions.scala
deleted file mode 100644
index f210186..0000000
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSynonymsSuggestions.scala
+++ /dev/null
@@ -1,420 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.server.inspections.inspectors
-
-import java.util
-import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
-import java.util.concurrent.{ConcurrentHashMap, CopyOnWriteArrayList, CountDownLatch, TimeUnit}
-import java.util.{List ⇒ JList}
-
-import com.google.gson.Gson
-import com.google.gson.reflect.TypeToken
-import io.opencensus.trace.Span
-import org.apache.http.HttpResponse
-import org.apache.http.client.ResponseHandler
-import org.apache.http.client.methods.HttpPost
-import org.apache.http.entity.StringEntity
-import org.apache.http.impl.client.HttpClients
-import org.apache.http.util.EntityUtils
-import org.apache.nlpcraft.common.config.NCConfigurable
-import org.apache.nlpcraft.common.inspections.{NCInspection, NCInspector}
-import org.apache.nlpcraft.common.makro.NCMacroParser
-import org.apache.nlpcraft.common.nlp.core.NCNlpPorterStemmer
-import org.apache.nlpcraft.common.util.NCUtils
-import org.apache.nlpcraft.common.{NCE, NCService}
-
-import scala.collection.JavaConverters._
-import scala.collection.{Seq, mutable}
-import scala.concurrent.ExecutionContext.Implicits.global
-
-// TODO:
-object NCInspectorSynonymsSuggestions extends NCService with NCInspector {
- // For context word server requests.
- private final val MAX_LIMIT: Int = 10000
- private final val BATCH_SIZE = 20
-
- // For warnings.
- private final val MIN_CNT_INTENT = 5
- private final val MIN_CNT_MODEL = 20
-
- private object Config extends NCConfigurable {
- val urlOpt: Option[String] = getStringOpt("nlpcraft.server.ctxword.url")
- val suggestionsMinScore: Int = getInt("nlpcraft.server.ctxword.suggestions.minScore")
-
- @throws[NCE]
- def check(): Unit =
- if (suggestionsMinScore < 0 || suggestionsMinScore > 1)
- throw new NCE("Invalid 'nlpcraft.server.ctxword.suggestions.minScore' parameter value. It should be double value between 0 and 1, inclusive")
- }
-
- Config.check()
-
- case class Suggestion(word: String, score: Double)
- case class RequestData(sentence: String, example: String, elementId: String, index: Int)
- case class RestRequestSentence(text: String, indexes: JList[Int])
- case class RestRequest(sentences: JList[RestRequestSentence], limit: Int, min_score: Double)
- case class Word(word: String, stem: String) {
- require(!word.contains(" "), s"Word cannot contains spaces: $word")
- require(
- word.forall(ch ⇒
- ch.isLetterOrDigit ||
- ch == '\'' ||
- SEPARATORS.contains(ch)
- ),
- s"Unsupported symbols: $word"
- )
- }
-
- case class SuggestionResult(
- synonym: String,
- ctxWorldServerScore: Double,
- suggestedCount: Int
- )
-
- private final val GSON = new Gson
- private final val TYPE_RESP = new TypeToken[JList[JList[Suggestion]]]() {}.getType
- private final val SEPARATORS = Seq('?', ',', '.', '-', '!')
-
- private final val HANDLER: ResponseHandler[Seq[Seq[Suggestion]]] =
- (resp: HttpResponse) ⇒ {
- val code = resp.getStatusLine.getStatusCode
- val e = resp.getEntity
-
- val js = if (e != null) EntityUtils.toString(e) else null
-
- if (js == null)
- throw new RuntimeException(s"Unexpected empty response [code=$code]")
-
- code match {
- case 200 ⇒
- val data: JList[JList[Suggestion]] = GSON.fromJson(js, TYPE_RESP)
-
- data.asScala.map(p ⇒ if (p.isEmpty) Seq.empty else p.asScala.tail)
-
- case 400 ⇒ throw new RuntimeException(js)
- case _ ⇒ throw new RuntimeException(s"Unexpected response [code=$code, response=$js]")
- }
- }
-
- private def split(s: String): Seq[String] = s.split(" ").toSeq.map(_.trim).filter(_.nonEmpty)
- private def toStem(s: String): String = split(s).map(NCNlpPorterStemmer.stem).mkString(" ")
- private def toStemWord(s: String): String = NCNlpPorterStemmer.stem(s)
-
- /**
- *
- * @param seq1
- * @param seq2
- */
- private def getAllSlices(seq1: Seq[String], seq2: Seq[String]): Seq[Int] = {
- val seq = mutable.Buffer.empty[Int]
-
- var i = seq1.indexOfSlice(seq2)
-
- while (i >= 0) {
- seq += i
-
- i = seq1.indexOfSlice(seq2, i + 1)
- }
-
- seq
- }
-
- override def inspect(mdlId: String, prevLayerInspection: Option[NCInspection], parent: Span = null): NCInspection =
- startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
- val m: util.Map[String, AnyRef] =
- prevLayerInspection.
- getOrElse(throw new NCE(s"Missed previous inspection data for model: $mdlId")).
- data.asInstanceOf[util.Map[String, AnyRef]]
-
- val macrosJ = m.get("macros").asInstanceOf[util.Map[String, String]]
- val elementsSynonymsJ = m.get("elementsSynonyms").asInstanceOf[util.Map[String, util.List[String]]]
- val intentsSamplesJ = m.get("intentsSamples").asInstanceOf[util.Map[String, util.List[String]]]
-
- require(macrosJ != null)
- require(elementsSynonymsJ != null)
- require(intentsSamplesJ != null)
-
- val macros = macrosJ.asScala
- val elementsSynonyms = elementsSynonymsJ.asScala.map(p ⇒ p._1 → p._2.asScala)
- val intentsSamples = intentsSamplesJ.asScala.map(p ⇒ p._1 → p._2.asScala)
-
- if (intentsSamples.isEmpty)
- NCInspection(
- errors = Some(Seq(s"Missed intents samples for: '$mdlId'")),
- warnings = None,
- suggestions = None
- )
- else {
- val url = s"${Config.urlOpt.getOrElse(throw new NCE("Context word server is not configured"))}/suggestions"
-
- val allSamplesCnt = intentsSamples.map { case (_, samples) ⇒ samples.size }.sum
-
- val warns = mutable.ArrayBuffer.empty[String]
-
- if (allSamplesCnt < MIN_CNT_MODEL)
- warns +=
- s"Model: '$mdlId' has too small intents samples count: $allSamplesCnt. " +
- s"Potentially is can be not enough for suggestions service high quality work. " +
- s"Try to increase their count at least to $MIN_CNT_MODEL."
-
- else {
- val ids =
- intentsSamples.
- filter { case (_, samples) ⇒ samples.size < MIN_CNT_INTENT }.
- map { case (intentId, _) ⇒ intentId }
-
- if (ids.nonEmpty)
- warns +=
- s"Models '$mdlId' has intents: [${ids.mkString(", ")}] with too small intents samples count." +
- s"Potentially it can be not enough for suggestions service high quality work. " +
- s"Try to increase their count at least to $MIN_CNT_INTENT."
- }
-
- val parser = new NCMacroParser()
-
- macros.foreach { case (name, str) ⇒ parser.addMacro(name, str) }
-
- // Note that we don't use system tokenizer, because ContextWordServer doesn't have this tokenizer.
- // We just split examples words with spaces. Also we divide SEPARATORS as separated words.
- val examples =
- intentsSamples.
- flatMap { case (_, samples) ⇒ samples }.
- map(ex ⇒ SEPARATORS.foldLeft(ex)((s, ch) ⇒ s.replaceAll(s"\\$ch", s" $ch "))).
- map(ex ⇒ {
- val seq = ex.split(" ")
-
- seq → seq.map(toStemWord)
- }).
- toMap
-
- val elemSyns =
- elementsSynonyms.map { case (elemId, syns) ⇒ elemId → syns.flatMap(parser.expand) }.
- map { case (id, seq) ⇒ id → seq.map(txt ⇒ split(txt).map(p ⇒ Word(p, toStemWord(p)))) }
-
- val allReqs =
- elemSyns.map {
- case (elemId, syns) ⇒
- val normSyns: Seq[Seq[Word]] = syns.filter(_.size == 1)
- val synsStems = normSyns.map(_.map(_.stem))
- val synsWords = normSyns.map(_.map(_.word))
-
- val reqs =
- examples.flatMap { case (exampleWords, exampleStems) ⇒
- val exampleIdxs = synsStems.flatMap(synStems ⇒ getAllSlices(exampleStems, synStems))
-
- def mkRequestData(idx: Int, synStems: Seq[String], synStemsIdx: Int): RequestData = {
- val fromIncl = idx
- val toExcl = idx + synStems.length
-
- RequestData(
- sentence = exampleWords.zipWithIndex.flatMap {
- case (exampleWord, i) ⇒
- i match {
- case x if x == fromIncl ⇒ synsWords(synStemsIdx)
- case x if x > fromIncl && x < toExcl ⇒ Seq.empty
- case _ ⇒ Seq(exampleWord)
- }
- }.mkString(" "),
- example = exampleWords.mkString(" "),
- elementId = elemId,
- index = idx
- )
- }
-
- (for (idx ← exampleIdxs; (synStems, i) ← synsStems.zipWithIndex)
- yield mkRequestData(idx, synStems, i)).distinct
- }
-
- elemId → reqs.toSet
- }.filter(_._2.nonEmpty)
-
- val noExElems =
- elementsSynonyms.
- filter { case (elemId, syns) ⇒ syns.nonEmpty && !allReqs.contains(elemId) }.
- map { case (elemId, _) ⇒ elemId }
-
- if (noExElems.nonEmpty)
- warns +=
- "Some elements don't have synonyms in their intent samples, " +
- s"so the service can't suggest any new synonyms for such elements: [${noExElems.mkString(", ")}]"
-
- val allReqsCnt = allReqs.map(_._2.size).sum
- val allSynsCnt = elemSyns.map(_._2.size).sum
-
- logger.info(s"Data prepared. Request is going to execute on ContextWord Server " +
- s"[examples=${examples.size}, " +
- s"synonyms=$allSynsCnt, " +
- s"requests=$allReqsCnt]"
- )
-
- if (allReqsCnt == 0)
- NCInspection(
- errors = Some(Seq(s"Suggestions cannot be prepared: '$mdlId'. Samples don't contain synonyms")),
- warnings = None,
- suggestions = None
- )
-
- else {
- val allSuggs = new ConcurrentHashMap[String, JList[Suggestion]]()
- val cdl = new CountDownLatch(1)
- val debugs = mutable.HashMap.empty[RequestData, Seq[Suggestion]]
- val cnt = new AtomicInteger(0)
-
- val client = HttpClients.createDefault
- val err = new AtomicReference[Throwable]()
-
- for ((elemId, reqs) ← allReqs; batch ← reqs.sliding(BATCH_SIZE, BATCH_SIZE).map(_.toSeq)) {
- NCUtils.asFuture(
- _ ⇒ {
- val post = new HttpPost(url)
-
- post.setHeader("Content-Type", "application/json")
-
- post.setEntity(
- new StringEntity(
- GSON.toJson(
- RestRequest(
- sentences = batch.map(p ⇒ RestRequestSentence(p.sentence, Seq(p.index).asJava)).asJava,
- // ContextWord server range is (0, 2), input range is (0, 1)
- min_score = Config.suggestionsMinScore * 2,
- // We set big limit value and in fact only minimal score is taken into account.
- limit = MAX_LIMIT
- )
- ),
- "UTF-8"
- )
- )
-
- val resps: Seq[Seq[Suggestion]] =
- try
- client.execute(post, HANDLER)
- finally
- post.releaseConnection()
-
- require(batch.size == resps.size, s"Batch: ${batch.size}, responses: ${resps.size}")
-
- batch.zip(resps).foreach { case (req, resp) ⇒ debugs += req → resp }
-
- val i = cnt.addAndGet(batch.size)
-
- logger.debug(s"Executed: $i requests...")
-
- allSuggs.
- computeIfAbsent(elemId, (_: String) ⇒ new CopyOnWriteArrayList[Suggestion]()).
- addAll(resps.flatten.asJava)
-
- if (i == allReqsCnt)
- cdl.countDown()
- },
- (e: Throwable) ⇒ {
- err.compareAndSet(null, e)
-
- cdl.countDown()
- },
- (_: Unit) ⇒ ()
- )
- }
-
- cdl.await(Long.MaxValue, TimeUnit.MILLISECONDS)
-
- if (err.get() != null)
- throw new NCE("Error during work with ContextWord Server", err.get())
-
- val allSynsStems = elemSyns.flatMap(_._2).flatten.map(_.stem).toSet
-
- val nonEmptySuggs = allSuggs.asScala.map(p ⇒ p._1 → p._2.asScala).filter(_._2.nonEmpty)
-
- val res = mutable.HashMap.empty[String, mutable.ArrayBuffer[SuggestionResult]]
-
- nonEmptySuggs.
- foreach { case (elemId, elemSuggs) ⇒
- elemSuggs.
- map(sugg ⇒ (sugg, toStem(sugg.word))).
- groupBy { case (_, stem) ⇒ stem }.
- // Drops already defined.
- filter { case (stem, _) ⇒ !allSynsStems.contains(stem) }.
- map { case (_, group) ⇒
- val seq = group.map { case (sugg, _) ⇒ sugg }.sortBy(-_.score)
-
- // Drops repeated.
- (seq.head, seq.length)
- }.
- toSeq.
- map { case (sugg, cnt) ⇒ (sugg, cnt, sugg.score * cnt / elemSuggs.size) }.
- sortBy { case (_, _, sumFactor) ⇒ -sumFactor }.
- zipWithIndex.
- foreach { case ((sugg, cnt, _), _) ⇒
- val seq =
- res.get(elemId) match {
- case Some(seq) ⇒ seq
- case None ⇒
- val buf = mutable.ArrayBuffer.empty[SuggestionResult]
-
- res += elemId → buf
-
- buf
- }
-
- seq += SuggestionResult(sugg.word, sugg.score, cnt)
- }
- }
-
- logger.whenDebugEnabled({
- logger.debug("Request information:")
-
- var i = 1
-
- debugs.groupBy(_._1.example).foreach { case (_, m) ⇒
- m.toSeq.sortBy(_._1.sentence).foreach { case (req, suggs) ⇒
- val s =
- split(req.sentence).
- zipWithIndex.map { case (w, i) ⇒ if (i == req.index) s"<<<$w>>>" else w }.
- mkString(" ")
-
- logger.debug(
- s"$i. " +
- s"Request=$s, " +
- s"suggestions=[${suggs.map(_.word).mkString(", ")}], " +
- s"element=${req.elementId}"
- )
-
- i = i + 1
- }
- }
- })
-
- val resJ: util.Map[String, JList[util.HashMap[String, Any]]] =
- res.map { case (id, data) ⇒
- id → data.map(d ⇒ {
- val m = new util.HashMap[String, Any]()
-
- m.put("synonym", d.synonym)
- m.put("ctxWorldServerScore", d.ctxWorldServerScore)
- m.put("suggestedCount", d.suggestedCount)
-
- m
- }).asJava
- }.asJava
-
- NCInspection(
- errors = None, warnings = if (warns.isEmpty) None else Some(warns), suggestions = Some(Seq(resJ))
- )
- }
- }
- }
-}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections2/NCInspectionManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections2/NCInspectionManager.scala
deleted file mode 100644
index d81bfe1..0000000
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections2/NCInspectionManager.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.server.inspections2
-
-import org.apache.nlpcraft.common.NCService
-import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.inspections2.{NCInspection, NCInspectionResult}
-
-/**
- *
- */
-object NCInspectionManager extends NCService {
- override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ ⇒
- // TODO
-
- super.start(parent)
- }
-
- override def stop(parent: Span): Unit = startScopedSpan("stop", parent) { _ ⇒
- super.stop()
-
- // TODO
- }
-
- /**
- *
- * @param mdlId Model ID.
- * @param inspId Inspection ID.
- * @param inspArgs Inspection arguments as JSON string.
- * @param parent Optional parent trace span.
- * @return
- */
- def inspect(mdlId: String, inspId: String, inspArgs: String, parent: Span = null): NCInspectionResult = ???
-
- /**
- * Gets all supported server and probe inspections.
- *
- * @param parent
- * @return
- */
- def getInspections(parent: Span = null): List[NCInspection] = ???
-}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index 0f99b5b..3a957d7 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -31,8 +31,8 @@ import io.opencensus.trace.Span
import org.apache.nlpcraft.common.ascii.NCAsciiTable
import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.common.crypto.NCCipher
-import org.apache.nlpcraft.common.inspections.{NCInspection, NCInspectionType}
-import org.apache.nlpcraft.common.inspections.NCInspectionType._
+import org.apache.nlpcraft.common.inspections.NCInspectionResult
+import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
import org.apache.nlpcraft.common.nlp.NCNlpSentence
import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.socket.NCSocket
@@ -57,7 +57,8 @@ import scala.util.{Failure, Success}
*/
object NCProbeManager extends NCService {
private final val GSON = new Gson()
- private final val TYPE_INSPECTION_RESP = new TypeToken[util.Map[String, NCInspection]]() {}.getType
+ private final val TYPE_INSPECTION_RESP = new TypeToken[NCInspectionResultImpl]() {}.getType
+ private final val TYPE_MODEL_INFO_RESP = new TypeToken[util.HashMap[String, AnyRef]]() {}.getType
// Type safe and eager configuration container.
private[probe] object Config extends NCConfigurable {
@@ -146,7 +147,8 @@ object NCProbeManager extends NCService {
@volatile private var pool: ExecutorService = _
@volatile private var isStopping: AtomicBoolean = _
- @volatile private var inspections: ConcurrentHashMap[String, Promise[Map[NCInspectionType, NCInspection]]] = _
+ @volatile private var probeInspecs: ConcurrentHashMap[String, Promise[NCInspectionResult]] = _
+ @volatile private var modelsInfo: ConcurrentHashMap[String, Promise[Map[String, AnyRef]]] = _
/**
*
@@ -167,7 +169,8 @@ object NCProbeManager extends NCService {
isStopping = new AtomicBoolean(false)
- inspections = new ConcurrentHashMap[String, Promise[Map[NCInspectionType, NCInspection]]]()
+ probeInspecs = new ConcurrentHashMap[String, Promise[NCInspectionResult]]()
+ modelsInfo = new ConcurrentHashMap[String, Promise[Map[String, AnyRef]]]()
pool = Executors.newFixedThreadPool(Config.poolSize)
@@ -207,7 +210,8 @@ object NCProbeManager extends NCService {
U.stopThread(dnSrv)
U.stopThread(upSrv)
- inspections = null
+ probeInspecs = null
+ modelsInfo = null
super.stop()
}
@@ -682,20 +686,26 @@ object NCProbeManager extends NCService {
logger.error(s"Received message from unknown probe (ignoring): $probeKey]")
else {
val typ = probeMsg.getType
+
+ def processPromise[T](promises: ConcurrentHashMap[String, Promise[T]], typ: java.lang.reflect.Type): Unit = {
+ val promise = promises.remove(probeMsg.data[String]("reqGuid"))
+
+ if (promise != null) {
+ val r: T = GSON.fromJson(probeMsg.data[String]("resp"), typ)
+
+ println("+!r=" + r)
+
+ promise.success(r)
+
+ println("+!r!!!=" + r)
+ }
+ }
typ match {
case "P2S_PING" ⇒ ()
- case "P2S_MODEL_INSPECTION" ⇒
- val promise = inspections.remove(probeMsg.data[String]("reqGuid"))
-
- if (promise != null) {
- val respJs: util.Map[String, NCInspection] =
- GSON.fromJson(probeMsg.data[String]("resp"), TYPE_INSPECTION_RESP)
-
- promise.success(respJs.asScala.map(p ⇒ NCInspectionType.withName(p._1) → p._2))
- }
-
+ case "P2S_PROBE_INSPECTION" ⇒ processPromise(probeInspecs, TYPE_INSPECTION_RESP)
+ case "P2S_MODEL_INFO" ⇒ processPromise(modelsInfo, TYPE_MODEL_INFO_RESP)
case "P2S_ASK_RESULT" ⇒
val srvReqId = probeMsg.data[String]("srvReqId")
@@ -969,24 +979,70 @@ object NCProbeManager extends NCService {
}
}
- def inspect(mdlId: String, types: Seq[NCInspectionType], parent: Span = null): Future[Map[NCInspectionType, NCInspection]] =
- startScopedSpan("inspect", parent, "modelId" → mdlId, "types" → types.map(_.toString)) { _ ⇒
- getProbeForModelId(mdlId) match {
- case Some(probe) ⇒
- val msg = NCProbeMessage(
- "S2P_MODEL_INSPECTION",
- "mdlId" → mdlId,
- "types" → new java.util.ArrayList(types.map(_.toString).asJava)
- )
+ /**
+ *
+ * @param mdlId
+ * @param inspId
+ * @param args
+ * @param parent
+ * @return
+ */
+ def getProbeInspection(mdlId: String, inspId: String, args: String, parent: Span = null): Future[NCInspectionResult] =
+ startScopedSpan("inspect", parent, "modelId" → mdlId, "inspId" → inspId) { _ ⇒
+ probePromise(
+ parent,
+ mdlId,
+ probeInspecs,
+ "S2P_PROBE_INSPECTION",
+ "mdlId" → mdlId,
+ "inspId" → inspId,
+ "args" → GSON.toJson(args)
+ )
+ }
+
+ /**
+ *
+ * @param mdlId
+ * @param parent
+ * @return
+ */
+ def getModelInfo(mdlId: String, parent: Span = null): Future[Map[String, AnyRef]] =
+ startScopedSpan("getModelInfo", parent, "modelId" → mdlId) { _ ⇒
+ probePromise(
+ parent,
+ mdlId,
+ modelsInfo,
+ "S2P_MODEL_INFO",
+ "mdlId" → mdlId
+ )
+ }
+
+ /**
+ *
+ * @param parent
+ * @param mdlId
+ * @param promises
+ * @param msgId
+ * @param mdgParams
+ */
+ private def probePromise[T](
+ parent: Span,
+ mdlId: String,
+ promises: ConcurrentHashMap[String, Promise[T]],
+ msgId: String,
+ mdgParams: (String, Serializable)*
+ ): Future[T] =
+ getProbeForModelId(mdlId) match {
+ case Some(probe) ⇒
+ val msg = NCProbeMessage(msgId, mdgParams:_*)
- val promise = Promise[Map[NCInspectionType, NCInspection]]()
+ val promise = Promise[T]()
- inspections.put(msg.getGuid, promise)
+ promises.put(msg.getGuid, promise)
- sendToProbe(probe.probeKey, msg, parent)
+ sendToProbe(probe.probeKey, msg, parent)
- promise.future
- case None ⇒ throw new NCE(s"Probe not found for model: '$mdlId''")
- }
+ promise.future
+ case None ⇒ throw new NCE(s"Probe not found for model: '$mdlId''")
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
index 96bdaa4..88be3f9 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
@@ -34,19 +34,18 @@ import org.apache.nlpcraft.server.apicodes.NCApiStatusCode.{API_OK, _}
import org.apache.nlpcraft.server.company.NCCompanyManager
import org.apache.nlpcraft.server.feedback.NCFeedbackManager
import org.apache.nlpcraft.server.mdo.{NCQueryStateMdo, NCUserMdo}
-import org.apache.nlpcraft.server.inspections.NCServerInspectorManager
import org.apache.nlpcraft.server.opencensus.NCOpenCensusServerStats
import org.apache.nlpcraft.server.probe.NCProbeManager
import org.apache.nlpcraft.server.query.NCQueryManager
import org.apache.nlpcraft.server.user.NCUserManager
import spray.json.DefaultJsonProtocol._
-import spray.json.{JsValue, RootJsonFormat}
+import spray.json.{DeserializationException, JsObject, JsValue, RootJsonFormat}
import scala.collection.JavaConverters._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
import akka.http.scaladsl.coding.Coders
-import org.apache.nlpcraft.common.inspections.NCInspectionType
+import org.apache.nlpcraft.server.inspections.NCInspectionManager
/**
* REST API default implementation.
@@ -323,6 +322,30 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
/**
*
+ * @param js
+ * @param fn
+ */
+ @throws[InvalidField]
+ protected def convert[T](js: JsObject, fn: String, extractor: JsValue ⇒ T): T =
+ try
+ extractor(js.fields(fn))
+ catch {
+ case _: Exception ⇒ throw InvalidField(fn)
+ }
+
+ @throws[InvalidField]
+ protected def convertOpt[T](js: JsObject, fn: String, extractor: JsValue ⇒ T): Option[T] =
+ try
+ js.fields.get(fn) match {
+ case Some(v) ⇒ Some(extractor(v))
+ case None ⇒ None
+ }
+ catch {
+ case _: Exception ⇒ throw InvalidField(fn)
+ }
+
+ /**
+ *
* @return
*/
protected def signin$(): Route = {
@@ -411,19 +434,13 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
protected def ask$Sync(reqJs: JsValue, usrAgent: Option[String], rmtAddr: RemoteAddress): Future[String] = {
val obj = reqJs.asJsObject()
- def getOpt[T](name: String, convert: JsValue ⇒ T): Option[T] =
- obj.fields.get(name) match {
- case Some(v) ⇒ Some(convert(v))
- case None ⇒ None
- }
-
- val acsTok = obj.fields("acsTok").convertTo[String]
- val txt = obj.fields("txt").convertTo[String]
- val mdlId = obj.fields("mdlId").convertTo[String]
- val data = getOpt("data", (js: JsValue) ⇒ js.compactPrint)
- val enableLog = getOpt("enableLog", (js: JsValue) ⇒ js.convertTo[Boolean])
- val usrExtIdOpt = getOpt("usrExtId", (js: JsValue) ⇒ js.convertTo[String])
- val usrIdOpt = getOpt("usrId", (js: JsValue) ⇒ js.convertTo[Long])
+ val acsTok: String = convert(obj, "acsTok", (js: JsValue) ⇒ js.convertTo[String])
+ val txt: String = convert(obj, "txt", (js: JsValue) ⇒ js.convertTo[String])
+ val mdlId: String = convert(obj, "mdlId", (js: JsValue) ⇒ js.convertTo[String])
+ val data: Option[String] = convertOpt(obj, "data", (js: JsValue) ⇒ js.compactPrint)
+ val enableLog: Option[Boolean] = convertOpt(obj, "enableLog", (js: JsValue) ⇒ js.convertTo[Boolean])
+ val usrExtIdOpt: Option[String] = convertOpt(obj, "usrExtId", (js: JsValue) ⇒ js.convertTo[String])
+ val usrIdOpt: Option[Long] = convertOpt(obj, "usrId", (js: JsValue) ⇒ js.convertTo[Long])
startScopedSpan(
"ask$Sync",
@@ -629,41 +646,24 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
protected def inspect$(reqJs: JsValue): Future[String] = {
val obj = reqJs.asJsObject()
- val acsTok = obj.fields("acsTok").convertTo[String]
- val mdlId = obj.fields("mdlId").convertTo[String]
- val types = obj.fields("types").convertTo[Seq[String]]
+ val acsTok: String = convert(obj, "acsTok", (js: JsValue) ⇒ js.convertTo[String])
+ val mdlId: String = convert(obj, "mdlId", (js: JsValue) ⇒ js.convertTo[String])
+ val inspId: String = convert(obj, "inspId", (js: JsValue) ⇒ js.convertTo[String])
+ val args: Option[String] = convertOpt(obj, "args", v ⇒ v.compactPrint)
startScopedSpan("modelEnhance$", "mdlId" → mdlId, "acsTok" → acsTok) { span ⇒
checkLength("acsTok", acsTok, 256)
checkLength("mdlId", mdlId, 32)
-
- val typesVals =
- if (types.size == 1 && types.head.toLowerCase == "all")
- NCInspectionType.values.toSeq
- else
- types.map(typ ⇒
- try
- NCInspectionType.withName(typ.toUpperCase)
- catch {
- case _: Exception ⇒ throw InvalidField("types")
- }
- )
+ checkLength("inspId", inspId, 32)
val admin = authenticateAsAdmin(acsTok)
if (!NCProbeManager.getAllProbes(admin.companyId, span).exists(_.models.exists(_.id == mdlId)))
throw new NCE(s"Probe not found for model: $mdlId")
- NCServerInspectorManager.
- inspect(mdlId, typesVals, span).collect {
- // We have to use GSON (not spray) here to serialize `result` field.
- case res ⇒
- GSON.toJson(
- Map(
- "status" → API_OK.toString,
- "result" → res.asJava
- ).asJava
- )
+ NCInspectionManager.inspect(mdlId, inspId, args.orNull, span).collect {
+ // We have to use GSON (not spray) here to serialize `result` field.
+ case res ⇒ GSON.toJson(Map("status" → API_OK.toString, "result" → res).asJava)
}
}
}