You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by ar...@apache.org on 2020/09/07 06:34:36 UTC
[incubator-nlpcraft] 03/03: WIP.
This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a commit to branch NLPCRAFT-41
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit fd7057612ccf336ff14b80a3a6f971a5a2089643
Author: Aaron Radzinski <ar...@datalingvo.com>
AuthorDate: Sun Sep 6 23:27:54 2020 -0700
WIP.
---
.../nlpcraft/common/inspections/NCInspection.scala | 9 +--
.../common/inspections/NCInspectionParameter.scala | 6 --
.../common/inspections/NCInspectionService.scala | 59 ++++++++++++++-
.../common/inspections/impl/NCInspectionImpl.scala | 5 +-
.../impl/NCInspectionParameterImpl.scala | 1 -
.../inspections/impl/NCInspectionResultImpl.scala | 8 +--
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 2 +-
.../mgrs/inspections/NCInspectionManager.scala | 35 +++++----
.../mgrs/inspections/inspectors/NCInspector.scala | 63 ----------------
.../inspectors/NCInspectorIntents.scala | 83 ----------------------
.../inspections/inspectors/NCInspectorMacros.scala | 59 ---------------
.../inspectors/NCInspectorSynonyms.scala | 79 --------------------
.../inspectors/NCIntentsInspection.scala | 70 ++++++++++++++++++
.../inspectors/NCMacrosInspection.scala | 47 ++++++++++++
.../inspectors/NCSynonymsInspection.scala | 72 +++++++++++++++++++
.../server/inspection/NCInspectionManager.scala | 17 ++---
.../inspection/impl/NCSuggestionInspection.scala | 13 +++-
.../nlpcraft/server/probe/NCProbeManager.scala | 8 +--
.../nlpcraft/server/rest/NCBasicRestApi.scala | 16 ++---
openapi/nlpcraft_swagger.yml | 4 +-
20 files changed, 301 insertions(+), 355 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
index 61c1965..7a0a855 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspection.scala
@@ -22,14 +22,7 @@ package org.apache.nlpcraft.common.inspections
*/
trait NCInspection {
/**
- * Globally unique, internal inspection ID.
- *
- * @return
- */
- def id(): String
-
- /**
- * User-visible name of the inspection.
+ * Globally unique, user-visible name of the inspection.
*
* @return
*/
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionParameter.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionParameter.scala
index 27b43c1..25378e7 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionParameter.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionParameter.scala
@@ -25,12 +25,6 @@ trait NCInspectionParameter {
*
* @return
*/
- def id(): String
-
- /**
- *
- * @return
- */
def name(): String
/**
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
index 63efeb9..aefd7aa 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/NCInspectionService.scala
@@ -4,8 +4,12 @@ import java.util.concurrent.{ExecutorService, Executors}
import io.opencensus.trace.Span
import org.apache.nlpcraft.common.NCService
+import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
import org.apache.nlpcraft.common.util.NCUtils
+import org.apache.nlpcraft.model.NCModel
+import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
+import scala.collection.mutable
import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future}
/**
@@ -21,12 +25,57 @@ trait NCInspectionService extends NCService {
/**
*
* @param mdlId
- * @param inspId
+ * @param inspName
* @param args
* @param parent
* @return
*/
- def inspect(mdlId: String, inspId: String, args: Option[String], parent: Span = null): Future[NCInspectionResult]
+ def inspect(mdlId: String, inspName: String, args: Option[String], parent: Span = null): Future[NCInspectionResult] =
+ startScopedSpan(
+ "inspect",
+ parent,
+ "modelId" → mdlId,
+ "inspName" -> inspName) { _ ⇒
+ Future {
+ val now = System.currentTimeMillis()
+
+ val errs = mutable.Buffer.empty[String]
+ val warns = mutable.Buffer.empty[String]
+ val suggs = mutable.Buffer.empty[String]
+
+ NCModelManager.getModel(mdlId) match {
+ case Some(x) => bodyOnProbe(x.model, args, suggs, warns, errs)
+ case None => errs += s"Model not found: $mdlId"
+ }
+
+ NCInspectionResultImpl(
+ inspectionId = inspName,
+ modelId = mdlId,
+ durationMs = System.currentTimeMillis() - now,
+ timestamp = now,
+ warnings = warns,
+ suggestions = suggs,
+ errors = errs
+ )
+ }(getExecutor)
+ }
+
+ /**
+ * Convenient adapter for the probe-side inspection implementation.
+ *
+ * @param mdl
+ * @param args
+ * @param suggs Mutable collector for suggestions.
+ * @param warns Mutable collector for warnings.
+ * @param errs Mutable collector for errors.
+ */
+ protected def bodyOnProbe(
+ mdl: NCModel,
+ args: Option[String],
+ suggs: mutable.Buffer[String],
+ warns: mutable.Buffer[String],
+ errs: mutable.Buffer[String]
+ ) = {}
/**
*
@@ -34,6 +83,12 @@ trait NCInspectionService extends NCService {
*/
def getExecutor = executor
+ /**
+ *
+ * @return
+ */
+ def getName: String
+
override def start(parent: Span): NCService =
startScopedSpan("start", parent) { _ ⇒
pool = Executors.newCachedThreadPool()
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionImpl.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionImpl.scala
index e5ee15b..11a8b24 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionImpl.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionImpl.scala
@@ -22,7 +22,6 @@ import org.apache.nlpcraft.common.inspections.{NCInspection, NCInspectionParamet
import scala.collection.JavaConverters.seqAsJavaListConverter
case class NCInspectionImpl(
- id: String,
name: String,
synopsis: String,
parameters: java.util.List[NCInspectionParameter],
@@ -32,15 +31,13 @@ case class NCInspectionImpl(
object NCInspectionImpl {
def apply(
- id: String,
name: String,
synopsis: String,
- parameters: Seq[NCInspectionParameter],
+ parameters: Seq[NCInspectionParameter] = Seq.empty,
description: String,
isServerSide: Boolean,
) : NCInspectionImpl =
NCInspectionImpl(
- id,
name,
synopsis,
parameters = parameters.asJava,
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionParameterImpl.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionParameterImpl.scala
index cf8f5f2..9c02d73 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionParameterImpl.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionParameterImpl.scala
@@ -20,7 +20,6 @@ package org.apache.nlpcraft.common.inspections.impl
import org.apache.nlpcraft.common.inspections.NCInspectionParameter
case class NCInspectionParameterImpl(
- id: String,
name: String,
value: String,
valueType: String,
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionResultImpl.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionResultImpl.scala
index 5be8bb7..ac70546 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionResultImpl.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/inspections/impl/NCInspectionResultImpl.scala
@@ -36,12 +36,12 @@ object NCInspectionResultImpl {
def apply(
inspectionId: String,
modelId: String,
- inspectionArguments: Option[String],
+ inspectionArguments: Option[String] = None,
durationMs: Long,
timestamp: Long,
- errors: Seq[String],
- warnings: Seq[String],
- suggestions: Seq[AnyRef]
+ errors: Seq[String] = Seq.empty,
+ warnings: Seq[String] = Seq.empty,
+ suggestions: Seq[AnyRef] = Seq.empty
): NCInspectionResultImpl =
new NCInspectionResultImpl(
inspectionId,
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
index f2c333e..7e80549 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
@@ -110,7 +110,7 @@ object NCCommandManager extends NCService {
case "S2P_PROBE_INSPECTION" ⇒
NCInspectionManager.inspect(
mdlId = msg.data[String]("mdlId"),
- inspId = msg.data[String]("inspId"),
+ inspName = msg.data[String]("inspName"),
args = msg.dataOpt[String]("args"),
span
).onComplete {
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
index 3bc5ad3..93f39d7 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
@@ -21,23 +21,23 @@ import io.opencensus.trace.Span
import org.apache.nlpcraft.common.inspections.NCInspectionResult
import org.apache.nlpcraft.common.{NCE, NCService}
import org.apache.nlpcraft.model.opencensus.stats.NCOpenCensusModelStats
-import org.apache.nlpcraft.probe.mgrs.inspections.inspectors.{NCInspectorIntents, NCInspectorMacros, NCInspectorSynonyms}
+import org.apache.nlpcraft.probe.mgrs.inspections.inspectors.{NCIntentsInspection, NCMacrosInspection, NCSynonymsInspection}
import scala.concurrent.Future
/**
- * TODO:
+ * Probe-side inspection manager.
*/
object NCInspectionManager extends NCService with NCOpenCensusModelStats {
- private final val INSPECTORS =
- Map(
- "macros" → NCInspectorMacros,
- "synonyms" → NCInspectorSynonyms,
- "intents" → NCInspectorIntents
+ private final val PROBE_INSPECTIONS =
+ Seq(
+ NCMacrosInspection,
+ NCSynonymsInspection,
+ NCIntentsInspection
)
override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ ⇒
- INSPECTORS.values.foreach(_.start(parent))
+ PROBE_INSPECTIONS.foreach(_.start(parent))
super.start(parent)
}
@@ -45,21 +45,26 @@ object NCInspectionManager extends NCService with NCOpenCensusModelStats {
override def stop(parent: Span): Unit = startScopedSpan("stop", parent) { _ ⇒
super.stop()
- INSPECTORS.values.foreach(_.stop(parent))
+ PROBE_INSPECTIONS.foreach(_.stop(parent))
}
/**
*
* @param mdlId Model ID.
- * @param inspId Inspection ID.
+ * @param inspName Inspection ID.
* @param args Inspection arguments.
* @param parent Optional parent trace span.
*/
- def inspect(mdlId: String, inspId: String, args: Option[String], parent: Span = null): Future[NCInspectionResult] =
- startScopedSpan("inspect", parent, "modelId" → mdlId, "inspectionId" → inspId) { _ ⇒
- INSPECTORS.get(inspId) match {
- case Some(inspector) ⇒ inspector.inspect(mdlId, inspId, args)
- case None ⇒ throw new NCE(s"Unsupported inspection: $inspId")
+ def inspect(mdlId: String, inspName: String, args: Option[String], parent: Span = null): Future[NCInspectionResult] =
+ startScopedSpan(
+ "inspect",
+ parent,
+ "modelId" → mdlId,
+ "inspName" → inspName,
+ "args" -> args.orNull) { _ ⇒
+ PROBE_INSPECTIONS.find(_.getName == inspName) match {
+ case Some(insp) ⇒ insp.inspect(mdlId, inspName, args)
+ case None ⇒ throw new NCE(s"Unsupported inspection: $inspName")
}
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
deleted file mode 100644
index e2a6852..0000000
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
-
-import java.util.concurrent.{ExecutorService, Executors}
-
-import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.NCService
-import org.apache.nlpcraft.common.inspections.NCInspectionResult
-import org.apache.nlpcraft.common.util.NCUtils
-
-import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future}
-
-/**
- * // TODO: duplicated with server component (same API) etc - should we move it into common?
- */
-private[inspections] trait NCInspector extends NCService {
- @volatile private var pool: ExecutorService = _
- @volatile protected var executor: ExecutionContextExecutor = _
-
- /**
- *
- * @param mdlId
- * @param inspId
- * @param args
- * @param parent
- * @return
- */
- def inspect(mdlId: String, inspId: String, args: Option[String], parent: Span = null): Future[NCInspectionResult]
-
- override def start(parent: Span): NCService =
- startScopedSpan("start", parent) { _ ⇒
- pool = Executors.newCachedThreadPool()
- executor = ExecutionContext.fromExecutor(pool)
-
- super.start(parent)
- }
-
- override def stop(parent: Span): Unit =
- startScopedSpan("stop", parent) { _ ⇒
- super.stop(parent)
-
- NCUtils.shutdownPools(pool)
-
- executor = null
- pool = null
- }
-}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
deleted file mode 100644
index 9983b0d..0000000
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
-
-import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.NCE
-import org.apache.nlpcraft.common.inspections.NCInspectionResult
-import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
-import org.apache.nlpcraft.common.makro.NCMacroParser
-import org.apache.nlpcraft.common.nlp.core.NCNlpPorterStemmer
-import org.apache.nlpcraft.model.intent.impl.NCIntentScanner
-import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
-
-import scala.collection.JavaConverters._
-import scala.collection._
-import scala.concurrent.Future
-
-// TODO:
-object NCInspectorIntents extends NCInspector {
- private final val SEPARATORS = Seq('?', ',', '.', '-', '!')
-
- override def inspect(mdlId: String, inspId: String, args: Option[String], parent: Span = null): Future[NCInspectionResult] =
- startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
- Future {
- val now = System.currentTimeMillis()
-
- val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new NCE(s"Model not found: '$mdlId'")).model
-
- val res = NCIntentScanner.scanIntentsSamples(mdl.proxy)
-
- val warns = mutable.ArrayBuffer.empty[String] ++ res.warnings
-
- val parser = new NCMacroParser
-
- mdl.getMacros.asScala.foreach { case (name, str) ⇒ parser.addMacro(name, str) }
-
- val allSyns: Set[Seq[String]] =
- mdl.getElements.
- asScala.
- flatMap(_.getSynonyms.asScala.flatMap(parser.expand)).
- map(NCNlpPorterStemmer.stem).map(_.split(" ").toSeq).
- toSet
-
- res.samples.
- flatMap { case (_, samples) ⇒ samples.map(_.toLowerCase) }.
- // Note that we don't use system tokenizer, because ContextWordServer doesn't have this tokenizer.
- // We just split examples words with spaces. Also we divide SEPARATORS as separated words.
- map(s ⇒ s → SEPARATORS.foldLeft(s)((s, ch) ⇒ s.replaceAll(s"\\$ch", s" $ch "))).
- foreach { case (s, sNorm) ⇒
- val seq: Seq[String] = sNorm.split(" ").map(NCNlpPorterStemmer.stem)
-
- if (!allSyns.exists(_.intersect(seq).nonEmpty))
- warns += s"Sample: '$s' doesn't contain synonyms"
- }
-
- NCInspectionResultImpl(
- inspectionId = inspId,
- modelId = mdlId,
- inspectionArguments = None,
- durationMs = System.currentTimeMillis() - now,
- timestamp = now,
- errors = Seq.empty,
- warnings = warns,
- suggestions = Seq.empty
- )
- }(executor)
- }
-}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
deleted file mode 100644
index 19ab343..0000000
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
-
-import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.NCE
-import org.apache.nlpcraft.common.inspections.NCInspectionResult
-import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
-import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
-
-import scala.collection.JavaConverters._
-import scala.collection.Seq
-import scala.concurrent.Future
-
-// TODO:
-object NCInspectorMacros extends NCInspector {
- override def inspect(mdlId: String, inspId: String, args: Option[String], parent: Span = null): Future[NCInspectionResult] =
- startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
- Future {
- val now = System.currentTimeMillis()
-
- val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new NCE(s"Model not found: '$mdlId'")).model
-
- val syns = mdl.getElements.asScala.flatMap(_.getSynonyms.asScala)
-
- val warns =
- mdl.getMacros.asScala.keys.
- // TODO: is it valid check? (simple contains ?)
- flatMap(m ⇒ if (syns.exists(_.contains(m))) None else Some(s"Macro is not used: $m")).
- toSeq
-
- NCInspectionResultImpl(
- inspectionId = inspId,
- modelId = mdlId,
- inspectionArguments = None,
- durationMs = System.currentTimeMillis() - now,
- timestamp = now,
- errors = Seq.empty,
- warnings = warns,
- suggestions = Seq.empty
- )
- }(executor)
- }
-}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
deleted file mode 100644
index 2011351..0000000
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
-
-import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.NCE
-import org.apache.nlpcraft.common.inspections.NCInspectionResult
-import org.apache.nlpcraft.common.inspections.impl.NCInspectionResultImpl
-import org.apache.nlpcraft.common.makro.NCMacroParser
-import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
-
-import scala.collection.JavaConverters._
-import scala.collection.{Seq, mutable}
-import scala.concurrent.Future
-
-// TODO:
-object NCInspectorSynonyms extends NCInspector {
- private final val TOO_MANY_SYNS = 10000
-
- override def inspect(mdlId: String, inspId: String, args: Option[String], parent: Span = null): Future[NCInspectionResult] =
- startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
- Future {
- val now = System.currentTimeMillis()
-
- val mdl = NCModelManager.getModel(mdlId).getOrElse(throw new NCE(s"Model not found: '$mdlId'")).model
-
- val warns = mutable.ArrayBuffer.empty[String]
-
- val parser = new NCMacroParser()
-
- mdl.getMacros.asScala.foreach { case (name, str) ⇒ parser.addMacro(name, str) }
-
- val mdlSyns = mdl.getElements.asScala.map(p ⇒ p.getId → p.getSynonyms.asScala.flatMap(parser.expand))
-
- mdlSyns.foreach { case (elemId, syns) ⇒
- val size = syns.size
-
- if (size == 0)
- warns += s"Element: '$elemId' doesn't have synonyms"
- else if (size > TOO_MANY_SYNS)
- warns += s"Element: '$elemId' has too many synonyms: $size"
-
- val others = mdlSyns.filter { case (othId, _) ⇒ othId != elemId }
-
- val intersects =
- others.filter { case (_, othSyns) ⇒ othSyns.intersect(syns).nonEmpty }.toMap.keys.mkString(",")
-
- if (intersects.nonEmpty)
- warns += s"Element: '$elemId' has same synonyms with '$intersects'"
- }
-
- NCInspectionResultImpl(
- inspectionId = inspId,
- modelId = mdlId,
- inspectionArguments = None,
- durationMs = System.currentTimeMillis() - now,
- timestamp = now,
- errors = Seq.empty,
- warnings = warns,
- suggestions = Seq.empty
- )
- }(executor)
- }
-}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCIntentsInspection.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCIntentsInspection.scala
new file mode 100644
index 0000000..592a606
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCIntentsInspection.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
+
+import org.apache.nlpcraft.common.inspections.NCInspectionService
+import org.apache.nlpcraft.common.makro.NCMacroParser
+import org.apache.nlpcraft.common.nlp.core.NCNlpPorterStemmer
+import org.apache.nlpcraft.model.NCModel
+import org.apache.nlpcraft.model.intent.impl.NCIntentScanner
+
+import scala.collection.JavaConverters._
+import scala.collection._
+
+/**
+ * Inspection for model's intents.
+ */
+object NCIntentsInspection extends NCInspectionService {
+ private final val SEPARATORS = Seq('?', ',', '.', '-', '!')
+
+ override def getName: String = "intents"
+
+ override def bodyOnProbe(
+ mdl: NCModel,
+ args: Option[String],
+ suggs: mutable.Buffer[String],
+ warns: mutable.Buffer[String],
+ errs: mutable.Buffer[String]): Unit = {
+ if (args.isDefined && args.get.nonEmpty)
+ warns += s"Invalid inspection arguments (ignoring): ${args.get}"
+
+ val res = NCIntentScanner.scanIntentsSamples(mdl)
+
+ val parser = new NCMacroParser
+
+ mdl.getMacros.asScala.foreach { case (name, str) ⇒ parser.addMacro(name, str) }
+
+ val allSyns: Set[Seq[String]] =
+ mdl.getElements.
+ asScala.
+ flatMap(_.getSynonyms.asScala.flatMap(parser.expand)).
+ map(NCNlpPorterStemmer.stem).map(_.split(" ").toSeq).
+ toSet
+
+ res.samples.
+ flatMap { case (_, samples) ⇒ samples.map(_.toLowerCase) }.
+ map(s ⇒ s → SEPARATORS.foldLeft(s)((s, ch) ⇒ s.replaceAll(s"\\$ch", s" $ch "))).
+ foreach {
+ case (s, sNorm) ⇒
+ val seq: Seq[String] = sNorm.split(" ").map(NCNlpPorterStemmer.stem)
+
+ if (!allSyns.exists(_.intersect(seq).nonEmpty))
+ warns += s"Sample: '$s' doesn't contain synonyms"
+ }
+ }
+}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCMacrosInspection.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCMacrosInspection.scala
new file mode 100644
index 0000000..713343b
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCMacrosInspection.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
+
+import org.apache.nlpcraft.common.inspections.NCInspectionService
+import org.apache.nlpcraft.model.NCModel
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+/**
+ * Inspection for model's macros.
+ */
+object NCMacrosInspection extends NCInspectionService {
+ override def getName: String = "macros"
+
+ override def bodyOnProbe(
+ mdl: NCModel,
+ args: Option[String],
+ suggs: mutable.Buffer[String],
+ warns: mutable.Buffer[String],
+ errs: mutable.Buffer[String]): Unit = {
+ if (args.isDefined && args.get.nonEmpty)
+ warns += s"Invalid inspection arguments (ignoring): ${args.get}"
+
+ val syns = mdl.getElements.asScala.flatMap(_.getSynonyms.asScala)
+
+ warns ++= mdl.getMacros.asScala.keys.flatMap(makro ⇒
+ if (syns.exists(_.contains(makro))) None else Some(s"Unused macro: $makro")
+ )
+ }
+}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCSynonymsInspection.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCSynonymsInspection.scala
new file mode 100644
index 0000000..bc0d92c
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCSynonymsInspection.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.probe.mgrs.inspections.inspectors
+
+import org.apache.nlpcraft.common.inspections.NCInspectionService
+import org.apache.nlpcraft.common.makro.NCMacroParser
+import org.apache.nlpcraft.model.NCModel
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+/**
+ * Inspection for model's synonyms.
+ */
+object NCSynonymsInspection extends NCInspectionService {
+ // Pretty arbitrary number...
+ // TODO: make it part of inspection configuration.
+ private final val TOO_MANY_SYNS = 10000
+
+ override def getName: String = "synonyms"
+
+ override def bodyOnProbe(
+ mdl: NCModel,
+ args: Option[String],
+ suggs: mutable.Buffer[String],
+ warns: mutable.Buffer[String],
+ errs: mutable.Buffer[String]): Unit = {
+ if (args.isDefined && args.get.nonEmpty)
+ warns += s"Invalid inspection arguments (ignoring): ${args.get}"
+
+ val parser = new NCMacroParser()
+
+ mdl.getMacros.asScala.foreach { case (name, str) ⇒ parser.addMacro(name, str) }
+
+ val mdlSyns = mdl.getElements.asScala.map(p ⇒ p.getId → p.getSynonyms.asScala.flatMap(parser.expand))
+
+ mdlSyns.foreach { case (elemId, syns) ⇒
+ val size = syns.size
+
+ if (size == 0)
+ warns += s"Element '$elemId' doesn't have synonyms."
+ else if (size > TOO_MANY_SYNS)
+ warns += s"Element '$elemId' has too many ($size) synonyms. Make sure this is truly necessary."
+
+ val others = mdlSyns.filter {
+ case (otherId, _) ⇒ otherId != elemId
+ }
+
+ val cross = others.filter {
+ case (_, othSyns) ⇒ othSyns.intersect(syns).nonEmpty
+ }.toMap.keys.mkString(",")
+
+ if (cross.nonEmpty)
+ errs += s"Element '$elemId' has duplicate synonyms with elements '$cross'."
+ }
+ }
+}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspection/NCInspectionManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspection/NCInspectionManager.scala
index 6fe5793..cff52c7 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspection/NCInspectionManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspection/NCInspectionManager.scala
@@ -28,12 +28,11 @@ import scala.collection.Map
import scala.concurrent.Future
/**
- * Model inspection manager.
+ * Server-side inspection manager.
*/
object NCInspectionManager extends NCService {
private final val ALL_INSPECTIONS: Seq[NCInspection] = Seq(
NCInspectionImpl(
- id = "macros",
name = "macros",
synopsis = "macros",
parameters = Seq.empty,
@@ -41,7 +40,6 @@ object NCInspectionManager extends NCService {
isServerSide = false
),
NCInspectionImpl(
- id = "intents",
name = "intents",
synopsis = "intents",
parameters = Seq.empty,
@@ -49,7 +47,6 @@ object NCInspectionManager extends NCService {
isServerSide = false
),
NCInspectionImpl(
- id = "synonyms",
name = "synonyms",
synopsis = "synonyms",
parameters = Seq.empty,
@@ -57,12 +54,10 @@ object NCInspectionManager extends NCService {
isServerSide = false
),
NCInspectionImpl(
- id = "suggestions",
name = "suggestions",
synopsis = "suggestions",
parameters = Seq(
NCInspectionParameterImpl(
- id = "minScore",
name = "minScore",
value = "minScore",
valueType = "double",
@@ -94,14 +89,14 @@ object NCInspectionManager extends NCService {
/**
*
* @param mdlId Model ID.
- * @param inspId Inspection ID.
+ * @param inspName Inspection name.
* @param args Inspection arguments .
* @param parent Optional parent trace span.
*/
- def inspect(mdlId: String, inspId: String, args: Option[String], parent: Span = null): Future[NCInspectionResult] =
- SRV_INSPECTIONS.get(inspId) match {
- case Some(insp) ⇒ insp.inspect(mdlId, inspId, args, parent)
- case None ⇒ NCProbeManager.runInspection(mdlId, inspId, args, parent)
+ def inspect(mdlId: String, inspName: String, args: Option[String], parent: Span = null): Future[NCInspectionResult] =
+ SRV_INSPECTIONS.get(inspName) match {
+ case Some(insp) ⇒ insp.inspect(mdlId, inspName, args, parent)
+ case None ⇒ NCProbeManager.runInspection(mdlId, inspName, args, parent)
}
/**
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspection/impl/NCSuggestionInspection.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspection/impl/NCSuggestionInspection.scala
index 90eaf25..0158140 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspection/impl/NCSuggestionInspection.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspection/impl/NCSuggestionInspection.scala
@@ -127,7 +127,14 @@ object NCSuggestionInspection extends NCInspectionService {
seq
}
- override def inspect(mdlId: String, inspId: String, args: Option[String], parent: Span = null): Future[NCInspectionResult] =
+
+ /**
+ *
+ * @return
+ */
+ override def getName: String = "suggestions"
+
+ override def inspect(mdlId: String, inspName: String, args: Option[String], parent: Span = null): Future[NCInspectionResult] =
startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
val now = System.currentTimeMillis()
@@ -176,7 +183,7 @@ object NCSuggestionInspection extends NCInspectionService {
def onError(err: String): Unit =
promise.success(
NCInspectionResultImpl(
- inspectionId = inspId,
+ inspectionId = inspName,
modelId = mdlId,
inspectionArguments = None,
durationMs = System.currentTimeMillis() - now,
@@ -410,7 +417,7 @@ object NCSuggestionInspection extends NCInspectionService {
promise.success(
NCInspectionResultImpl(
- inspectionId = inspId,
+ inspectionId = inspName,
modelId = mdlId,
inspectionArguments = None,
durationMs = System.currentTimeMillis() - now,
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index 8a30e63..94d2b1f 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -980,14 +980,14 @@ object NCProbeManager extends NCService {
/**
*
* @param mdlId
- * @param inspId
+ * @param inspName
* @param args
* @param parent
* @return
*/
- def runInspection(mdlId: String, inspId: String, args: Option[String], parent: Span = null): Future[NCInspectionResult] =
- startScopedSpan("runInspection", parent, "modelId" → mdlId, "inspId" → inspId) { _ ⇒
- val params = mutable.HashMap.empty[String, Serializable] ++ Map("mdlId" → mdlId, "inspId" → inspId)
+ def runInspection(mdlId: String, inspName: String, args: Option[String], parent: Span = null): Future[NCInspectionResult] =
+ startScopedSpan("runInspection", parent, "modelId" → mdlId, "inspName" → inspName) { _ ⇒
+ val params = mutable.HashMap.empty[String, Serializable] ++ Map("mdlId" → mdlId, "inspName" → inspName)
if (args.isDefined)
params += "args" → GSON.toJson(args.get)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
index 44b8b28..c7132ac 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
@@ -88,7 +88,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
private final val STD_FIELD_LENGTHS = Map[String, Int](
"acsTok" -> 256,
"mdlId" -> 32,
- "inspId" -> 32,
+ "inspName" -> 32,
"userExtId" -> 64,
"extId" -> 64,
"name" -> 64,
@@ -746,14 +746,14 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
val acsTok: String = convert(obj, "acsTok", (js: JsValue) ⇒ js.convertTo[String])
val mdlId: String = convert(obj, "mdlId", (js: JsValue) ⇒ js.convertTo[String])
- val inspId: String = convert(obj, "inspId", (js: JsValue) ⇒ js.convertTo[String])
+ val inspName: String = convert(obj, "inspName", (js: JsValue) ⇒ js.convertTo[String])
val args: Option[String] = convertOpt(obj, "args", v ⇒ v.compactPrint)
startScopedSpan("modelEnhance$", "mdlId" → mdlId, "acsTok" → acsTok) { span ⇒
checkLength(
"acsTok" -> acsTok,
"mdlId" -> mdlId,
- "inspId" -> inspId
+ "inspName" -> inspName
)
val admUsr = authenticateAsAdmin(acsTok)
@@ -761,7 +761,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
if (!NCProbeManager.getAllProbes(admUsr.companyId, span).exists(_.models.exists(_.id == mdlId)))
throw new NCE(s"Probe not found for model: $mdlId")
- val fut = NCInspectionManager.inspect(mdlId, inspId, args, span)
+ val fut = NCInspectionManager.inspect(mdlId, inspName, args, span)
fut.failed.collect {
case e ⇒ onError(e)
@@ -786,7 +786,6 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
acsTok: String
)
case class Parameter_Inspection$All(
- id: String,
name: String,
value: String,
valueType: String,
@@ -794,7 +793,6 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
description: String
)
case class Inspection_Inspection$All(
- id: String,
name: String,
synopsis: String,
parameters: Seq[Parameter_Inspection$All],
@@ -807,8 +805,8 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
)
implicit val reqFmt: RootJsonFormat[Req_Inspection$All] = jsonFormat1(Req_Inspection$All)
- implicit val paramFmt: RootJsonFormat[Parameter_Inspection$All] = jsonFormat6(Parameter_Inspection$All)
- implicit val inspFmt: RootJsonFormat[Inspection_Inspection$All] = jsonFormat6(Inspection_Inspection$All)
+ implicit val paramFmt: RootJsonFormat[Parameter_Inspection$All] = jsonFormat5(Parameter_Inspection$All)
+ implicit val inspFmt: RootJsonFormat[Inspection_Inspection$All] = jsonFormat5(Inspection_Inspection$All)
implicit val resFmt: RootJsonFormat[Res_Inspection$All] = jsonFormat2(Res_Inspection$All)
entity(as[Req_Inspection$All]) { req ⇒
@@ -818,12 +816,10 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w
authenticateAsAdmin(req.acsTok)
val inspections = NCInspectionManager.allInspections(span).map(i ⇒ Inspection_Inspection$All(
- id = i.id(),
name = i.name(),
synopsis = i.synopsis(),
parameters = i.parameters().asScala.map(p ⇒
Parameter_Inspection$All(
- id = p.id(),
name = p.name(),
value = p.value(),
valueType = p.valueType(),
diff --git a/openapi/nlpcraft_swagger.yml b/openapi/nlpcraft_swagger.yml
index 8b22d9f..1ae7c1f 100644
--- a/openapi/nlpcraft_swagger.yml
+++ b/openapi/nlpcraft_swagger.yml
@@ -176,7 +176,7 @@ paths:
required:
- acsTok
- mdlId
- - inspId
+ - inspName
properties:
acsTok:
type: string
@@ -186,7 +186,7 @@ paths:
type: string
maxLength: 32
description: ID of the model to perform given inspection on
- inspId:
+ inspName:
type: string
maxLength: 32
description: Inspection ID