You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2021/04/14 17:59:29 UTC
[incubator-nlpcraft] branch NLPCRAFT-287 updated (130b3cf ->
9beb631)
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a change to branch NLPCRAFT-287
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
from 130b3cf WIP.
new e6451e5 WIP.
new 101e215 WIP.
new 24a169c WIP.
new de6df64 WIP.
new facba33 WIP.
add a3f9f2b start_with() => starts_with, end_with => ends_with() for consistency with JDK.
new 9beb631 Merge branch 'master' into NLPCRAFT-287
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../model/intent/compiler/NCIdlCompilerBase.scala | 8 +-
.../nlpcraft/model/intent/compiler/antlr4/NCIdl.g4 | 4 +-
.../intent/compiler/antlr4/NCIdlBaseListener.java | 2 +-
.../model/intent/compiler/antlr4/NCIdlLexer.interp | 2 +-
.../model/intent/compiler/antlr4/NCIdlLexer.java | 685 +++++++++++----------
.../intent/compiler/antlr4/NCIdlListener.java | 2 +-
.../model/intent/compiler/antlr4/NCIdlParser.java | 2 +-
.../apache/nlpcraft/probe/mgrs/NCProbeModel.scala | 10 +-
.../nlpcraft/probe/mgrs/NCProbeSynonym.scala | 97 ++-
.../probe/mgrs/deploy/NCDeployManager.scala | 34 +-
.../nlpcraft/probe/mgrs/model/NCModelManager.scala | 37 +-
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 281 ++++-----
.../org/apache/nlpcraft/model/NCIdlSpec.scala | 2 +-
.../org/apache/nlpcraft/model/NCIdlSpec2.scala | 2 +-
.../nlpcraft/model/intent/idl/NCIdlTestSpec.scala | 2 +-
.../intent/idl/compiler/NCIdlCompilerSpec.scala | 8 +-
.../compiler/functions/NCIdlFunctionsStrings.scala | 8 +-
.../nlpcraft/model/intent/idl/idl_test_model.yaml | 2 +-
.../model/NCEnricherNestedModelSpec.scala | 48 +-
19 files changed, 604 insertions(+), 632 deletions(-)
[incubator-nlpcraft] 03/06: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-287
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 24a169c0e5346c9d9e9c25066a4b11d85cffa5ff
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Wed Apr 14 13:15:35 2021 +0300
WIP.
---
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 22 +++++-----
.../model/NCEnricherNestedModelSpec.scala | 48 +++++++++++-----------
2 files changed, 34 insertions(+), 36 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
index 29ec156..8ca93f2 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
@@ -36,7 +36,6 @@ import scala.collection.convert.DecorateAsScala
import scala.collection.mutable.ArrayBuffer
import scala.collection.{Map, Seq, mutable}
-
/**
* Model elements enricher.
*/
@@ -501,10 +500,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
mdl.elements.keys.map(k ⇒ k → mutable.ArrayBuffer.empty[Seq[Int]])
lazy val idlCache = mutable.HashSet.empty[Seq[Complex]]
- var found = false
-
def add(typ: String, elm: NCElement, res: Seq[NlpToken], allToksIdxs: Seq[Int], s: Synonym, parts: Seq[TokType] = Seq.empty): Unit = {
- found = true
val resIdxs = res.map(_.index)
val continuous = U.isContinuous(resIdxs.sorted)
@@ -555,7 +551,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
case _ ⇒ false
}
- found = false
+ var found = false
// 1.1 Continuous.
if (simpleEnabled)
@@ -563,14 +559,18 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
case Some(h) ⇒
def tryMap(syns: Map[String, Synonym], notFound: () ⇒ Unit): Unit =
syns.get(tokStems) match {
- case Some(s) ⇒ add("simple continuous", elm, toks, tokIdxs, s)
+ case Some(s) ⇒
+ found = true
+ add("simple continuous", elm, toks, tokIdxs, s)
case None ⇒ notFound()
}
def tryScan(syns: Seq[Synonym]): Unit =
for (s ← syns if !found)
- if (s.isMatch(toks))
+ if (s.isMatch(toks)) {
+ found = true
add("simple continuous scan", elm, toks, tokIdxs, s)
+ }
tryMap(
h.txtDirectSynonyms,
@@ -585,8 +585,8 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
}
// 1.2 Sparse.
- if (simpleEnabled && !found)
- for (s ← get(mdl.sparseSynonyms, elemId) if !found)
+ if (simpleEnabled)
+ for (s ← get(mdl.sparseSynonyms, elemId))
s.sparseMatch(toks) match {
case Some(res) ⇒ add("simple sparse", elm, res, tokIdxs, s)
case None ⇒ // No-op.
@@ -594,11 +594,9 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
// 2. IDL.
if (state != SIMPLE && mdl.hasIdlSynonyms) {
- found = false
-
// 2.1 Sparse.
if (mdl.hasIdlSynonyms)
- for (s ← get(mdl.idlSynonyms, elemId); comb ← idlCombs if !found)
+ for (s ← get(mdl.idlSynonyms, elemId); comb ← idlCombs)
s.idlMatch(comb.map(_.data), req) match {
case Some(res) ⇒
val typ = if (s.sparse) "IDL sparse" else "IDL continuous"
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCEnricherNestedModelSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCEnricherNestedModelSpec.scala
index 00ee50f..91d037b 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCEnricherNestedModelSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCEnricherNestedModelSpec.scala
@@ -79,34 +79,34 @@ class NCEnricherNestedModelSpec2 extends NCEnricherNestedModelSpec1 {
@Test
def test2(): Unit =
runBatch(
-// _ ⇒ checkExists(
-// "test tomorrow",
-// usr(text = "test tomorrow", id = "x3")
-// ),
-// _ ⇒ checkExists(
-// "tomorrow test",
-// usr(text = "tomorrow test", id = "x3")
-// ),
+ _ ⇒ checkExists(
+ "test tomorrow",
+ usr(text = "test tomorrow", id = "x3")
+ ),
+ _ ⇒ checkExists(
+ "tomorrow test",
+ usr(text = "tomorrow test", id = "x3")
+ ),
_ ⇒ checkExists(
"test xxx tomorrow",
usr(text = "test tomorrow", id = "x3"),
nlp(text = "xxx"),
),
-// _ ⇒ checkExists(
-// "y the y",
-// usr(text = "y y", id = "y3"),
-// nlp(text = "the", isStop = true)
-// ),
-// _ ⇒ checkExists(
-// "y xxx y",
-// usr(text = "y y", id = "y3"),
-// nlp(text = "xxx")
-// ),
-// _ ⇒ checkExists(
-// "aaa y xxx y",
-// nlp(text = "aaa"),
-// usr(text = "y y", id = "y3"),
-// nlp(text = "xxx")
-// )
+ _ ⇒ checkExists(
+ "y the y",
+ usr(text = "y y", id = "y3"),
+ nlp(text = "the", isStop = true)
+ ),
+ _ ⇒ checkExists(
+ "y xxx y",
+ usr(text = "y y", id = "y3"),
+ nlp(text = "xxx")
+ ),
+ _ ⇒ checkExists(
+ "aaa y xxx y",
+ nlp(text = "aaa"),
+ usr(text = "y y", id = "y3"),
+ nlp(text = "xxx")
+ )
)
}
\ No newline at end of file
[incubator-nlpcraft] 01/06: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-287
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit e6451e5cef4fe054501755688136822f6118fa8e
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Wed Apr 14 10:31:48 2021 +0300
WIP.
---
.../main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala | 4 ----
1 file changed, 4 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala
index 0a4bda6..dc9a1f5 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala
@@ -86,7 +86,6 @@ class NCProbeSynonym(
/**
*
* @param toks
- * @return
*/
private def trySparseMatch0[T](toks: Seq[T], isMatch: (T, NCProbeSynonymChunk) ⇒ Boolean, getIndex: T ⇒ Int): Option[Seq[T]] = {
require(toks != null)
@@ -162,7 +161,6 @@ class NCProbeSynonym(
/**
*
* @param toks
- * @return
*/
def isMatch(toks: NCNlpSentenceTokenBuffer): Boolean = {
require(toks != null)
@@ -181,7 +179,6 @@ class NCProbeSynonym(
/**
*
* @param toks
- * @return
*/
def trySparseMatch(toks: NCNlpSentenceTokenBuffer): Option[Seq[NCNlpSentenceToken]] = {
require(toks != null)
@@ -198,7 +195,6 @@ class NCProbeSynonym(
*
* @param tows
* @param req
- * @return
*/
def isMatch(tows: Seq[NCDslContent], req: NCRequest): Boolean = {
require(tows != null)
[incubator-nlpcraft] 05/06: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-287
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit facba336dd6c9af04654e2cde381ffe085b107e7
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Wed Apr 14 18:14:31 2021 +0300
WIP.
---
.../apache/nlpcraft/probe/mgrs/NCProbeModel.scala | 1 +
.../nlpcraft/probe/mgrs/model/NCModelManager.scala | 8 -
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 193 ++++++++++-----------
3 files changed, 95 insertions(+), 107 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeModel.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeModel.scala
index a4e55a6..2670fb7 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeModel.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeModel.scala
@@ -50,4 +50,5 @@ case class NCProbeModel(
) {
def hasIdlSynonyms(elemId: String): Boolean = idlSynonyms.contains(elemId)
def hasIdlSynonyms: Boolean = idlSynonyms.nonEmpty
+ def hasNoIdlSynonyms: Boolean = continuousSynonyms.nonEmpty || sparseSynonyms.nonEmpty
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
index 2b8313c..3bd052c 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
@@ -58,14 +58,6 @@ object NCModelManager extends NCService with DecorateAsScala {
data.values.foreach(w ⇒ {
val mdl = w.model
- // TODO:
- val elemId = "col:orders_order_date"
-
- println("w.directSynonyms="+w.continuousSynonyms.getOrElse(elemId, Map.empty).mkString("\n"))
- println("w.sparseSynonyms="+w.sparseSynonyms.getOrElse(elemId, Seq.empty).mkString("\n"))
- println("w.idlSynonyms="+w.idlSynonyms.getOrElse(elemId, Seq.empty).mkString("\n"))
- println
-
val contCnt = w.continuousSynonyms.flatMap(_._2.map(_._2.count)).sum
val sparseCnt = w.sparseSynonyms.map(_._2.size).sum
val allIdlSyns = w.idlSynonyms.values.flatten
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
index f01619c..79bcf3f 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
@@ -147,6 +147,8 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
ackStopped()
}
+ def isComplex(mdl: NCProbeModel): Boolean = mdl.hasIdlSynonyms || !mdl.model.getParsers.isEmpty
+
/**
*
* @param ns
@@ -156,7 +158,6 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
* @param syn
* @param metaOpt
* @param parts
- * @param allToksIdxs
* @param continuous
*/
private def mark(
@@ -164,11 +165,10 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
elem: NCElement,
toks: Seq[NlpToken],
direct: Boolean,
- syn: Option[Synonym],
- metaOpt: Option[Map[String, Object]],
- parts: Seq[TokType],
- allToksIdxs: Seq[Int],
- continuous: java.lang.Boolean
+ continuous: java.lang.Boolean,
+ syn: Option[Synonym] = None,
+ parts: Seq[TokType] = Seq.empty,
+ metaOpt: Option[Map[String, Object]] = None
): Unit = {
val params = mutable.ArrayBuffer.empty[(String, AnyRef)]
@@ -176,7 +176,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
params += "direct" → direct.asInstanceOf[AnyRef]
// Internal usage.
- params += "allToksIndexes" → allToksIdxs.asJava
+ params += "sortedTokensIndexes" → toks.map(_.index).sorted.asJava
params += "continuous" → continuous
syn match {
@@ -287,20 +287,16 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
)
// Checks element's tokens.
- val idxs = matchedToks.map(_.index)
- val continuous = U.isContinuous(idxs.sorted)
+ val idxs = matchedToks.map(_.index).sorted
- if (!alreadyMarked(matchedToks, idxs, continuous, elemId))
+ if (!alreadyMarked(ns, elemId, matchedToks, idxs))
mark(
ns,
elem = mdl.elements.getOrElse(elemId, throw new NCE(s"Custom model parser returned unknown element ID: $elemId")),
toks = matchedToks,
direct = true,
- syn = None,
- metaOpt = Some(e.getMetadata.asScala),
- parts = Seq.empty,
- idxs,
- continuous
+ U.isContinuous(idxs),
+ metaOpt = Some(e.getMetadata.asScala)
)
})
}
@@ -333,39 +329,6 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
/**
*
- * @param toks
- * @param elemId
- */
- private def alreadyMarked(toks: Seq[NlpToken], allToksIndexes: Seq[Int], continuous: Boolean, elemId: String): Boolean = {
- toks.forall(t ⇒ t.isTypeOf(elemId)) &&
- toks.head.filter(_.noteType == elemId).exists(n ⇒ n.tokenIndexes.toSet == toks.map(_.index).toSet)
- toks.flatten.exists(n ⇒
- n.noteType == elemId && {
- if (n.data("continuous").asInstanceOf[Boolean])
- false
- else {
- if (continuous)
- false
- else
- n.data("allToksIndexes").asInstanceOf[JList[Int]].asScala.containsSlice(allToksIndexes)
- }
- }
- )
- }
-
- /**
- *
- * @param seq
- * @param s
- */
- private def toPartsComplex(seq: Seq[Complex], s: Synonym): Seq[TokType] =
- seq.zip(s.map(_.kind)).flatMap {
- case (complex, kind) ⇒ if (complex.isToken) Some(complex.token → kind)
- else None
- }
-
- /**
- *
* @param seq
* @param s
*/
@@ -468,6 +431,46 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
}).seq
}
+ private def add(
+ ns: NCNlpSentence,
+ contCache: mutable.Map[String, ArrayBuffer[Seq[Int]]],
+ typ: String,
+ elm: NCElement,
+ res: Seq[NlpToken],
+ allToksIdxs: Seq[Int],
+ syn: Synonym,
+ parts: Seq[TokType] = Seq.empty)
+ : Unit = {
+ val resIdxs = res.map(_.index)
+
+ val continuous = U.isContinuous(resIdxs.sorted)
+
+ if (continuous && resIdxs == allToksIdxs)
+ contCache(elm.getId) += allToksIdxs
+
+ val ok = !alreadyMarked(ns, elm.getId, res, allToksIdxs)
+
+ if (ok) {
+ val direct = syn.isDirect && U.isIncreased(resIdxs)
+
+ mark(ns, elm, res, direct, continuous, syn = Some(syn), parts)
+ }
+
+ if (DEEP_DEBUG && ok)
+ println(
+ s"${if (ok) "Added" else "Skipped"} element [" +
+ s"id=${elm.getId}, " +
+ s"type=$typ, " +
+ s"text='${res.map(_.origText).mkString(" ")}', " +
+ s"indexes=${resIdxs.mkString("[", ",", "]")}, " +
+ s"allTokensIndexes=${allToksIdxs.mkString("[", ",", "]")}, " +
+ s"continuous=$continuous, " +
+ s"synonym=$syn" +
+ s"]"
+ )
+ }
+
+
/**
*
* @param mdl
@@ -493,41 +496,9 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
if (DEEP_DEBUG)
println(s"Execution started [simpleEnabled=$simpleEnabled, idlEnabled=$idlEnabled]")
- val contCache =
- mutable.HashMap.empty[String, mutable.ArrayBuffer[Seq[Int]]] ++
- mdl.elements.keys.map(k ⇒ k → mutable.ArrayBuffer.empty[Seq[Int]])
+ val contCache = mutable.HashMap.empty ++ mdl.elements.keys.map(k ⇒ k → mutable.ArrayBuffer.empty[Seq[Int]])
lazy val idlCache = mutable.HashSet.empty[Seq[Complex]]
- def add(typ: String, elm: NCElement, res: Seq[NlpToken], allToksIdxs: Seq[Int], s: Synonym, parts: Seq[TokType] = Seq.empty): Unit = {
- val resIdxs = res.map(_.index)
-
- val continuous = U.isContinuous(resIdxs.sorted)
-
- if (continuous && resIdxs == allToksIdxs)
- contCache(elm.getId) += allToksIdxs
-
- val added = !alreadyMarked(res, allToksIdxs, continuous, elm.getId)
-
- if (added) {
- val direct = s.isDirect && U.isIncreased(resIdxs)
-
- mark(ns, elm, res, direct, syn = Some(s), metaOpt = None, parts, allToksIdxs, continuous)
- }
-
- if (DEEP_DEBUG)
- println(
- s"${if (added) "Added" else "Skipped"} element [" +
- s"id=${elm.getId}, " +
- s"type=$typ, " +
- s"text='${res.map(_.origText).mkString(" ")}', " +
- s"indexes=${resIdxs.mkString("[", ",", "]")}, " +
- s"allTokensIndexes=${allToksIdxs.mkString("[", ",", "]")}, " +
- s"continuous=$continuous, " +
- s"synonym=$s" +
- s"]"
- )
- }
-
for (toks ← combosToks) {
val tokIdxs = toks.map(_.index)
lazy val tokStems = toks.map(_.stem).mkString(" ")
@@ -537,8 +508,8 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
elm ← mdl.elements.values;
elemId = elm.getId
if
- !contCache(elemId).exists(_.containsSlice(tokIdxs)) &&
- !alreadyMarked(toks, tokIdxs, continuous = true, elemId) // Checks whole tokens slice.
+ !contCache(elemId).exists(_.containsSlice(tokIdxs)) &&
+ !alreadyMarked(ns, elemId, toks, tokIdxs)
) {
// 1. SIMPLE.
if (simpleEnabled && (if (idlEnabled) mdl.hasIdlSynonyms(elemId) else !mdl.hasIdlSynonyms(elemId))) {
@@ -551,7 +522,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
syns.get(tokStems) match {
case Some(s) ⇒
found = true
- add("simple continuous", elm, toks, tokIdxs, s)
+ add(ns, contCache,"simple continuous", elm, toks, tokIdxs, s)
case None ⇒ notFound()
}
@@ -559,7 +530,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
for (s ← syns if !found)
if (s.isMatch(toks)) {
found = true
- add("simple continuous scan", elm, toks, tokIdxs, s)
+ add(ns, contCache, "simple continuous scan", elm, toks, tokIdxs, s)
}
tryMap(
@@ -575,23 +546,24 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
}
// 1.2 Sparse.
- for (s ← get(mdl.sparseSynonyms, elemId))
- s.sparseMatch(toks) match {
- case Some(res) ⇒ add("simple sparse", elm, res, tokIdxs, s)
- case None ⇒ // No-op.
- }
+ if (!found)
+ for (s ← get(mdl.sparseSynonyms, elemId))
+ s.sparseMatch(toks) match {
+ case Some(res) ⇒ add(ns, contCache, "simple sparse", elm, res, tokIdxs, s)
+ case None ⇒ // No-op.
+ }
}
// 2. IDL.
if (idlEnabled) {
- val idlCombs = mkComplexCombinations(ch, toks, idlCache.toSet)
+ lazy val idlCombs = mkComplexCombinations(ch, toks, idlCache.toSet)
for (s ← get(mdl.idlSynonyms, elemId); comb ← idlCombs)
s.idlMatch(comb.map(_.data), req) match {
case Some(res) ⇒
val typ = if (s.sparse) "IDL sparse" else "IDL continuous"
- add(typ, elm, toTokens(res, ns), tokIdxs, s, toParts(res, s))
+ add(ns, contCache, typ, elm, toTokens(res, ns), tokIdxs, s, toParts(res, s))
idlCache += comb
case None ⇒ // No-op.
@@ -611,22 +583,45 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
lazy val h = mkComplexes(mdl, ns)
- val idlEnabled = mdl.hasIdlSynonyms
-
if (ns.firstProbePhase) {
ns.firstProbePhase = false
- execute(mdl, ns, combToks, simpleEnabled = true, idlEnabled = false, req, h, parent)
- execute(mdl, ns, combToks, simpleEnabled = true, idlEnabled, req, h, parent)
+ if (mdl.hasNoIdlSynonyms)
+ execute(mdl, ns, combToks, simpleEnabled = true, idlEnabled = false, req, h, parent)
+ execute(mdl, ns, combToks, simpleEnabled = mdl.hasNoIdlSynonyms, mdl.hasIdlSynonyms, req, h, parent)
}
else {
- if (idlEnabled)
- execute(mdl, ns, combToks, simpleEnabled = false, idlEnabled, req, h, parent)
+ if (mdl.hasIdlSynonyms)
+ execute(mdl, ns, combToks, simpleEnabled = false, idlEnabled = true, req, h, parent)
}
processParsers(mdl, ns, span, req)
}
}
- def isComplex(mdl: NCProbeModel): Boolean = mdl.hasIdlSynonyms || !mdl.model.getParsers.isEmpty
+ /**
+ *
+ * @param ns
+ * @param elemId
+ * @param toks
+ * @param allSortedSliceIdxs
+ */
+ private def alreadyMarked(ns: NCNlpSentence, elemId: String, toks: Seq[NlpToken], allSortedSliceIdxs: Seq[Int]): Boolean = {
+ lazy val toksIdxsSorted = toks.map(_.index).sorted
+ lazy val isCont = U.isContinuous(toksIdxsSorted)
+
+ ns.flatten.exists(
+ n ⇒
+ n.noteType == elemId &&
+ {
+ lazy val nToksIdxsSorted = n.data[JList[Int]]("sortedTokensIndexes").asScala
+
+ n.data[Boolean]("continuous") && allSortedSliceIdxs.containsSlice(nToksIdxsSorted) ||
+ {
+ nToksIdxsSorted == toksIdxsSorted ||
+ isCont && U.isContinuous(nToksIdxsSorted) && nToksIdxsSorted.containsSlice(toksIdxsSorted)
+ }
+ }
+ )
+ }
}
\ No newline at end of file
[incubator-nlpcraft] 04/06: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-287
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit de6df644338de14f622b4ca3819cd53f2e00c39c
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Wed Apr 14 14:53:13 2021 +0300
WIP.
---
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 93 ++++++++++------------
1 file changed, 44 insertions(+), 49 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
index 8ca93f2..f01619c 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
@@ -128,14 +128,6 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
case class ComplexHolder(complexesWords: Seq[Complex], complexes: Seq[ComplexSeq])
- object State extends Enumeration {
- type State = Value
-
- val SIMPLE, IDL_FIRST, IDL_NEXT = Value
- }
-
- import State._
-
/**
*
* @param parent Optional parent span.
@@ -481,19 +473,25 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
* @param mdl
* @param ns
* @param combosToks
- * @param state
+ * @param simpleEnabled
+ * @param idlEnabled
* @param req
+ * @param ch
+ * @param span
*/
private def execute(
mdl: NCProbeModel,
ns: NCNlpSentence,
combosToks: Seq[Seq[NlpToken]],
- state: State, req: NCRequest,
- h: ⇒ ComplexHolder, span: Span
+ simpleEnabled: Boolean,
+ idlEnabled: Boolean,
+ req: NCRequest,
+ ch: ⇒ ComplexHolder,
+ span: Span
): Unit =
startScopedSpan("execute", span, "srvReqId" → ns.srvReqId, "mdlId" → mdl.model.getId, "txt" → ns.text) { _ ⇒
if (DEEP_DEBUG)
- println(s"Execution started [state=$state]")
+ println(s"Execution started [simpleEnabled=$simpleEnabled, idlEnabled=$idlEnabled]")
val contCache =
mutable.HashMap.empty[String, mutable.ArrayBuffer[Seq[Int]]] ++
@@ -519,20 +517,19 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
if (DEEP_DEBUG)
println(
s"${if (added) "Added" else "Skipped"} element [" +
- s"id=${elm.getId}, " +
- s"type=$typ, " +
- s"text='${res.map(_.origText).mkString(" ")}', " +
- s"indexes=${resIdxs.mkString("[", ",", "]")}, " +
- s"allTokensIndexes=${allToksIdxs.mkString("[", ",", "]")}, " +
- s"continuous=$continuous, " +
- s"synonym=$s" +
- s"]"
+ s"id=${elm.getId}, " +
+ s"type=$typ, " +
+ s"text='${res.map(_.origText).mkString(" ")}', " +
+ s"indexes=${resIdxs.mkString("[", ",", "]")}, " +
+ s"allTokensIndexes=${allToksIdxs.mkString("[", ",", "]")}, " +
+ s"continuous=$continuous, " +
+ s"synonym=$s" +
+ s"]"
)
}
for (toks ← combosToks) {
val tokIdxs = toks.map(_.index)
- lazy val idlCombs: Seq[Seq[Complex]] = mkComplexCombinations(h, toks, idlCache.toSet)
lazy val tokStems = toks.map(_.stem).mkString(" ")
// Attempt to match each element.
@@ -544,17 +541,10 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
!alreadyMarked(toks, tokIdxs, continuous = true, elemId) // Checks whole tokens slice.
) {
// 1. SIMPLE.
- val simpleEnabled =
- state match {
- case SIMPLE ⇒ !mdl.hasIdlSynonyms(elemId)
- case IDL_FIRST ⇒ mdl.hasIdlSynonyms(elemId)
- case _ ⇒ false
- }
-
- var found = false
+ if (simpleEnabled && (if (idlEnabled) mdl.hasIdlSynonyms(elemId) else !mdl.hasIdlSynonyms(elemId))) {
+ // 1.1 Continuous.
+ var found = false
- // 1.1 Continuous.
- if (simpleEnabled)
fastAccess(mdl.continuousSynonyms, elemId, toks.length) match {
case Some(h) ⇒
def tryMap(syns: Map[String, Synonym], notFound: () ⇒ Unit): Unit =
@@ -584,28 +574,28 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
case None ⇒ // No-op.
}
- // 1.2 Sparse.
- if (simpleEnabled)
+ // 1.2 Sparse.
for (s ← get(mdl.sparseSynonyms, elemId))
s.sparseMatch(toks) match {
case Some(res) ⇒ add("simple sparse", elm, res, tokIdxs, s)
case None ⇒ // No-op.
}
+ }
// 2. IDL.
- if (state != SIMPLE && mdl.hasIdlSynonyms) {
- // 2.1 Sparse.
- if (mdl.hasIdlSynonyms)
- for (s ← get(mdl.idlSynonyms, elemId); comb ← idlCombs)
- s.idlMatch(comb.map(_.data), req) match {
- case Some(res) ⇒
- val typ = if (s.sparse) "IDL sparse" else "IDL continuous"
-
- add(typ, elm, toTokens(res, ns), tokIdxs, s, toParts(res, s))
-
- idlCache += comb
- case None ⇒ // No-op.
- }
+ if (idlEnabled) {
+ val idlCombs = mkComplexCombinations(ch, toks, idlCache.toSet)
+
+ for (s ← get(mdl.idlSynonyms, elemId); comb ← idlCombs)
+ s.idlMatch(comb.map(_.data), req) match {
+ case Some(res) ⇒
+ val typ = if (s.sparse) "IDL sparse" else "IDL continuous"
+
+ add(typ, elm, toTokens(res, ns), tokIdxs, s, toParts(res, s))
+
+ idlCache += comb
+ case None ⇒ // No-op.
+ }
}
}
}
@@ -617,16 +607,21 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
startScopedSpan("enrich", parent, "srvReqId" → ns.srvReqId, "mdlId" → mdl.model.getId, "txt" → ns.text) { span ⇒
val req = NCRequestImpl(senMeta, ns.srvReqId)
- val combosToks = combos(ns)
+ val combToks = combos(ns)
lazy val h = mkComplexes(mdl, ns)
- execute(mdl, ns, combosToks, if (ns.firstProbePhase) SIMPLE else IDL_NEXT, req, h, parent)
+ val idlEnabled = mdl.hasIdlSynonyms
if (ns.firstProbePhase) {
ns.firstProbePhase = false
- execute(mdl, ns, combosToks, IDL_FIRST, req, h, parent)
+ execute(mdl, ns, combToks, simpleEnabled = true, idlEnabled = false, req, h, parent)
+ execute(mdl, ns, combToks, simpleEnabled = true, idlEnabled, req, h, parent)
+ }
+ else {
+ if (idlEnabled)
+ execute(mdl, ns, combToks, simpleEnabled = false, idlEnabled, req, h, parent)
}
processParsers(mdl, ns, span, req)
[incubator-nlpcraft] 06/06: Merge branch 'master' into NLPCRAFT-287
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-287
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 9beb6310bf833564b1585120a3a74b748a9c1f4b
Merge: facba33 a3f9f2b
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Wed Apr 14 20:58:56 2021 +0300
Merge branch 'master' into NLPCRAFT-287
# Conflicts:
# nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/compiler/antlr4/NCIdlBaseListener.java
# nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/compiler/antlr4/NCIdlLexer.java
# nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/compiler/antlr4/NCIdlListener.java
# nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/compiler/antlr4/NCIdlParser.java
.../model/intent/compiler/NCIdlCompilerBase.scala | 8 +-
.../nlpcraft/model/intent/compiler/antlr4/NCIdl.g4 | 4 +-
.../intent/compiler/antlr4/NCIdlBaseListener.java | 2 +-
.../model/intent/compiler/antlr4/NCIdlLexer.interp | 2 +-
.../model/intent/compiler/antlr4/NCIdlLexer.java | 685 +++++++++++----------
.../intent/compiler/antlr4/NCIdlListener.java | 2 +-
.../model/intent/compiler/antlr4/NCIdlParser.java | 2 +-
.../compiler/functions/NCIdlFunctionsStrings.scala | 8 +-
8 files changed, 357 insertions(+), 356 deletions(-)
[incubator-nlpcraft] 02/06: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-287
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 101e2153bfe4ba02de4f42facc1471894271e4e4
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Wed Apr 14 12:08:58 2021 +0300
WIP.
---
.../apache/nlpcraft/probe/mgrs/NCProbeModel.scala | 9 +-
.../nlpcraft/probe/mgrs/NCProbeSynonym.scala | 95 +++++++++++-----------
.../probe/mgrs/deploy/NCDeployManager.scala | 34 ++++----
.../nlpcraft/probe/mgrs/model/NCModelManager.scala | 39 +++++----
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 49 +++++------
.../org/apache/nlpcraft/model/NCIdlSpec.scala | 2 +-
.../org/apache/nlpcraft/model/NCIdlSpec2.scala | 2 +-
.../nlpcraft/model/intent/idl/NCIdlTestSpec.scala | 2 +-
.../intent/idl/compiler/NCIdlCompilerSpec.scala | 8 +-
.../nlpcraft/model/intent/idl/idl_test_model.yaml | 2 +-
10 files changed, 118 insertions(+), 124 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeModel.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeModel.scala
index 1c21cb9..a4e55a6 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeModel.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeModel.scala
@@ -30,7 +30,7 @@ import scala.collection.{Map, Seq}
* @param intents
* @param continuousSynonyms
* @param sparseSynonyms
- * @param dslSynonyms
+ * @param idlSynonyms
* @param exclStopWordsStems
* @param suspWordsStems
* @param elements
@@ -41,14 +41,13 @@ case class NCProbeModel(
intents: Seq[NCIdlIntent],
continuousSynonyms: Map[String /*Element ID*/ , Map[Int /*Synonym length*/ , NCProbeSynonymsWrapper]], // Fast access map.
sparseSynonyms: Map[String /*Element ID*/, Seq[NCProbeSynonym]],
- dslSynonyms: Map[String /*Element ID*/ , Seq[NCProbeSynonym]], // Fast access map.
+ idlSynonyms: Map[String /*Element ID*/ , Seq[NCProbeSynonym]], // Fast access map.
addStopWordsStems: Set[String],
exclStopWordsStems: Set[String],
suspWordsStems: Set[String],
elements: Map[String /*Element ID*/ , NCElement],
samples: Set[(String, Seq[Seq[String]])]
) {
- def hasDslSynonyms(elemId: String): Boolean = dslSynonyms.contains(elemId)
- def hasDslSynonyms: Boolean = dslSynonyms.nonEmpty
- def hasSparseSynonyms: Boolean = sparseSynonyms.nonEmpty
+ def hasIdlSynonyms(elemId: String): Boolean = idlSynonyms.contains(elemId)
+ def hasIdlSynonyms: Boolean = idlSynonyms.nonEmpty
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala
index dc9a1f5..207bc02 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala
@@ -17,10 +17,11 @@
package org.apache.nlpcraft.probe.mgrs
+import org.apache.nlpcraft.common.U
import org.apache.nlpcraft.common.nlp.{NCNlpSentenceToken, NCNlpSentenceTokenBuffer}
import org.apache.nlpcraft.model._
import org.apache.nlpcraft.model.intent.NCIdlContext
-import org.apache.nlpcraft.probe.mgrs.NCProbeSynonym.NCDslContent
+import org.apache.nlpcraft.probe.mgrs.NCProbeSynonym.NCIdlContent
import org.apache.nlpcraft.probe.mgrs.NCProbeSynonymChunkKind._
import scala.collection.mutable
@@ -49,7 +50,8 @@ class NCProbeSynonym(
lazy val isTextOnly: Boolean = forall(_.kind == TEXT)
lazy val regexChunks: Int = count(_.kind == REGEX)
- lazy val dslChunks: Int = count(_.kind == IDL)
+ lazy val idlChunks: Int = count(_.kind == IDL)
+ lazy val hasIdl: Boolean = idlChunks != 0
lazy val isValueSynonym: Boolean = value != null
lazy val stems: String = map(_.wordStem).mkString(" ")
lazy val stemsHash: Int = stems.hashCode
@@ -86,11 +88,18 @@ class NCProbeSynonym(
/**
*
* @param toks
+ * @param isMatch
+ * @param getIndex
+ * @param shouldBeNeighbors
+ * @tparam T
+ * @return
*/
- private def trySparseMatch0[T](toks: Seq[T], isMatch: (T, NCProbeSynonymChunk) ⇒ Boolean, getIndex: T ⇒ Int): Option[Seq[T]] = {
- require(toks != null)
- require(toks.nonEmpty)
-
+ private def sparseMatch0[T](
+ toks: Seq[T],
+ isMatch: (T, NCProbeSynonymChunk) ⇒ Boolean,
+ getIndex: T ⇒ Int,
+ shouldBeNeighbors: Boolean
+ ): Option[Seq[T]] =
if (toks.size >= this.size) {
lazy val res = mutable.ArrayBuffer.empty[T]
lazy val all = mutable.HashSet.empty[T]
@@ -125,14 +134,21 @@ class NCProbeSynonym(
state = -1
}
- if (state != -1 && all.size == res.size)
+ if (
+ state != -1 &&
+ all.size == res.size
+ &&
+ (
+ !shouldBeNeighbors ||
+ U.isIncreased(res.map(getIndex).sorted)
+ )
+ )
Some(res)
else
None
}
else
None
- }
/**
*
@@ -140,7 +156,7 @@ class NCProbeSynonym(
* @param chunk
* @param req
*/
- private def isMatch(tow: NCDslContent, chunk: NCProbeSynonymChunk, req: NCRequest): Boolean = {
+ private def isMatch(tow: NCIdlContent, chunk: NCProbeSynonymChunk, req: NCRequest): Boolean = {
def get0[T](fromToken: NCToken ⇒ T, fromWord: NCNlpSentenceToken ⇒ T): T =
if (tow.isLeft) fromToken(tow.left.get) else fromWord(tow.right.get)
@@ -164,7 +180,7 @@ class NCProbeSynonym(
*/
def isMatch(toks: NCNlpSentenceTokenBuffer): Boolean = {
require(toks != null)
- require(!sparse)
+ require(!sparse && !hasIdl)
if (toks.length == length) {
if (isTextOnly)
@@ -180,29 +196,11 @@ class NCProbeSynonym(
*
* @param toks
*/
- def trySparseMatch(toks: NCNlpSentenceTokenBuffer): Option[Seq[NCNlpSentenceToken]] = {
+ def sparseMatch(toks: NCNlpSentenceTokenBuffer): Option[Seq[NCNlpSentenceToken]] = {
require(toks != null)
- require(sparse, s"Unexpected call on: $this")
-
- trySparseMatch0(
- toks,
- isMatch,
- (t: NCNlpSentenceToken) ⇒ t.startCharIndex
- )
- }
-
- /**
- *
- * @param tows
- * @param req
- */
- def isMatch(tows: Seq[NCDslContent], req: NCRequest): Boolean = {
- require(tows != null)
+ require(sparse && !hasIdl)
- if (tows.length == length && tows.count(_.isLeft) >= dslChunks)
- tows.zip(this).sortBy(p ⇒ getSort(p._2.kind)).forall { case (tow, chunk) ⇒ isMatch(tow, chunk, req) }
- else
- false
+ sparseMatch0(toks, isMatch, (t: NCNlpSentenceToken) ⇒ t.startCharIndex, shouldBeNeighbors = false)
}
/**
@@ -210,15 +208,16 @@ class NCProbeSynonym(
* @param tows
* @param req
*/
- def trySparseMatch(tows: Seq[NCDslContent], req: NCRequest): Option[Seq[NCDslContent]] = {
+ def idlMatch(tows: Seq[NCIdlContent], req: NCRequest): Option[Seq[NCIdlContent]] = {
require(tows != null)
require(req != null)
- require(sparse, s"Unexpected call on: $this")
+ require(hasIdl)
- trySparseMatch0(
+ sparseMatch0(
tows,
- (t: NCDslContent, chunk: NCProbeSynonymChunk) ⇒ isMatch(t, chunk, req),
- (t: NCDslContent) ⇒ if (t.isLeft) t.left.get.getStartCharIndex else t.right.get.startCharIndex
+ (t: NCIdlContent, chunk: NCProbeSynonymChunk) ⇒ isMatch(t, chunk, req),
+ (t: NCIdlContent) ⇒ if (t.isLeft) t.left.get.getStartCharIndex else t.right.get.startCharIndex,
+ shouldBeNeighbors = !sparse
)
}
@@ -226,7 +225,7 @@ class NCProbeSynonym(
// Orders synonyms from least to most significant.
override def compare(that: NCProbeSynonym): Int = {
- require(sparse == that.sparse, s"Invalid comparing [this=$this, that=$that]")
+ require(hasIdl || sparse == that.sparse, s"Invalid comparing [this=$this, that=$that]")
def compareIsValueSynonym(): Int =
isValueSynonym match {
@@ -235,7 +234,7 @@ class NCProbeSynonym(
case _ ⇒ 0
}
-
+
if (that == null)
1
else
@@ -264,9 +263,9 @@ class NCProbeSynonym(
case false if that.isTextOnly ⇒ -1
case true if that.isTextOnly ⇒ compareIsValueSynonym()
case _ ⇒
- val thisDynCnt = regexChunks + dslChunks
- val thatDynCnt = that.regexChunks + that.dslChunks
-
+ val thisDynCnt = regexChunks + idlChunks
+ val thatDynCnt = that.regexChunks + that.idlChunks
+
// Less PoS/regex/IDL chunks means less uncertainty, i.e. larger weight.
if (thisDynCnt < thatDynCnt)
1
@@ -278,41 +277,41 @@ class NCProbeSynonym(
}
}
}
-
+
override def canEqual(other: Any): Boolean = other.isInstanceOf[NCProbeSynonym]
-
+
override def equals(other: Any): Boolean = other match {
case that: NCProbeSynonym ⇒
super.equals(that) &&
(that canEqual this) &&
isTextOnly == that.isTextOnly &&
regexChunks == that.regexChunks &&
- dslChunks == that.dslChunks &&
+ idlChunks == that.idlChunks &&
isValueSynonym == that.isValueSynonym &&
isElementId == that.isElementId &&
isValueName == that.isValueName &&
value == that.value
case _ ⇒ false
}
-
+
override def hashCode(): Int = {
val state = Seq(
super.hashCode(),
isTextOnly,
regexChunks,
- dslChunks,
+ idlChunks,
isValueSynonym,
isElementId,
isValueName,
value
)
-
+
state.map(p ⇒ if (p == null) 0 else p.hashCode()).foldLeft(0)((a, b) ⇒ 31 * a + b)
}
}
object NCProbeSynonym {
- type NCDslContent = Either[NCToken, NCNlpSentenceToken]
+ type NCIdlContent = Either[NCToken, NCNlpSentenceToken]
/**
*
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
index aeda6c7..b7db8dc 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
@@ -195,10 +195,8 @@ object NCDeployManager extends NCService with DecorateAsScala {
val syns = mutable.HashSet.empty[SynonymHolder]
def ok(b: Boolean, exp: Boolean): Boolean = if (exp) b else !b
- def filterDsl(syns: Set[SynonymHolder], dsl: Boolean): Set[SynonymHolder] =
- syns.filter(s ⇒ ok(s.syn.exists(_.kind == IDL), dsl))
- def filterSparse(syns: Set[SynonymHolder], sparse: Boolean): Set[SynonymHolder] =
- syns.filter(s ⇒ ok(s.syn.sparse, sparse))
+ def idl(syns: Set[SynonymHolder], idl: Boolean): Set[SynonymHolder] = syns.filter(s ⇒ ok(s.syn.hasIdl, idl))
+ def sparse(syns: Set[SynonymHolder], sp: Boolean): Set[SynonymHolder] = syns.filter(s ⇒ ok(s.syn.sparse, sp))
var cnt = 0
val maxCnt = mdl.getMaxTotalSynonyms
@@ -219,8 +217,8 @@ object NCDeployManager extends NCService with DecorateAsScala {
s"]"
)
- val sparseFlag = elm.isSparse.orElse(mdl.isSparse)
- val permuteFlag = elm.isPermutateSynonyms.orElse(mdl.isPermutateSynonyms)
+ val sparseElem = elm.isSparse.orElse(mdl.isSparse)
+ val permuteElem = elm.isPermutateSynonyms.orElse(mdl.isPermutateSynonyms)
def addSynonym(
isElementId: Boolean,
@@ -264,19 +262,19 @@ object NCDeployManager extends NCService with DecorateAsScala {
)
}
- def hasDsl(chunks: Seq[NCProbeSynonymChunk]) = chunks.exists(_.kind == IDL)
+ val sp = sparseElem && chunks.size > 1
if (
- permuteFlag &&
- !sparseFlag &&
+ permuteElem &&
+ !sparseElem &&
!isElementId &&
chunks.forall(_.wordStem != null)
)
simplePermute(chunks).map(p ⇒ p.map(_.wordStem) → p).toMap.values.foreach(seq ⇒
- add(seq, isDirect = seq == chunks, perm = true, sparse = hasDsl(seq))
+ add(seq, isDirect = seq == chunks, perm = true, sparse = sp)
)
else
- add(chunks, isDirect = true, perm = permuteFlag, sparse = hasDsl(chunks) || (sparseFlag && chunks.size > 1))
+ add(chunks, isDirect = true, perm = permuteElem, sparse = sp)
}
/**
@@ -483,7 +481,7 @@ object NCDeployManager extends NCService with DecorateAsScala {
// Scan for intent annotations in the model class.
val intents = scanIntents(mdl)
-
+
var solver: NCIntentSolver = null
if (intents.nonEmpty) {
@@ -508,15 +506,15 @@ object NCDeployManager extends NCService with DecorateAsScala {
def toMap(set: Set[SynonymHolder]): Map[String, Seq[NCProbeSynonym]] =
set.groupBy(_.elmId).map(p ⇒ p._1 → p._2.map(_.syn).toSeq.sorted.reverse)
- val notDsl = filterDsl(syns.toSet, dsl = false)
+ val simple = idl(syns.toSet, idl = false)
NCProbeModel(
model = mdl,
solver = solver,
intents = intents.map(_._1).toSeq,
- continuousSynonyms = mkFastAccessMap(filterSparse(notDsl, sparse = false), NCProbeSynonymsWrapper(_)),
- sparseSynonyms = toMap(filterSparse(notDsl, sparse = true)),
- dslSynonyms = toMap(filterDsl(syns.toSet, dsl = true)),
+ continuousSynonyms = mkFastAccessMap(sparse(simple, sp = false), NCProbeSynonymsWrapper(_)),
+ sparseSynonyms = toMap(sparse(simple, sp = true)),
+ idlSynonyms = toMap(idl(syns.toSet, idl = true)),
addStopWordsStems = addStopWords,
exclStopWordsStems = exclStopWords,
suspWordsStems = suspWords,
@@ -1004,8 +1002,8 @@ object NCDeployManager extends NCService with DecorateAsScala {
}
// IDL-based synonym.
else if (startsAndEnds(IDL_FIX, chunk)) {
- val dsl = stripSuffix(IDL_FIX, chunk)
- val compUnit = NCIdlCompiler.compileSynonym(dsl, mdl, mdl.getOrigin)
+ val idl = stripSuffix(IDL_FIX, chunk)
+ val compUnit = NCIdlCompiler.compileSynonym(idl, mdl, mdl.getOrigin)
val x = NCProbeSynonymChunk(alias = compUnit.alias.orNull, kind = IDL, origText = chunk, idlPred = compUnit.pred)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
index 18009f8..2b8313c 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/model/NCModelManager.scala
@@ -59,30 +59,33 @@ object NCModelManager extends NCService with DecorateAsScala {
val mdl = w.model
// TODO:
- println("w.directSynonyms="+w.continuousSynonyms.getOrElse("col:orders_order_date", Map.empty).mkString("\n"))
- println("w.sparseSynonyms="+w.sparseSynonyms.getOrElse("col:orders_order_date", Seq.empty).mkString("\n"))
- println("w.synonymsDsl="+w.dslSynonyms.getOrElse("col:orders_order_date", Seq.empty).mkString("\n"))
+ val elemId = "col:orders_order_date"
+
+ println("w.directSynonyms="+w.continuousSynonyms.getOrElse(elemId, Map.empty).mkString("\n"))
+ println("w.sparseSynonyms="+w.sparseSynonyms.getOrElse(elemId, Seq.empty).mkString("\n"))
+ println("w.idlSynonyms="+w.idlSynonyms.getOrElse(elemId, Seq.empty).mkString("\n"))
println
- val synContCnt = w.continuousSynonyms.flatMap(_._2.map(_._2.count)).sum
- val synSparseCnt = w.sparseSynonyms.map(_._2.size).sum
- val synDslCnt = w.dslSynonyms.map(_._2.size).sum
- val elmCnt = w.elements.keySet.size
- val intentCnt = w.intents.size
+ val contCnt = w.continuousSynonyms.flatMap(_._2.map(_._2.count)).sum
+ val sparseCnt = w.sparseSynonyms.map(_._2.size).sum
+ val allIdlSyns = w.idlSynonyms.values.flatten
+ val sparseIdlCnt = allIdlSyns.count(_.sparse)
+ val contIdlCnt = allIdlSyns.size - sparseIdlCnt
def withWarn(i: Int): String = if (i == 0) s"0 ${r("(!)")}" else i.toString
tbl += Seq(
- s"Name: ${bo(c(mdl.getName))}",
- s"ID: ${bo(mdl.getId)}",
- s"Version: ${mdl.getVersion}",
- s"Origin: ${mdl.getOrigin}",
- s"Elements: ${withWarn(elmCnt)}",
- s"Synonyms: ${withWarn(elmCnt)}",
- s" - Continuous: $synContCnt",
- s" - Sparse: $synSparseCnt",
- s" - DSL(Sparse): $synDslCnt",
- s"Intents: ${withWarn(intentCnt)}"
+ s"Name: ${bo(c(mdl.getName))}",
+ s"ID: ${bo(mdl.getId)}",
+ s"Version: ${mdl.getVersion}",
+ s"Origin: ${mdl.getOrigin}",
+ s"Elements: ${withWarn(w.elements.keySet.size)}",
+ s"Synonyms:",
+ s" - Simple, continuous: $contCnt",
+ s" - Simple, sparse: $sparseCnt",
+ s" - IDL, continuous: $contIdlCnt",
+ s" - IDL, Sparse: $sparseIdlCnt",
+ s"Intents: ${withWarn(w.intents.size)}"
)
})
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
index 713132c..29ec156 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
@@ -21,7 +21,7 @@ import io.opencensus.trace.Span
import org.apache.nlpcraft.common._
import org.apache.nlpcraft.common.nlp.{NCNlpSentenceToken ⇒ NlpToken, _}
import org.apache.nlpcraft.model._
-import org.apache.nlpcraft.probe.mgrs.NCProbeSynonym.NCDslContent
+import org.apache.nlpcraft.probe.mgrs.NCProbeSynonym.NCIdlContent
import org.apache.nlpcraft.probe.mgrs.NCProbeSynonymChunkKind.{NCSynonymChunkKind, _}
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher
import org.apache.nlpcraft.probe.mgrs.nlp.impl.NCRequestImpl
@@ -73,7 +73,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
}
case class Complex(
- data: NCDslContent,
+ data: NCIdlContent,
isToken: Boolean,
isWord: Boolean,
token: NCToken,
@@ -132,7 +132,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
object State extends Enumeration {
type State = Value
- val SIMPLE, DSL_FIRST, DSL_NEXT = Value
+ val SIMPLE, IDL_FIRST, IDL_NEXT = Value
}
import State._
@@ -378,7 +378,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
* @param seq
* @param s
*/
- private def toParts(seq: Seq[NCDslContent], s: Synonym): Seq[TokType] =
+ private def toParts(seq: Seq[NCIdlContent], s: Synonym): Seq[TokType] =
seq.zip(s.map(_.kind)).flatMap {
case (complex, kind) ⇒ if (complex.isLeft) Some(complex.left.get → kind) else None
}
@@ -388,7 +388,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
* @param tows
* @param ns
*/
- private def toTokens(tows: Seq[NCDslContent], ns: NCNlpSentence): Seq[NlpToken] =
+ private def toTokens(tows: Seq[NCIdlContent], ns: NCNlpSentence): Seq[NlpToken] =
(
tows.filter(_.isRight).map(_.right.get) ++
tows.filter(_.isLeft).map(_.left.get).
@@ -499,7 +499,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
val contCache =
mutable.HashMap.empty[String, mutable.ArrayBuffer[Seq[Int]]] ++
mdl.elements.keys.map(k ⇒ k → mutable.ArrayBuffer.empty[Seq[Int]])
- lazy val dslCache = mutable.HashSet.empty[Seq[Complex]]
+ lazy val idlCache = mutable.HashSet.empty[Seq[Complex]]
var found = false
@@ -536,7 +536,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
for (toks ← combosToks) {
val tokIdxs = toks.map(_.index)
- lazy val dslCombs: Seq[Seq[Complex]] = mkComplexCombinations(h, toks, dslCache.toSet)
+ lazy val idlCombs: Seq[Seq[Complex]] = mkComplexCombinations(h, toks, idlCache.toSet)
lazy val tokStems = toks.map(_.stem).mkString(" ")
// Attempt to match each element.
@@ -550,8 +550,8 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
// 1. SIMPLE.
val simpleEnabled =
state match {
- case SIMPLE ⇒ !mdl.hasDslSynonyms(elemId)
- case DSL_FIRST ⇒ mdl.hasDslSynonyms(elemId)
+ case SIMPLE ⇒ !mdl.hasIdlSynonyms(elemId)
+ case IDL_FIRST ⇒ mdl.hasIdlSynonyms(elemId)
case _ ⇒ false
}
@@ -587,32 +587,27 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
// 1.2 Sparse.
if (simpleEnabled && !found)
for (s ← get(mdl.sparseSynonyms, elemId) if !found)
- s.trySparseMatch(toks) match {
+ s.sparseMatch(toks) match {
case Some(res) ⇒ add("simple sparse", elm, res, tokIdxs, s)
case None ⇒ // No-op.
}
- // 2. DSL.
- if (state != SIMPLE && mdl.hasDslSynonyms) {
+ // 2. IDL.
+ if (state != SIMPLE && mdl.hasIdlSynonyms) {
found = false
// 2.1 Sparse.
- if (mdl.hasDslSynonyms)
- for (s ← get(mdl.dslSynonyms, elemId); comb ← dslCombs if !found)
- s.trySparseMatch(comb.map(_.data), req) match {
+ if (mdl.hasIdlSynonyms)
+ for (s ← get(mdl.idlSynonyms, elemId); comb ← idlCombs if !found)
+ s.idlMatch(comb.map(_.data), req) match {
case Some(res) ⇒
- add("DSL sparse", elm, toTokens(res, ns), tokIdxs, s, toParts(res, s))
+ val typ = if (s.sparse) "IDL sparse" else "IDL continuous"
- dslCache += comb
+ add(typ, elm, toTokens(res, ns), tokIdxs, s, toParts(res, s))
+
+ idlCache += comb
case None ⇒ // No-op.
}
- // 2.2 Continuous.
- else
- for (s ← get(mdl.dslSynonyms, elemId); comb ← dslCombs if !found)
- if (s.isMatch(comb.map(_.data), req)) {
- add("DSL continuous", elm, toks, tokIdxs, s, toPartsComplex(comb, s))
- dslCache += comb
- }
}
}
}
@@ -628,17 +623,17 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
lazy val h = mkComplexes(mdl, ns)
- execute(mdl, ns, combosToks, if (ns.firstProbePhase) SIMPLE else DSL_NEXT, req, h, parent)
+ execute(mdl, ns, combosToks, if (ns.firstProbePhase) SIMPLE else IDL_NEXT, req, h, parent)
if (ns.firstProbePhase) {
ns.firstProbePhase = false
- execute(mdl, ns, combosToks, DSL_FIRST, req, h, parent)
+ execute(mdl, ns, combosToks, IDL_FIRST, req, h, parent)
}
processParsers(mdl, ns, span, req)
}
}
- def isComplex(mdl: NCProbeModel): Boolean = mdl.dslSynonyms.nonEmpty || !mdl.model.getParsers.isEmpty
+ def isComplex(mdl: NCProbeModel): Boolean = mdl.hasIdlSynonyms || !mdl.model.getParsers.isEmpty
}
\ No newline at end of file
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIdlSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIdlSpec.scala
index a02424e..c92108b 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIdlSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIdlSpec.scala
@@ -28,7 +28,7 @@ import scala.language.implicitConversions
* IDL test model.
*/
class NCIdlSpecModel extends NCModelAdapter(
- "nlpcraft.intents.dsl.test", "IDL Test Model", "1.0"
+ "nlpcraft.intents.idl.test", "IDL Test Model", "1.0"
) {
private implicit def convert(s: String): NCResult = NCResult.text(s)
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIdlSpec2.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIdlSpec2.scala
index 31b4808..dd29cca 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIdlSpec2.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIdlSpec2.scala
@@ -27,7 +27,7 @@ import scala.language.implicitConversions
* IDL test model.
*/
class NCIdlSpecModel2 extends NCModelAdapter(
- "nlpcraft.intents.dsl.test", "IDL Test Model", "1.0"
+ "nlpcraft.intents.idl.test", "IDL Test Model", "1.0"
) {
override def getElements: util.Set[NCElement] = Set(NCTestElement("a"))
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/NCIdlTestSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/NCIdlTestSpec.scala
index 424da4a..75b45e2 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/NCIdlTestSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/NCIdlTestSpec.scala
@@ -40,7 +40,7 @@ class NCIdlTestSpec {
if (NCEmbeddedProbe.start(null, Collections.singletonList(classOf[NCIdlTestModel].getName))) {
cli = new NCTestClientBuilder().newBuilder.build
- cli.open("nlpcraft.dsl.test")
+ cli.open("nlpcraft.idl.test")
}
}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/compiler/NCIdlCompilerSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/compiler/NCIdlCompilerSpec.scala
index 0f09a66..6b5da23 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/compiler/NCIdlCompilerSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/compiler/NCIdlCompilerSpec.scala
@@ -37,16 +37,16 @@ class NCIdlCompilerSpec {
/**
*
- * @param dsl
+ * @param idl
*/
- private def checkCompileOk(dsl: String): Unit =
+ private def checkCompileOk(idl: String): Unit =
try {
- NCIdlCompiler.compileIntents(dsl, MODEL, MODEL_ID)
+ NCIdlCompiler.compileIntents(idl, MODEL, MODEL_ID)
assert(true)
}
catch {
- case e: Exception ⇒ assert(false, e)
+ case e: Exception ⇒ assert(assertion = false, e)
}
/**
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/idl_test_model.yaml b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/idl_test_model.yaml
index b436110..a9b70f1 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/idl_test_model.yaml
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/idl_test_model.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-id: "nlpcraft.dsl.test"
+id: "nlpcraft.idl.test"
name: "IDL Test Model"
version: "1.0"
description: "IDL test model."