You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2021/03/04 13:14:53 UTC
[incubator-nlpcraft] branch master updated: Parts detection fixed.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/master by this push:
new 60bedc3 Parts detection fixed.
60bedc3 is described below
commit 60bedc363851814310d251e29de4bbe45f0e3128
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Thu Mar 4 16:14:38 2021 +0300
Parts detection fixed.
---
.../nlpcraft/probe/mgrs/NCProbeSynonym.scala | 24 ++++-----
.../probe/mgrs/deploy/NCDeployManager.scala | 2 +-
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 59 +++++++++++-----------
.../abstract/NCAbstractTokensVariantsSpec.scala | 56 +++++++++++---------
.../model/properties/NCTokensPropertiesSpec.scala | 4 +-
5 files changed, 76 insertions(+), 69 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala
index 6ed8f44..b8b7dc6 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeSynonym.scala
@@ -69,11 +69,10 @@ class NCProbeSynonym(
def isMatch(toks: NCNlpSentenceTokenBuffer): Boolean = {
require(toks != null)
- val ok =
+ if (toks.length == length) {
if (isTextOnly)
toks.stemsHash == stemsHash && toks.stems == stems
else
- // Same length.
toks.zip(this).sortBy(p ⇒ getSort(p._2.kind)).forall {
case (tok, chunk) ⇒
chunk.kind match {
@@ -83,9 +82,9 @@ class NCProbeSynonym(
case _ ⇒ throw new AssertionError()
}
}
-
- // Should be called only for valid tokens count (validation optimized for performance reasons)
- ok && toks.length == length
+ }
+ else
+ false
}
/**
@@ -100,27 +99,26 @@ class NCProbeSynonym(
type Word = NCNlpSentenceToken
type TokenOrWord = Either[Token, Word]
- val ok =
- // Same length.
+ if (tows.length == length && tows.count(_.isLeft) >= dslChunks)
tows.zip(this).sortBy(p ⇒ getSort(p._2.kind)).forall {
case (tow, chunk) ⇒
def get0[T](fromToken: Token ⇒ T, fromWord: Word ⇒ T): T =
if (tow.isLeft) fromToken(tow.left.get) else fromWord(tow.right.get)
chunk.kind match {
- case TEXT ⇒ chunk.wordStem == get0((t: Token) ⇒ t.stem, (w: Word) ⇒ w.stem)
+ case TEXT ⇒ chunk.wordStem == get0(_.stem, _.stem)
case REGEX ⇒
val r = chunk.regex
- r.matcher(get0((t: Token) ⇒ t.origText, (w: Word) ⇒ w.origText)).matches() ||
- r.matcher(get0((t: Token) ⇒ t.normText, (w: Word) ⇒ w.normText)).matches()
- case DSL ⇒ get0((t: Token) ⇒ chunk.dslPred.apply(t), (_: Word) ⇒ false)
+ r.matcher(get0(_.origText, _.origText)).matches() ||
+ r.matcher(get0(_.normText, _.normText)).matches()
+ case DSL ⇒ get0(t ⇒ chunk.dslPred.apply(t), _ ⇒ false)
case _ ⇒ throw new AssertionError()
}
}
- // Should be called only for valid tokens count (validation optimized for performance reasons)
- ok && tows.length == length
+ else
+ false
}
override def toString(): String = mkString(" ")
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
index a68e305..21eaaab 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
@@ -503,7 +503,7 @@ object NCDeployManager extends NCService with DecorateAsScala {
solver = solver,
intents = intents.keySet.toSeq,
synonyms = mkFastAccessMap(filter(syns, dsl = false), NCProbeSynonymsWrapper(_)),
- synonymsDsl = mkFastAccessMap(filter(syns, dsl = true), seq ⇒ seq),
+ synonymsDsl = mkFastAccessMap(filter(syns, dsl = true), _.sorted.reverse),
addStopWordsStems = addStopWords.toSet,
exclStopWordsStems = exclStopWords.toSet,
suspWordsStems = suspWords.toSet,
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
index c4a7936..2a9dec0 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
@@ -17,21 +17,22 @@
package org.apache.nlpcraft.probe.mgrs.nlp.enrichers.model
-import java.io.Serializable
-import java.util
import io.opencensus.trace.Span
import org.apache.nlpcraft.common._
import org.apache.nlpcraft.common.nlp.{NCNlpSentenceToken, NCNlpSentenceTokenBuffer, _}
import org.apache.nlpcraft.model._
+import org.apache.nlpcraft.probe.mgrs.NCProbeSynonymChunkKind.{NCSynonymChunkKind, TEXT}
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher
import org.apache.nlpcraft.probe.mgrs.nlp.impl.NCRequestImpl
-import org.apache.nlpcraft.probe.mgrs.{NCProbeModel, NCProbeSynonym, NCProbeSynonymsWrapper, NCProbeVariants}
+import org.apache.nlpcraft.probe.mgrs.{NCProbeModel, NCProbeSynonym, NCProbeVariants}
+import java.io.Serializable
+import java.util
import scala.collection.JavaConverters._
-import scala.compat.java8.OptionConverters._
import scala.collection.convert.DecorateAsScala
import scala.collection.mutable.ArrayBuffer
import scala.collection.{Map, Seq, mutable}
+import scala.compat.java8.OptionConverters._
/**
* Model elements enricher.
@@ -63,7 +64,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
element: NCElement,
tokens: Seq[NCNlpSentenceToken],
synonym: NCProbeSynonym,
- parts: Seq[NCToken]
+ parts: Seq[(NCToken, NCSynonymChunkKind)]
) extends Ordered[ElementMatch] {
// Tokens sparsity.
lazy val sparsity: Int = tokens.zipWithIndex.tail.map {
@@ -198,7 +199,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
direct: Boolean,
syn: Option[NCProbeSynonym],
metaOpt: Option[Map[String, Object]],
- parts: Seq[NCToken]
+ parts: Seq[(NCToken, NCSynonymChunkKind)]
): Unit = {
val params = mutable.ArrayBuffer.empty[(String, AnyRef)]
@@ -219,16 +220,16 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
if (parts.nonEmpty) {
val partsData: Seq[util.HashMap[String, Any]] =
- parts.map(part ⇒ {
+ parts.map { case (part, kind) ⇒
val m = new util.HashMap[String, Any]()
- m.put("id", part.getId)
+ m.put("id", if (kind == TEXT) "nlpcraft:nlp" else part.getId)
m.put("startcharindex", part.getStartCharIndex)
m.put("endcharindex", part.getEndCharIndex)
m.put(TOK_META_ALIASES_KEY, part.getMetadata.get(TOK_META_ALIASES_KEY))
m
- })
+ }
params += "parts" → partsData.asJava
}
@@ -262,7 +263,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
* @param toks
* @return
*/
- protected def combos[T](toks: Seq[T]): Seq[Seq[T]] =
+ private def combos[T](toks: Seq[T]): Seq[Seq[T]] =
(for (n ← toks.size until 0 by -1) yield toks.sliding(n)).flatten.map(p ⇒ p)
/**
@@ -295,7 +296,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
varToks.flatMap(t ⇒
// Single word token is not split as words - token.
// Partly (not strict in) token - word.
- if ((toksComb.contains(t) || isSingleWord(t)) && inStrict(t))
+ if (inStrict(t) && (toksComb.contains(t) || isSingleWord(t)))
Seq(Complex(Left(t)))
else
t.wordIndexes.filter(nlpWordIdxs.contains).map(i ⇒ Complex(Right(initialSen(i))))
@@ -354,7 +355,7 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
toks.map(t ⇒ (t.origText, t.index)).mkString(" ")
var permCnt = 0
- var collapsedSens: Seq[Seq[NCToken]] = null
+ lazy val collapsedSens = NCProbeVariants.convert(ns.srvReqId, mdl, ns.clone().collapse(mdl.model)).map(_.asScala)
/**
*
@@ -365,17 +366,19 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
for (toks ← combos(perm)) {
val key = toks.map(_.index).sorted
- val sparsity = U.calcSparsity(key)
if (!cache.contains(key)) {
- var seq: Seq[Seq[Complex]] = null
+ cache += key
+
+ lazy val dslCombs = convert(ns, collapsedSens, toks).groupBy(_.length)
+ lazy val sparsity = U.calcSparsity(key)
// Attempt to match each element.
for (elm ← mdl.elements.values if !alreadyMarked(toks, elm.getId)) {
var found = false
def addMatch(
- elm: NCElement, toks: Seq[NCNlpSentenceToken], syn: NCProbeSynonym, parts: Seq[NCToken]
+ elm: NCElement, toks: Seq[NCNlpSentenceToken], syn: NCProbeSynonym, parts: Seq[(NCToken, NCSynonymChunkKind)]
): Unit =
if (
(elm.getJiggleFactor.isEmpty || elm.getJiggleFactor.get() >= sparsity) &&
@@ -425,25 +428,21 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
if (mdl.synonymsDsl.nonEmpty) {
found = false
- if (collapsedSens == null)
- collapsedSens =
- NCProbeVariants.
- convert(ns.srvReqId, mdl, ns.clone().collapse(mdl.model)).map(_.asScala)
-
- if (seq == null)
- seq = convert(ns, collapsedSens, toks)
-
for (
- comb ← seq;
- syn ← fastAccess(mdl.synonymsDsl, elm.getId, comb.length).getOrElse(Seq.empty)
- if !found
+ (len, seq) ← dslCombs;
+ syn ← fastAccess(mdl.synonymsDsl, elm.getId, len).getOrElse(Seq.empty);
+ comb ← seq if !found;
+ data = comb.map(_.data)
)
- if (syn.isMatch(comb.map(_.data)))
- addMatch(elm, toks, syn, comb.filter(_.isToken).map(_.token))
+ if (syn.isMatch(data)) {
+ val parts = comb.zip(syn.map(_.kind)).flatMap {
+ case (complex, kind) ⇒ if (complex.isToken) Some(complex.token → kind) else None
+ }
+
+ addMatch(elm, toks, syn, parts)
+ }
}
}
-
- cache += key
}
}
}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensVariantsSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensVariantsSpec.scala
index 8912bc8..35e8e87 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensVariantsSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensVariantsSpec.scala
@@ -30,6 +30,11 @@ class NCAbstractTokensModelVariants extends NCAbstractTokensModel {
private def checkText(t: NCToken, txt: String): Unit =
require(t.getOriginalText == txt, s"Expected text: $txt, token: $t")
+ private def checkToken(t: NCToken, id: String, txt: String): Unit = {
+ checkId(t, id)
+ checkText(t, txt)
+ }
+
override def onContext(ctx: NCContext): NCResult = {
val variants = ctx.getVariants.asScala
@@ -49,6 +54,18 @@ class NCAbstractTokensModelVariants extends NCAbstractTokensModel {
)
}
+ def checkWrapAnyWord(t: NCToken, any: String): Unit = {
+ val parts = t.getPartTokens.asScala
+
+ require(parts.size == 2)
+
+ checkToken(parts.head, "nlpcraft:nlp", "the")
+ checkToken(parts.last, "anyWord", any)
+
+ require(parts.last.isAbstract, s"Unexpected abstract token: ${parts.last}")
+
+ }
+
ctx.getRequest.getNormalizedText match {
case "word the word" ⇒
require(variants.size == 1)
@@ -57,20 +74,10 @@ class NCAbstractTokensModelVariants extends NCAbstractTokensModel {
require(toks.size == 2)
- checkId(toks.head, "nlpcraft:nlp")
- checkText(toks.head, "word")
-
- checkId(toks.last, "wrapAnyWord")
- checkText(toks.last, "the word")
-
- val t2Parts = toks.last.getPartTokens.asScala
+ checkToken(toks.head, "nlpcraft:nlp", "word")
+ checkToken(toks.last, "wrapAnyWord", "the word")
- require(t2Parts.size == 2)
-
- checkId(t2Parts.head,"anyWord")
- checkId(t2Parts.last, "anyWord")
-
- t2Parts.foreach(t ⇒ require(t.isAbstract, s"Unexpected abstract token: $t"))
+ checkWrapAnyWord(toks.last, "word")
case "10 w1 10 w2" ⇒
require(variants.nonEmpty)
@@ -85,16 +92,16 @@ class NCAbstractTokensModelVariants extends NCAbstractTokensModel {
require(toks.size == 2)
- checkText(toks.head, "10")
- checkText(toks.last,"w1 10 w2")
+ checkToken(toks.head, "nlpcraft:nlp", "10")
+ checkToken(toks.last,"wrapNum", "w1 10 w2")
val t2Parts = toks.last.getPartTokens.asScala
require(t2Parts.size == 3)
- checkId(t2Parts.head,"nlpcraft:nlp")
- checkId(t2Parts(1),"nlpcraft:num")
- checkId(t2Parts.last,"nlpcraft:nlp")
+ checkToken(t2Parts.head,"nlpcraft:nlp", "w1")
+ checkToken(t2Parts(1),"nlpcraft:num", "10")
+ checkToken(t2Parts.last,"nlpcraft:nlp", "w2")
case "before limit top 6 the any" ⇒
require(variants.nonEmpty)
@@ -109,8 +116,8 @@ class NCAbstractTokensModelVariants extends NCAbstractTokensModel {
require(toks.size == 2)
- checkText(toks.head, "before limit top 6")
- checkText(toks.last,"the any")
+ checkToken(toks.head, "wrapLimit", "before limit top 6")
+ checkToken(toks.last, "wrapAnyWord", "the any")
val wrap = toks.head.getPartTokens.asScala
@@ -118,6 +125,7 @@ class NCAbstractTokensModelVariants extends NCAbstractTokensModel {
checkLimit(wrap.last)
+ checkWrapAnyWord(toks.last, "any")
case "a wrap before limit top 6 the any" ⇒
require(variants.nonEmpty)
@@ -131,9 +139,9 @@ class NCAbstractTokensModelVariants extends NCAbstractTokensModel {
require(toks.size == 3)
- checkText(toks.head, "a")
- checkText(toks(1), "wrap before limit top 6")
- checkText(toks.last,"the any")
+ checkToken(toks.head, "nlpcraft:nlp", "a")
+ checkToken(toks(1), "wrapWrapLimit", "wrap before limit top 6")
+ checkToken(toks.last, "wrapAnyWord", "the any")
val wrap = toks(1).getPartTokens.asScala
@@ -147,6 +155,8 @@ class NCAbstractTokensModelVariants extends NCAbstractTokensModel {
require(wrapLimit.getPartTokens.size == 3, s"Parts count: ${wrapLimit.getPartTokens.size()}")
checkLimit(wrapLimit.getPartTokens.asScala.last)
+
+ checkWrapAnyWord(toks.last, "any")
case _ ⇒ throw new AssertionError(s"Unexpected request: ${ctx.getRequest.getNormalizedText}")
}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/properties/NCTokensPropertiesSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/properties/NCTokensPropertiesSpec.scala
index 4bfd9b8..20bbb56 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/properties/NCTokensPropertiesSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/properties/NCTokensPropertiesSpec.scala
@@ -242,7 +242,7 @@ class NCTokenPropertiesModel7 extends NCTokenPropertiesModelAbstract {
}
@NCTestEnvironment(model = classOf[NCTokenPropertiesModel7], startClient = true)
-class NCTokenPropertiesModel6Spe7 extends NCTestContext {
+class NCTokenPropertiesModel7Spec extends NCTestContext {
@Test
def test(): Unit = {
checkIntent("a b", "onAB")
@@ -281,7 +281,7 @@ class NCTokenPropertiesModel8 extends NCTokenPropertiesModelAbstract {
}
@NCTestEnvironment(model = classOf[NCTokenPropertiesModel8], startClient = true)
-class NCTokenPropertiesModel6Spe8 extends NCTestContext {
+class NCTokenPropertiesModel8Spec extends NCTestContext {
@Test
def test(): Unit = {
checkIntent("a b", "onAB")