You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2021/02/11 15:09:30 UTC
[incubator-nlpcraft] branch NLPCRAFT-236 created (now 6ec34b3)
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a change to branch NLPCRAFT-236
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
at 6ec34b3 WIP.
This branch includes the following new commits:
new 6ec34b3 WIP.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-nlpcraft] 01/01: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-236
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 6ec34b32dcf8cfd98ab97fb12fdde3cdfadf759f
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Thu Feb 11 18:08:39 2021 +0300
WIP.
---
.../apache/nlpcraft/common/nlp/NCNlpSentence.scala | 88 +++++++++++++++-------
.../examples/misc/geo/keycdn/GeoManager.java | 4 +-
.../apache/nlpcraft/model/NCModelFileAdapter.java | 7 ++
.../org/apache/nlpcraft/model/NCModelView.java | 5 ++
.../nlpcraft/model/impl/json/NCModelJson.java | 7 ++
.../nlpcraft/probe/mgrs/NCProbeVariants.scala | 45 +++++++++--
.../probe/mgrs/deploy/NCDeployManager.scala | 17 ++++-
.../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 4 +-
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 4 +-
.../scala/org/apache/nlpcraft/NCTestContext.scala | 31 +++++++-
.../nlpcraft/examples/time/NCTimeModelSpec.scala | 24 ++----
.../apache/nlpcraft/model/NCIntentDslSpec.scala | 14 +---
.../apache/nlpcraft/model/NCIntentDslSpec2.scala | 19 ++---
.../abstract/NCAbstractTokensEnricherSpec.scala | 48 ++++++++++++
.../abstract/NCAbstractTokensIntentsSpec.scala | 44 +++++++++++
.../model/abstract/NCAbstractTokensModel.scala | 49 ++++++++++++
.../nlpcraft/models/stm/NCStmTestModelSpec.scala | 23 ++----
.../mgrs/nlp/enrichers/NCDefaultTestModel.scala | 14 +---
.../mgrs/nlp/enrichers/NCEnricherBaseSpec.scala | 17 ++++-
.../nlp/enrichers/NCEnrichersTestContext.scala | 32 ++++++++
20 files changed, 383 insertions(+), 113 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentence.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentence.scala
index d1aeb60..0d65199 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentence.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentence.scala
@@ -19,6 +19,7 @@ package org.apache.nlpcraft.common.nlp
import org.apache.nlpcraft.common.NCE
import org.apache.nlpcraft.common.nlp.pos.NCPennTreebank
+import org.apache.nlpcraft.model.NCModel
import java.util
import java.util.Collections
@@ -542,6 +543,36 @@ class NCNlpSentence(
hash = null
}
+ private def dropAbstract(mdl: NCModel, ns: NCNlpSentence): Unit = {
+ val notes = ns.flatten
+
+ case class Key(id: String, start: Int, end: Int) {
+ private def in(i: Int): Boolean = i >= start && i <= end
+ def intersect(id: String, start: Int, end: Int): Boolean = in(start) || in(end)
+ }
+
+ val keys: Seq[Key] =
+ notes.filter(_.isUser).flatMap(n ⇒ {
+ val optList: Option[util.List[util.HashMap[String, Serializable]]] = n.dataOpt("parts")
+
+ optList
+ }).flatMap(_.asScala).map(map ⇒ Key(
+ map.get("id").asInstanceOf[String],
+ map.get("startcharindex").asInstanceOf[Int],
+ map.get("endcharindex").asInstanceOf[Int])
+ ).distinct
+
+ notes.filter(n ⇒ {
+ val noteToks = ns.tokens.filter(_.contains(n))
+
+ mdl.getAbstractTokens.contains(n.noteType) &&
+ !keys.exists(_.intersect(n.noteType, noteToks.head.startCharIndex, noteToks.last.startCharIndex))
+ }).foreach(ns.removeNote)
+ }
+
+ private def getNotNlpNotes(toks: Seq[NCNlpSentenceToken]): Seq[NCNlpSentenceNote] =
+ toks.flatten.filter(!_.isNlp).distinct
+
/**
* This collapser handles several tasks:
* - "overall" collapsing after all other individual collapsers had their turn.
@@ -549,42 +580,38 @@ class NCNlpSentence(
*
* In all cases of overlap (full or partial) - the "longest" note wins. In case of overlap and equal
* lengths - the winning note is chosen based on this priority.
- *
*/
@throws[NCE]
- def collapse(): Seq[NCNlpSentence] = {
+ def collapse(mdl: NCModel, lastPhase: Boolean = false): Seq[NCNlpSentence] = {
// Always deletes `similar` notes.
// Some words with same note type can be detected various ways.
// We keep only one variant - with `best` direct and sparsity parameters,
// other variants for these words are redundant.
val redundant: Seq[NCNlpSentenceNote] =
- this.flatten.filter(!_.isNlp).distinct.
- groupBy(_.getKey()).
- map(p ⇒ p._2.sortBy(p ⇒
- (
- // System notes don't have such flags.
- if (p.isUser) {
- if (p.isDirect)
- 0
+ this.flatten.filter(!_.isNlp).distinct.
+ groupBy(_.getKey()).
+ map(p ⇒ p._2.sortBy(p ⇒
+ (
+ // System notes don't have such flags.
+ if (p.isUser) {
+ if (p.isDirect)
+ 0
+ else
+ 1
+ }
else
- 1
- }
- else
- 0,
- if (p.isUser)
- p.sparsity
- else
- 0
- )
- )).
- flatMap(_.drop(1)).
- toSeq
+ 0,
+ if (p.isUser)
+ p.sparsity
+ else
+ 0
+ )
+ )).
+ flatMap(_.drop(1)).
+ toSeq
redundant.foreach(this.removeNote)
- def getNotNlpNotes(toks: Seq[NCNlpSentenceToken]): Seq[NCNlpSentenceNote] =
- toks.flatten.filter(!_.isNlp).distinct
-
val delCombs: Seq[NCNlpSentenceNote] =
getNotNlpNotes(this).
flatMap(note ⇒ getNotNlpNotes(this.slice(note.tokenFrom, note.tokenTo + 1)).filter(_ != note)).
@@ -621,9 +648,13 @@ class NCNlpSentence(
deleted += delComb
- val notNlpTypes = getNotNlpNotes(nsClone).map(_.noteType).distinct
+ if (lastPhase)
+ dropAbstract(mdl, nsClone)
- if (collapseSentence(nsClone, notNlpTypes)) Some(nsClone) else None
+ if (collapseSentence(nsClone, getNotNlpNotes(nsClone).map(_.noteType).distinct))
+ Some(nsClone)
+ else
+ None
}
else
None
@@ -667,6 +698,9 @@ class NCNlpSentence(
m.values.map(_.sentence).toSeq
}
else {
+ if (lastPhase)
+ dropAbstract(mdl, this)
+
if (collapseSentence(this, getNotNlpNotes(this).map(_.noteType).distinct))
Seq(this)
else
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/misc/geo/keycdn/GeoManager.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/misc/geo/keycdn/GeoManager.java
index e254c04..75cc316 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/misc/geo/keycdn/GeoManager.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/misc/geo/keycdn/GeoManager.java
@@ -59,7 +59,7 @@ public class GeoManager {
* @return Geo data. Optional.
*/
public Optional<GeoDataBean> get(NCRequest sen) {
- if (!sen.getRemoteAddress().isPresent()) {
+ if (sen.getRemoteAddress().isEmpty()) {
System.err.println("Geo data can't be found because remote address is not available in the sentence.");
return Optional.empty();
@@ -91,7 +91,7 @@ public class GeoManager {
HttpURLConnection conn = (HttpURLConnection)(new URL(URL + host).openConnection());
// This service requires "User-Agent" property for some reasons.
- conn.setRequestProperty("User-Agent", "rest");
+ conn.setRequestProperty("User-Agent", "keycdn-tools:https://nlpcraft.apache.org");
try (InputStream in = conn.getInputStream()) {
String enc = conn.getContentEncoding();
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelFileAdapter.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelFileAdapter.java
index c607677..883dfac 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelFileAdapter.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelFileAdapter.java
@@ -57,6 +57,7 @@ abstract public class NCModelFileAdapter extends NCModelAdapter {
private final NCModelJson proxy;
private final Set<String> suspWords;
private final Set<String> enabledToks;
+ private final Set<String> abstractToks;
private final Set<String> addStopwords;
private final Set<String> exclStopwords;
private final Set<String> intents;
@@ -112,6 +113,7 @@ abstract public class NCModelFileAdapter extends NCModelAdapter {
this.proxy = proxy;
this.suspWords = convert(proxy.getSuspiciousWords(), null);
this.enabledToks = convert(proxy.getEnabledBuiltInTokens(), NCModelView.DFLT_ENABLED_BUILTIN_TOKENS);
+ this.abstractToks = convert(proxy.getAbstractTokens(), Collections.emptySet());
this.addStopwords = convert(proxy.getAdditionalStopWords(), null);
this.exclStopwords = convert(proxy.getExcludedStopWords(), null);
this.elems = convertElements(proxy.getElements());
@@ -487,6 +489,11 @@ abstract public class NCModelFileAdapter extends NCModelAdapter {
}
@Override
+ public Set<String> getAbstractTokens() {
+ return abstractToks;
+ }
+
+ @Override
public List<NCCustomParser> getParsers() {
return parsers;
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelView.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelView.java
index f82a0ce..34426ad 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelView.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCModelView.java
@@ -913,6 +913,11 @@ public interface NCModelView extends NCMetadata {
return DFLT_ENABLED_BUILTIN_TOKENS;
}
+ // TODO:
+ default Set<String> getAbstractTokens() {
+ return Collections.emptySet();
+ }
+
/**
* Gets maximum number of unique synonyms per model element after which either warning or error will be
* triggered. Note that there is no technical limit on how many synonyms a model element can have apart
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCModelJson.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCModelJson.java
index d1e0f90..0ded090 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCModelJson.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/impl/json/NCModelJson.java
@@ -36,6 +36,7 @@ public class NCModelJson {
private String[] excludedStopWords;
private String[] suspiciousWords;
private String[] enabledBuiltInTokens;
+ private String[] abstractTokens;
private String[] intents;
private String[] parsers;
@@ -225,6 +226,12 @@ public class NCModelJson {
return enabledBuiltInTokens;
}
public void setEnabledBuiltInTokens(String[] enabledBuiltInTokens) { this.enabledBuiltInTokens = enabledBuiltInTokens; }
+ public String[] getAbstractTokens() {
+ return abstractTokens;
+ }
+ public void setAbstractTokens(String[] abstractTokens) {
+ this.abstractTokens = abstractTokens;
+ }
public String[] getIntents() {
return intents;
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeVariants.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeVariants.scala
index 351c8cb..caa5a4a 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeVariants.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeVariants.scala
@@ -38,13 +38,14 @@ object NCProbeVariants {
* @param mdl Probe model.
* @param srvReqId Server request ID.
* @param sens Sentences.
+ * @param lastPhase Flag.
*/
- def convert(srvReqId: String, mdl: NCProbeModel, sens: Seq[NCNlpSentence]): Seq[NCVariant] = {
+ def convert(srvReqId: String, mdl: NCProbeModel, sens: Seq[NCNlpSentence], lastPhase: Boolean = false): Seq[NCVariant] = {
val seq = sens.map(_.toSeq.map(nlpTok ⇒ NCTokenImpl(mdl, srvReqId, nlpTok) → nlpTok))
val toks = seq.map(_.map { case (tok, _) ⇒ tok })
case class Key(id: String, from: Int, to: Int)
-
+
val keys2Toks = toks.flatten.map(t ⇒ Key(t.getId, t.getStartCharIndex, t.getEndCharIndex) → t).toMap
val partsKeys = mutable.HashSet.empty[Key]
@@ -67,7 +68,20 @@ object NCProbeVariants {
)
)
- val parts = keys.map(keys2Toks)
+ val parts = keys.map(key ⇒ {
+ keys2Toks.get(key) match {
+ case Some(tok) ⇒ tok
+ case None ⇒
+ val toks =
+ keys2Toks.filter { case (k, _) ⇒ k.from == key.from && k.to == key.to }.values
+
+ require(toks.size == 1, s"Unexpected state [key=$key, tokens=${toks.mkString(",")}]")
+
+ val tok = toks.head
+
+ tok
+ }
+ })
parts.zip(list.asScala).foreach { case (part, map) ⇒
map.get(TOK_META_ALIASES_KEY) match {
@@ -84,13 +98,34 @@ object NCProbeVariants {
}
}
}
-
+
// We can't collapse parts earlier, because we need them here (setParts method, few lines above.)
- toks.filter(sen ⇒
+ var vars = toks.filter(sen ⇒
!sen.exists(t ⇒
t.getId != "nlpcraft:nlp" &&
partsKeys.contains(Key(t.getId, t.getStartCharIndex, t.getEndCharIndex))
)
).map(p ⇒ new NCVariantImpl(p.asJava))
+
+ if (lastPhase) {
+ if (vars.size > 1)
+ vars = vars.filter(v ⇒ !v.asScala.forall(_.getId == "nlpcraft:nlp"))
+//
+// // Assertions.
+// for (v ← vars;
+// toks = v.asScala;
+// (tok, idx) ← toks.filter(_.isUserDefined).zipWithIndex;
+// part ← tok.getPartTokens.asScala
+// ) {
+// require(part.getIndex < toks.size, s"Part has unexpected index [tokens=${toks.mkString(", ")}, token=$idx, part=$part]}")
+//
+// require(
+// toks(part.getIndex).getId == part.getId,
+// s"Part has unexpected ID [tokens=${toks.mkString(", ")}, token=$idx, part=$part]}"
+// )
+// }
+ }
+
+ vars
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
index 1119c27..0c51051 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
@@ -881,13 +881,14 @@ object NCDeployManager extends NCService with DecorateAsScala {
checkCollection("additionalStopWords", mdl.getAdditionalStopWords)
checkCollection("elements", mdl.getElements)
checkCollection("enabledBuiltInTokens", mdl.getEnabledBuiltInTokens)
+ checkCollection("abstractTokens", mdl.getAbstractTokens)
checkCollection("excludedStopWords", mdl.getExcludedStopWords)
checkCollection("parsers", mdl.getParsers)
checkCollection("suspiciousWords", mdl.getSuspiciousWords)
checkCollection("macros", mdl.getMacros)
checkCollection("metadata", mdl.getMetadata)
- val unsToks =
+ val unsToksBlt =
mdl.getEnabledBuiltInTokens.asScala.filter(t ⇒
// 'stanford', 'google', 'opennlp', 'spacy' - any names, not validated.
t == null ||
@@ -896,11 +897,21 @@ object NCDeployManager extends NCService with DecorateAsScala {
(t.startsWith("nlpcraft:") && !NCModelView.DFLT_ENABLED_BUILTIN_TOKENS.contains(t))
)
- if (unsToks.nonEmpty)
+ if (unsToksBlt.nonEmpty)
throw new NCE(s"Invalid token IDs for 'enabledBuiltInTokens' model property [" +
s"mdlId=${mdl.getId}, " +
- s"ids=${unsToks.mkString(", ")}" +
+ s"ids=${unsToksBlt.mkString(", ")}" +
s"]")
+
+ // We can't check other names because they can be created by custom parsers.
+ val unsToksAbstract = mdl.getAbstractTokens.asScala.filter(t ⇒ t == null || t == "nlpcraft:nlp")
+
+ if (unsToksAbstract.nonEmpty)
+ throw new NCE(s"Invalid token IDs for 'abstractToken' model property [" +
+ s"mdlId=${mdl.getId}, " +
+ s"ids=${unsToksAbstract.mkString(", ")}" +
+ s"]"
+ )
}
/**
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
index e304822..ed1b544 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
@@ -484,7 +484,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats {
s"]")
}
- nlpSen.clone().collapse().
+ nlpSen.clone().collapse(mdl.model, lastPhase = true).
// Sorted to support deterministic logs.
sortBy(p ⇒
p.map(p ⇒ {
@@ -525,7 +525,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats {
val meta = mutable.HashMap.empty[String, Any] ++ senMeta
val req = NCRequestImpl(meta, srvReqId)
- var senVars = NCProbeVariants.convert(srvReqId, mdl, sensSeq)
+ var senVars = NCProbeVariants.convert(srvReqId, mdl, sensSeq, lastPhase = true)
// Sentence variants can be filtered by model.
val fltSenVars: Seq[(NCVariant, Int)] =
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
index e2e2265..fe1cac5 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
@@ -387,7 +387,9 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
found = false
if (collapsedSens == null)
- collapsedSens = NCProbeVariants.convert(ns.srvReqId, mdl, ns.clone().collapse()).map(_.asScala)
+ collapsedSens =
+ NCProbeVariants.
+ convert(ns.srvReqId, mdl, ns.clone().collapse(mdl.model)).map(_.asScala)
if (seq == null)
seq = convert(ns, collapsedSens, toks)
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala
index 8bb384b..91d26d3 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala
@@ -20,6 +20,7 @@ package org.apache.nlpcraft
import org.apache.nlpcraft.model.tools.embedded.NCEmbeddedProbe
import org.apache.nlpcraft.model.tools.test.{NCTestClient, NCTestClientBuilder}
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
+import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue}
import org.junit.jupiter.api.TestInstance.Lifecycle
import org.junit.jupiter.api._
@@ -55,10 +56,10 @@ abstract class NCTestContext {
if (getClassAnnotation(info).isDefined)
stop0()
- private def getClassAnnotation(info: TestInfo) =
+ protected def getClassAnnotation(info: TestInfo): Option[NCTestEnvironment] =
if (info.getTestClass.isPresent) Option(info.getTestClass.get().getAnnotation(MDL_CLASS)) else None
- private def getMethodAnnotation(info: TestInfo): Option[NCTestEnvironment] =
+ protected def getMethodAnnotation(info: TestInfo): Option[NCTestEnvironment] =
if (info.getTestMethod.isPresent) Option(info.getTestMethod.get().getAnnotation(MDL_CLASS)) else None
@throws[Exception]
@@ -110,6 +111,32 @@ abstract class NCTestContext {
protected def afterProbeStop(): Unit = { }
+ /**
+ *
+ * @param txt
+ * @param intent
+ */
+ protected def checkIntent(txt: String, intent: String): Unit = {
+ val res = getClient.ask(txt)
+
+ assertTrue(res.isOk, s"Checked: $txt")
+ assertTrue(res.getResult.isPresent, s"Checked: $txt")
+ assertEquals(intent, res.getIntentId, s"Checked: $txt")
+ }
+
+ /**
+ * @param req
+ * @param expResp
+ */
+ protected def checkResult(req: String, expResp: String): Unit = {
+ val res = getClient.ask(req)
+
+ assertTrue(res.isOk)
+ assertTrue(res.getResult.isPresent)
+ assertEquals(expResp, res.getResult.get)
+ }
+
+
final protected def getClient: NCTestClient = {
if (cli == null)
throw new IllegalStateException("Client is not started.")
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/examples/time/NCTimeModelSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/examples/time/NCTimeModelSpec.scala
index 26ff1b8..c1bc764 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/examples/time/NCTimeModelSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/examples/time/NCTimeModelSpec.scala
@@ -17,38 +17,28 @@
package org.apache.nlpcraft.examples.time
-import java.io.IOException
-
import org.apache.nlpcraft.common.NCException
import org.apache.nlpcraft.{NCTestContext, NCTestEnvironment}
-import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
+import java.io.IOException
+
@NCTestEnvironment(model = classOf[TimeModel], startClient = true)
class NCTimeModelSpec extends NCTestContext {
@Test
@throws[NCException]
@throws[IOException]
private[time] def testIntentsPriorities(): Unit = {
- val cli = getClient
-
- def check(txt: String, id: String): Unit = {
- val res = cli.ask(txt)
-
- assertTrue(res.isOk)
- assertTrue(res.getIntentId == id)
- }
-
// intent1 must be winner for `What's the local time?` question, because exact matching.
// Accumulated history (geo:city information) must be ignored.
// 1. Without conversation.
- check("Show me time of the day in London.", "intent2")
- cli.clearConversation()
- check("What's the local time?", "intent1")
+ checkIntent("Show me time of the day in London.", "intent2")
+ getClient.clearConversation()
+ checkIntent("What's the local time?", "intent1")
// 2. The same with conversation.
- check("Show me time of the day in London.", "intent2")
- check("What's the local time?", "intent1")
+ checkIntent("Show me time of the day in London.", "intent2")
+ checkIntent("What's the local time?", "intent1")
}
}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIntentDslSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIntentDslSpec.scala
index 9946582..b662b07 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIntentDslSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIntentDslSpec.scala
@@ -55,21 +55,13 @@ class NCIntentDslSpecModel extends NCModelAdapter(
*/
@NCTestEnvironment(model = classOf[NCIntentDslSpecModel], startClient = true)
class NCIntentDslSpec extends NCTestContext {
- private def check(txt: String, intent: String): Unit = {
- val res = getClient.ask(txt)
-
- assertTrue(res.isOk, s"Checked: $txt")
- assertTrue(res.getResult.isPresent, s"Checked: $txt")
- assertEquals(intent, res.getIntentId, s"Checked: $txt")
- }
-
@Test
- def testBigCity(): Unit = check("Moscow", "bigCity")
+ def testBigCity(): Unit = checkIntent("Moscow", "bigCity")
@Test
- def testOtherCity(): Unit = check("San Francisco", "otherCity")
+ def testOtherCity(): Unit = checkIntent("San Francisco", "otherCity")
@Test
- def testUserPriority(): Unit = check("Paris", "userElement")
+ def testUserPriority(): Unit = checkIntent("Paris", "userElement")
}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIntentDslSpec2.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIntentDslSpec2.scala
index 06f1610..8bb1ba6 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIntentDslSpec2.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIntentDslSpec2.scala
@@ -18,7 +18,6 @@
package org.apache.nlpcraft.model
import org.apache.nlpcraft.{NCTestContext, NCTestEnvironment}
-import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue}
import org.junit.jupiter.api.Test
import java.util
@@ -57,21 +56,13 @@ class NCIntentDslSpecModel2 extends NCModelAdapter(
*/
@NCTestEnvironment(model = classOf[NCIntentDslSpecModel2], startClient = true)
class NCIntentDslSpec2 extends NCTestContext {
- private def check(txt: String, intent: String): Unit = {
- val res = getClient.ask(txt)
-
- assertTrue(res.isOk, s"Checked: $txt")
- assertTrue(res.getResult.isPresent, s"Checked: $txt")
- assertEquals(intent, res.getIntentId, s"Checked: $txt")
- }
-
@Test
def test(): Unit = {
- check("a", "a_11")
- check("a a", "a_23")
- check("a a a", "a_23")
- check("a a a a", "a_15")
- check("a a a a a a ", "a_plus")
+ checkIntent("a", "a_11")
+ checkIntent("a a", "a_23")
+ checkIntent("a a a", "a_23")
+ checkIntent("a a a a", "a_15")
+ checkIntent("a a a a a a ", "a_plus")
}
}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensEnricherSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensEnricherSpec.scala
new file mode 100644
index 0000000..2747c9a
--- /dev/null
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensEnricherSpec.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.model.`abstract`
+
+import org.apache.nlpcraft.NCTestEnvironment
+import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.{NCEnricherBaseSpec, NCEnrichersTestContext, NCTestNlpToken => nlp, NCTestUserToken => usr}
+import org.junit.jupiter.api.Test
+
+class NCAbstractTokensModelEnrichers extends NCAbstractTokensModel with NCEnrichersTestContext
+
+@NCTestEnvironment(model = classOf[NCAbstractTokensModelEnrichers], startClient = true)
+class NCAbstractTokensEnricherSpec extends NCEnricherBaseSpec {
+ @Test
+ def test(): Unit = {
+ // Checks that there aren't any other variants.
+ runBatch(
+ _ ⇒ checkAll(
+ "word the word",
+ Seq(
+ nlp(text = "word"),
+ usr("the word", "wrapAnyWord")
+ )
+ ),
+ _ ⇒ checkAll(
+ "10 w1 10 w2",
+ Seq(
+ nlp(text = "10"),
+ usr("w1 10 w2", "wrapNum")
+ )
+ )
+ )
+ }
+}
\ No newline at end of file
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensIntentsSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensIntentsSpec.scala
new file mode 100644
index 0000000..8006c82
--- /dev/null
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensIntentsSpec.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.model.`abstract`
+
+import org.apache.nlpcraft.model.{NCIntent, NCIntentMatch, NCResult}
+import org.apache.nlpcraft.{NCTestContext, NCTestEnvironment}
+import org.junit.jupiter.api.Test
+
+class NCAbstractTokensModelIntents extends NCAbstractTokensModel {
+ @NCIntent("intent=wrapAnyWordIntent term(t)={id == 'wrapAnyWord'}")
+ private def onWrapInternal(ctx: NCIntentMatch): NCResult = NCResult.text("OK")
+
+ @NCIntent("intent=wrapNumIntent term(t)={id == 'wrapNum'}")
+ private def onWrapNum(ctx: NCIntentMatch): NCResult = NCResult.text("OK")
+}
+
+@NCTestEnvironment(model = classOf[NCAbstractTokensModelIntents], startClient = true)
+class NCAbstractTokensIntentsSpec extends NCTestContext {
+ @Test
+ def test(): Unit = {
+ // First 'word' - will be deleted (abstract).
+ // Second 'word' - will be swallow (wrapAnyWord element).
+ checkIntent("word the word", "wrapAnyWordIntent")
+
+ // First numeric - will be deleted (abstract).
+ // Second numeric - will be swallow (wrapNum element).
+ checkIntent("10 w1 10 w2", "wrapNumIntent")
+ }
+}
\ No newline at end of file
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensModel.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensModel.scala
new file mode 100644
index 0000000..271818c
--- /dev/null
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/abstract/NCAbstractTokensModel.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.model.`abstract`
+
+import org.apache.nlpcraft.model.{NCElement, NCModelAdapter}
+
+import java.util
+import scala.collection.JavaConverters._
+
+class NCAbstractTokensModel extends NCModelAdapter(
+ "nlpcraft.abstract.elems.mdl.test", "Abstract Elements Test Model", "1.0"
+) {
+ private implicit val toList: String ⇒ util.List[String] = (s: String) ⇒ Seq(s).asJava
+
+ override def getElements: util.Set[NCElement] =
+ Set(
+ new NCElement {
+ override def getId: String = "anyWord"
+ override def getSynonyms: util.List[String] = "//[a-zA-Z0-9]+//"
+ },
+ new NCElement {
+ override def getId: String = "wrapAnyWord"
+ override def getSynonyms: util.List[String] = "the ^^[internal](id == 'anyWord')^^"
+ },
+ new NCElement {
+ override def getId: String = "wrapNum"
+ override def getSynonyms: util.List[String] = "w1 ^^id == 'nlpcraft:num'^^ w2"
+ }
+ ).asJava
+
+ override def getAbstractTokens: util.Set[String] = Set("nlpcraft:num", "anyWord").asJava
+ override def isPermutateSynonyms: Boolean = false
+ override def getJiggleFactor: Int = 0
+}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/NCStmTestModelSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/NCStmTestModelSpec.scala
index 31b8690..5fd0084 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/NCStmTestModelSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/models/stm/NCStmTestModelSpec.scala
@@ -18,7 +18,6 @@
package org.apache.nlpcraft.models.stm
import org.apache.nlpcraft.{NCTestContext, NCTestEnvironment}
-import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue}
import org.junit.jupiter.api.Test
/**
@@ -27,26 +26,14 @@ import org.junit.jupiter.api.Test
@NCTestEnvironment(model = classOf[NCStmTestModel], startClient = true)
class NCStmTestModelSpec extends NCTestContext {
/**
- * @param req
- * @param expResp
- */
- private def check(req: String, expResp: String): Unit = {
- val res = getClient.ask(req)
-
- assertTrue(res.isOk)
- assertTrue(res.getResult.isPresent)
- assertEquals(expResp, res.getResult.get)
- }
-
- /**
* Checks behaviour. It is based on intents and elements groups.
*/
@Test
private[stm] def test(): Unit = for (i ← 0 until 3) {
- check("sale", "sale")
- check("best", "sale_best_employee")
- check("buy", "buy")
- check("best", "buy_best_employee")
- check("sale", "sale")
+ checkResult("sale", "sale")
+ checkResult("best", "sale_best_employee")
+ checkResult("buy", "buy")
+ checkResult("best", "buy_best_employee")
+ checkResult("sale", "sale")
}
}
\ No newline at end of file
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCDefaultTestModel.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCDefaultTestModel.scala
index 8deb578..fe03c64 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCDefaultTestModel.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCDefaultTestModel.scala
@@ -17,19 +17,18 @@
package org.apache.nlpcraft.probe.mgrs.nlp.enrichers
-import java.util
-import java.util.Collections
-
-import org.apache.nlpcraft.model.{NCContext, NCElement, NCModelAdapter, NCResult, NCValue}
+import org.apache.nlpcraft.model.{NCElement, NCModelAdapter, NCResult, NCValue}
import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.NCDefaultTestModel._
+import java.util
+import java.util.Collections
import scala.collection.JavaConverters._
import scala.language.implicitConversions
/**
* Enrichers default test model.
*/
-class NCDefaultTestModel extends NCModelAdapter(ID, "Model enrichers test", "1.0") {
+class NCDefaultTestModel extends NCModelAdapter(ID, "Model enrichers test", "1.0") with NCEnrichersTestContext {
private implicit def convert(s: String): NCResult = NCResult.text(s)
override def getElements: util.Set[NCElement] =
@@ -61,11 +60,6 @@ class NCDefaultTestModel extends NCModelAdapter(ID, "Model enrichers test", "1.0
}).asJava
}
- override final def onContext(ctx: NCContext): NCResult =
- NCResult.text(
- NCTestSentence.serialize(ctx.getVariants.asScala.map(v ⇒ NCTestSentence(v.asScala.map(NCTestToken(_)))))
- )
-
final override def getId: String = ID
}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCEnricherBaseSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCEnricherBaseSpec.scala
index cfd39d0..26d536c 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCEnricherBaseSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCEnricherBaseSpec.scala
@@ -17,14 +17,29 @@
package org.apache.nlpcraft.probe.mgrs.nlp.enrichers
-import org.apache.nlpcraft.NCTestContext
+import org.apache.nlpcraft.{NCTestContext, NCTestEnvironment}
import org.junit.jupiter.api.Assertions.{assertTrue, fail}
+import org.junit.jupiter.api.{BeforeEach, TestInfo}
import org.scalatest.Assertions
/**
* Enrichers tests utility base class.
*/
abstract class NCEnricherBaseSpec extends NCTestContext {
+ @BeforeEach
+ def before(info: TestInfo): Unit = {
+ val env = getClassAnnotation(info).
+ getOrElse(
+ throw new IllegalStateException(
+ s"Enricher tests should ne annotated by model, see: ${classOf[NCTestEnvironment]}"
+ )
+ )
+
+ if (!(classOf[NCEnrichersTestContext]).isAssignableFrom(env.model()))
+ throw new IllegalStateException(
+ s"Enricher tests should ne annotated by model mixed with: ${classOf[NCEnrichersTestContext]}"
+ )
+ }
/**
* Checks single variant.
*
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCEnrichersTestContext.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCEnrichersTestContext.scala
new file mode 100644
index 0000000..c6c5de9
--- /dev/null
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCEnrichersTestContext.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.probe.mgrs.nlp.enrichers
+
+import org.apache.nlpcraft.model.{NCContext, NCModel, NCResult}
+
+import scala.collection.JavaConverters._
+
+/**
+ * Enricher test model behaviour.
+ */
+trait NCEnrichersTestContext extends NCModel {
+ override final def onContext(ctx: NCContext): NCResult =
+ NCResult.text(
+ NCTestSentence.serialize(ctx.getVariants.asScala.map(v ⇒ NCTestSentence(v.asScala.map(NCTestToken(_)))))
+ )
+}