You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2020/09/18 07:10:15 UTC
[incubator-nlpcraft] branch master updated (180d0b4 -> 449f965)
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
from 180d0b4 Improved logging for test framework (ANSI colored JSON).
new 60c3501 Update NCSuggestSynonymManager.scala
new 22edf82 Merge branch 'master' into NLPCRAFT-41
new 1d49198 Merge branch 'master' into NLPCRAFT-41
new d36a992 WIP.
new 461fd78 WIP.
new 4271471 Merge branch 'NLPCRAFT-41' into NLPCRAFT-117
new 3a21be0 WIP.
new 13b26e8 Merge branch 'master' into NLPCRAFT-117
new e93767b WIP.
new 3a2f270 Merge branch 'NLPCRAFT-117' into NLPCRAFT-41
new 67c7505 WIP.
new 449f965 WIP.
The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../model/intent/impl/NCIntentSolverEngine.scala | 29 ++++++++++---
.../server/sugsyn/NCSuggestSynonymManager.scala | 49 +++++++++++++---------
.../nlpcraft/server/rest/NCRestModelSpec.scala | 8 ++--
3 files changed, 56 insertions(+), 30 deletions(-)
[incubator-nlpcraft] 10/12: Merge branch 'NLPCRAFT-117' into
NLPCRAFT-41
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 3a2f2704dedf5a0e434b32d5cec2cb02362eaa17
Merge: d36a992 e93767b
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Fri Sep 18 09:44:18 2020 +0300
Merge branch 'NLPCRAFT-117' into NLPCRAFT-41
# Conflicts:
# nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
.../model/intent/impl/NCIntentSolverEngine.scala | 29 +++++++++++++++++-----
.../server/sugsyn/NCSuggestSynonymManager.scala | 10 +++-----
.../nlpcraft/server/rest/NCRestModelSpec.scala | 2 +-
3 files changed, 28 insertions(+), 13 deletions(-)
[incubator-nlpcraft] 09/12: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit d36a9920e6ef3bcb649ff7ef28b8faa79a0dea2e
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Fri Sep 18 09:30:56 2020 +0300
WIP.
---
.../server/sugsyn/NCSuggestSynonymManager.scala | 34 +++++++++++++++-------
.../nlpcraft/server/rest/NCRestModelSpec.scala | 8 ++---
2 files changed, 27 insertions(+), 15 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
index 9e4c358..b59f2dc 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
@@ -106,7 +106,7 @@ object NCSuggestSynonymManager extends NCService {
s"Unsupported symbols: $word"
)
}
- case class SuggestionResult(synonym: String, ctxWordSrvScore: Double, sgstCnt: Int)
+ case class SuggestionResult(synonym: String, factor: Double)
private def split(s: String): Seq[String] = s.split(" ").toSeq.map(_.trim).filter(_.nonEmpty)
private def toStem(s: String): String = split(s).map(NCNlpPorterStemmer.stem).mkString(" ")
@@ -191,7 +191,7 @@ object NCSuggestSynonymManager extends NCService {
promise.success(
NCSuggestSynonymResult(
modelId = mdlId,
- minScore = minScore,
+ minScore = 0,
durationMs = System.currentTimeMillis() - now,
timestamp = now,
error = err,
@@ -387,13 +387,13 @@ object NCSuggestSynonymManager extends NCService {
val seq = group.map { case (sgst, _) ⇒ sgst }.sortBy(-_.score)
// Drops repeated.
- (seq.head, seq.length)
+ (seq.head.word, seq.length, seq.map(_.score).sum / seq.size)
}.
toSeq.
- map { case (sgst, cnt) ⇒ (sgst, cnt, sgst.score * cnt / elemSgsts.size) }.
+ map { case (sgst, cnt, score) ⇒ (sgst, cnt, score * cnt / elemSgsts.size) }.
sortBy { case (_, _, sumFactor) ⇒ -sumFactor }.
zipWithIndex.
- foreach { case ((sgst, cnt, _), _) ⇒
+ foreach { case ((word, _, sumFactor), _) ⇒
val seq =
res.get(elemId) match {
case Some(seq) ⇒ seq
@@ -405,19 +405,31 @@ object NCSuggestSynonymManager extends NCService {
buf
}
- seq += SuggestionResult(sgst.word, sgst.score, cnt)
+ seq += SuggestionResult(word, sumFactor)
}
}
val resJ: util.Map[String, util.List[util.HashMap[String, Any]]] =
res.map { case (id, data) ⇒
- id → data.map(d ⇒ {
+ val factors = data.map(_.factor)
+
+ val min = factors.min
+ val max = factors.max
+ var delta = max - min
+
+ if (delta == 0)
+ delta = max
+
+ def normalize(v: Double): Double = (v - min) / delta
+
+ val norm = data.map(s ⇒ SuggestionResult(s.synonym, normalize(s.factor))).
+ filter(_.factor >= minScore)
+
+ id → norm.map(d ⇒ {
val m = new util.HashMap[String, Any]()
- m.put("synonym", d.synonym)
- // ContextWord server range is (0, 2)
- m.put("ctxWordServerScore", d.ctxWordSrvScore / 2)
- m.put("suggestedCount", d.sgstCnt)
+ m.put("synonym", d.synonym.toLowerCase)
+ m.put("factor", d.factor)
m
}).asJava
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
index 2c4ef04..444d38a 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
@@ -18,20 +18,20 @@
package org.apache.nlpcraft.server.rest
import org.apache.nlpcraft.NCTestEnvironment
-import org.apache.nlpcraft.examples.time.TimeModel
+import org.apache.nlpcraft.examples.alarm.AlarmModel
import org.junit.jupiter.api.Assertions._
import org.junit.jupiter.api.{Disabled, Test}
// Enable it and run if context word server started.
@Disabled
-@NCTestEnvironment(model = classOf[TimeModel], startClient = false)
+@NCTestEnvironment(model = classOf[AlarmModel], startClient = false)
class NCRestModelSpec extends NCRestSpec {
@Test
def test(): Unit = {
- post("model/sugsyn", "mdlId" → "nlpcraft.time.ex")(
+ post("model/sugsyn", "mdlId" → "nlpcraft.alarm.ex")(
("$.status", (status: String) ⇒ assertEquals("API_OK", status))
)
- post("model/sugsyn", "mdlId" → "nlpcraft.time.ex", "minScore" → 0.5)(
+ post("model/sugsyn", "mdlId" → "nlpcraft.alarm.ex", "minScore" → 0.5)(
("$.status", (status: String) ⇒ assertEquals("API_OK", status))
)
}
[incubator-nlpcraft] 04/12: Update NCSuggestSynonymManager.scala
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 60c35017c62eb45ce4cf0e8e4715b74398ba9808
Author: Aaron Radzinski <ar...@datalingvo.com>
AuthorDate: Thu Sep 17 11:09:03 2020 -0700
Update NCSuggestSynonymManager.scala
---
.../org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
index 8acee4d..9e4c358 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
@@ -211,7 +211,7 @@ object NCSuggestSynonymManager extends NCService {
if (allSamplesCnt < MIN_CNT_MODEL)
warns +=
- s"Model has too few intents samples: $allSamplesCnt. " +
+ s"Model has too few ($allSamplesCnt) intents samples. " +
s"It will negatively affect the quality of suggestions. " +
s"Try to increase overall sample count to at least $MIN_CNT_MODEL."
[incubator-nlpcraft] 02/12: Merge branch 'NLPCRAFT-41' into
NLPCRAFT-117
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 4271471fff023b9f6763765487c1a8132e78674a
Merge: 461fd78 f252759
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Wed Sep 16 17:20:19 2020 +0300
Merge branch 'NLPCRAFT-41' into NLPCRAFT-117
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 24 +---------------------
.../nlpcraft/server/rest/NCBasicRestApi.scala | 18 ++++++++--------
.../apache/nlpcraft/server/rest/NCRestSpec.scala | 2 +-
3 files changed, 11 insertions(+), 33 deletions(-)
[incubator-nlpcraft] 05/12: Merge branch 'master' into NLPCRAFT-41
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 22edf82a661d2f514c6302113b9f71d1f266b6b9
Merge: 60c3501 bed81f1
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Thu Sep 17 22:38:28 2020 +0300
Merge branch 'master' into NLPCRAFT-41
.../apache/nlpcraft/common/nlp/NCNlpSentence.scala | 22 ++--
.../mgrs/nlp/enrichers/sort/NCSortEnricher.scala | 125 ++++++++++++---------
.../scala/org/apache/nlpcraft/NCTestContext.scala | 116 +++++++++++++++++++
...CRestProbeSpec.scala => NCTestEnvironment.java} | 42 ++++---
.../nlpcraft/examples/sql/NCSqlExampleSpec.scala | 36 ++----
.../nlpcraft/examples/sql/NCSqlModelSpec.scala | 27 ++---
.../nlpcraft/examples/time/NCTimeModelSpec.scala | 33 +-----
.../model/conversation/NCConversationSpec.scala | 33 +-----
.../nlpcraft/models/stm/NCStmTestModelSpec.scala | 34 +-----
.../mgrs/nlp/enrichers/NCEnricherBaseSpec.scala | 45 ++------
.../nlp/enrichers/limit/NCEnricherLimitSpec.scala | 7 +-
.../model/NCEnricherNestedModelSpec.scala | 34 +++++-
.../nlp/enrichers/model/NCNestedTestModel.scala | 47 --------
.../relation/NCEnricherRelationSpec.scala | 7 +-
.../nlp/enrichers/sort/NCEnricherSortSpec.scala | 9 +-
.../nlpcraft/server/rest/NCRestAskSpec.scala | 10 +-
.../nlpcraft/server/rest/NCRestClearSpec.scala | 10 +-
.../nlpcraft/server/rest/NCRestModelSpec.scala | 11 +-
.../nlpcraft/server/rest/NCRestProbeSpec.scala | 14 +--
.../apache/nlpcraft/server/rest/NCRestSpec.scala | 3 +-
20 files changed, 319 insertions(+), 346 deletions(-)
[incubator-nlpcraft] 03/12: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 3a21be060c99c25192140dc7bff22ccd6110f005
Author: Aaron Radzinski <ar...@datalingvo.com>
AuthorDate: Wed Sep 16 12:39:31 2020 -0700
WIP.
---
.../apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala | 3 +++
.../apache/nlpcraft/model/intent/impl/NCIntentSolverVariant.scala | 6 +++---
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala
index 1e0e6cb..ed8a900 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala
@@ -436,12 +436,14 @@ object NCIntentSolverEngine extends LazyLogging with NCOpenCensusTrace {
// Term is missing. Stop further processing for this intent.
// This intent cannot be matched.
logger.trace(s"Term '$term' is missing for intent '$intentId' (stopping further processing).")
+
abort = true
}
}
if (abort) {
logger.info(s"Intent '$intentId' didn't match because of missing term $varStr.")
+
None
}
else if (senToks.exists(tok ⇒ !tok.used && tok.token.isUserDefined)) {
@@ -457,6 +459,7 @@ object NCIntentSolverEngine extends LazyLogging with NCOpenCensusTrace {
}
else if (!senToks.exists(tok ⇒ tok.used && !tok.conv)) {
logger.info(s"Intent '$intentId' didn't match because all tokens came from STM $varStr.")
+
None
}
else {
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverVariant.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverVariant.scala
index 0716494..a9e3aa9 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverVariant.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverVariant.scala
@@ -82,11 +82,11 @@ case class NCIntentSolverVariant(tokens: util.List[NCToken]) extends Ordered[NCI
override def toString: String =
s"Variant [" +
- s"userToks=$userToks" +
+ s"userToks=$userToks" +
s", wordCnt=$wordCnt" +
- s", totalUserDirect=$totalUserDirect" +
+ s", totalUserDirect=$totalUserDirect" +
s", avgWordsPerTok=$avgWordsPerTok" +
- s", sparsity=$totalSparsity" +
+ s", sparsity=$totalSparsity" +
s", toks=$tokens" +
"]"
}
\ No newline at end of file
[incubator-nlpcraft] 11/12: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 67c7505c33489c714b362ef2e2193aa9286873c9
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Fri Sep 18 09:45:30 2020 +0300
WIP.
---
.../test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
index bafe71d..444d38a 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
@@ -23,7 +23,7 @@ import org.junit.jupiter.api.Assertions._
import org.junit.jupiter.api.{Disabled, Test}
// Enable it and run if context word server started.
-//@Disabled
+@Disabled
@NCTestEnvironment(model = classOf[AlarmModel], startClient = false)
class NCRestModelSpec extends NCRestSpec {
@Test
[incubator-nlpcraft] 06/12: Merge branch 'master' into NLPCRAFT-117
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 13b26e880cadd415716ba7622c72b0518a4a52aa
Merge: 3a21be0 180d0b4
Author: Aaron Radzinski <ar...@datalingvo.com>
AuthorDate: Thu Sep 17 21:03:23 2020 -0700
Merge branch 'master' into NLPCRAFT-117
.../nlp/core/stanford/NCStanfordCoreManager.scala | 14 +-
.../nlp/core/stanford/NCStanfordNerEnricher.scala | 19 ++-
.../nlp/core/stanford/NCStanfordParser.scala | 13 +-
.../org/apache/nlpcraft/common/NCService.scala | 53 ++++---
.../common/extcfg/NCExternalConfigManager.scala | 15 +-
.../nlpcraft/common/makro/NCMacroParser.scala | 2 +-
.../apache/nlpcraft/common/nlp/NCNlpSentence.scala | 22 +--
.../common/nlp/core/NCNlpCoreManager.scala | 21 ++-
.../nlp/core/opennlp/NCOpenNlpTokenizer.scala | 25 +++-
.../common/nlp/dict/NCDictionaryManager.scala | 12 +-
.../common/nlp/numeric/NCNumericManager.scala | 10 +-
.../org/apache/nlpcraft/common/util/NCUtils.scala | 65 +++++++--
.../model/intent/impl/NCIntentSolver.scala | 4 +-
.../model/intent/impl/NCIntentSolverEngine.scala | 2 +-
.../model/tools/test/NCTestClientBuilder.java | 13 +-
.../test/impl/NCTestAutoModelValidatorImpl.scala | 5 +-
.../org/apache/nlpcraft/probe/NCProbeBoot.scala | 2 +-
.../nlpcraft/probe/mgrs/NCProbeSynonym.scala | 5 +-
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 18 +++
.../probe/mgrs/conn/NCConnectionManager.scala | 4 +-
.../mgrs/conversation/NCConversationManager.scala | 13 +-
.../probe/mgrs/deploy/NCDeployManager.scala | 159 ++++++++++-----------
.../mgrs/dialogflow/NCDialogFlowManager.scala | 13 +-
.../probe/mgrs/lifecycle/NCLifecycleManager.scala | 22 ++-
.../nlpcraft/probe/mgrs/model/NCModelManager.scala | 32 +++--
.../nlpcraft/probe/mgrs/nlp/NCProbeEnricher.scala | 3 +-
.../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 39 +++--
.../dictionary/NCDictionaryEnricher.scala | 23 +--
.../mgrs/nlp/enrichers/limit/NCLimitEnricher.scala | 16 ++-
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 13 +-
.../enrichers/relation/NCRelationEnricher.scala | 14 +-
.../mgrs/nlp/enrichers/sort/NCSortEnricher.scala | 140 ++++++++++--------
.../enrichers/stopword/NCStopWordEnricher.scala | 37 +++--
.../suspicious/NCSuspiciousNounsEnricher.scala | 13 +-
.../mgrs/nlp/validate/NCValidateManager.scala | 17 ++-
.../nlpcraft/server/company/NCCompanyManager.scala | 17 ++-
.../server/feedback/NCFeedbackManager.scala | 15 +-
.../apache/nlpcraft/server/geo/NCGeoManager.scala | 26 ++--
.../geo/tools/NCGeoSyntheticNamesGenerator.scala | 3 +-
.../lifecycle/NCServerLifecycleManager.scala | 21 ++-
.../server/nlp/core/NCNlpNerEnricher.scala | 4 +-
.../server/nlp/core/NCNlpServerManager.scala | 13 +-
.../nlp/core/google/NCGoogleNerEnricher.scala | 23 ++-
.../nlp/core/opennlp/NCOpenNlpNerEnricher.scala | 25 +++-
.../server/nlp/core/opennlp/NCOpenNlpParser.scala | 19 ++-
.../server/nlp/core/spacy/NCSpaCyNerEnricher.scala | 21 ++-
.../nlp/enrichers/NCServerEnrichmentManager.scala | 14 +-
.../nlp/enrichers/basenlp/NCBaseNlpEnricher.scala | 26 +++-
.../coordinate/NCCoordinatesEnricher.scala | 19 ++-
.../server/nlp/enrichers/date/NCDateEnricher.scala | 131 +++++++++--------
.../server/nlp/enrichers/geo/NCGeoEnricher.scala | 21 ++-
.../nlp/enrichers/numeric/NCNumericEnricher.scala | 23 ++-
.../nlp/enrichers/quote/NCQuoteEnricher.scala | 23 ++-
.../enrichers/stopword/NCStopWordEnricher.scala | 16 ++-
.../server/nlp/preproc/NCPreProcessManager.scala | 17 ++-
.../server/nlp/spell/NCSpellCheckManager.scala | 19 ++-
.../server/nlp/wordnet/NCWordNetManager.scala | 16 ++-
.../nlpcraft/server/probe/NCProbeManager.scala | 18 +--
.../server/proclog/NCProcessLogManager.scala | 13 +-
.../nlpcraft/server/query/NCQueryManager.scala | 21 ++-
.../nlpcraft/server/rest/NCBasicRestApi.scala | 16 +--
.../nlpcraft/server/rest/NCRestManager.scala | 15 +-
.../apache/nlpcraft/server/sql/NCSqlManager.scala | 16 ++-
.../server/sugsyn/NCSuggestSynonymManager.scala | 15 +-
.../apache/nlpcraft/server/tx/NCTxManager.scala | 29 ++--
.../nlpcraft/server/user/NCUserManager.scala | 13 +-
.../scala/org/apache/nlpcraft/NCTestContext.scala | 116 +++++++++++++++
.../org/apache/nlpcraft/NCTestEnvironment.java} | 40 ++++--
.../nlpcraft/examples/sql/NCSqlExampleSpec.scala | 36 ++---
.../nlpcraft/examples/sql/NCSqlModelSpec.scala | 27 ++--
.../nlpcraft/examples/time/NCTimeModelSpec.scala | 33 +----
.../model/conversation/NCConversationSpec.scala | 33 +----
.../nlpcraft/models/stm/NCStmTestModelSpec.scala | 34 +----
.../mgrs/nlp/enrichers/NCEnricherBaseSpec.scala | 45 ++----
.../nlp/enrichers/limit/NCEnricherLimitSpec.scala | 7 +-
.../model/NCEnricherNestedModelSpec.scala | 34 ++++-
.../nlp/enrichers/model/NCNestedTestModel.scala | 47 ------
.../relation/NCEnricherRelationSpec.scala | 7 +-
.../nlp/enrichers/sort/NCEnricherSortSpec.scala | 9 +-
.../nlpcraft/server/rest/NCRestAskSpec.scala | 10 +-
.../nlpcraft/server/rest/NCRestClearSpec.scala | 10 +-
.../nlpcraft/server/rest/NCRestModelSpec.scala | 11 +-
.../nlpcraft/server/rest/NCRestProbeSpec.scala | 14 +-
.../apache/nlpcraft/server/rest/NCRestSpec.scala | 3 +-
84 files changed, 1274 insertions(+), 788 deletions(-)
[incubator-nlpcraft] 01/12: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 461fd78c8fdbe20dc3178b8cbbdfb785d6a98947
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Wed Sep 16 15:35:44 2020 +0300
WIP.
---
.../model/intent/impl/NCIntentSolverEngine.scala | 19 ++++++++++++++-----
1 file changed, 14 insertions(+), 5 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala
index ac29a95..1e0e6cb 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala
@@ -249,11 +249,11 @@ object NCIntentSolverEngine extends LazyLogging with NCOpenCensusTrace {
tbl += (
Seq(
s"#${m.variantIdx + 1}",
- s"${ansiRedFg}<<best match>>$ansiReset"
+ s"$ansiRedFg<<best match>>$ansiReset"
),
Seq(
im.intent.id,
- s"${ansiRedFg}<<best match>>$ansiReset"
+ s"$ansiRedFg<<best match>>$ansiReset"
),
mkPickTokens(im)
)
@@ -404,6 +404,7 @@ object NCIntentSolverEngine extends LazyLogging with NCOpenCensusTrace {
// Check dialog flow first.
if (!intent.flow.isEmpty && !matchFlow(intent.flow, hist)) {
logger.info(s"Intent '$intentId' didn't match because of dialog flow $varStr.")
+
None
}
else {
@@ -459,9 +460,17 @@ object NCIntentSolverEngine extends LazyLogging with NCOpenCensusTrace {
None
}
else {
- val exactMatch = !senToks.exists(tok ⇒ !tok.used && !tok.token.isFreeWord)
-
- intentW.setWeight(0, if (exactMatch) 1 else 0)
+ val exactMatch = !(senToks ++ convToks).exists(tok ⇒ !tok.used && !tok.token.isFreeWord)
+
+ val mainWeight =
+ if (exactMatch && convToks.isEmpty)
+ 2
+ else if (exactMatch)
+ 1
+ else
+ 0
+
+ intentW.setWeight(0, mainWeight)
Some(IntentMatch(
tokenGroups = intentGrps.toList,
[incubator-nlpcraft] 08/12: Merge branch 'master' into NLPCRAFT-41
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 1d49198a313fe607214f06c1ae3ad348991062da
Merge: 22edf82 180d0b4
Author: Aaron Radzinski <ar...@datalingvo.com>
AuthorDate: Thu Sep 17 23:00:43 2020 -0700
Merge branch 'master' into NLPCRAFT-41
.../nlpcraft/common/makro/NCMacroParser.scala | 2 +-
.../org/apache/nlpcraft/common/util/NCUtils.scala | 56 +++++++++++++++++++---
.../model/intent/impl/NCIntentSolverEngine.scala | 2 +-
.../model/tools/test/NCTestClientBuilder.java | 13 +++--
.../org/apache/nlpcraft/probe/NCProbeBoot.scala | 2 +-
.../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 4 +-
6 files changed, 65 insertions(+), 14 deletions(-)
[incubator-nlpcraft] 07/12: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit e93767b3c4b7a976154f4a76f3f1b6d199580317
Author: Aaron Radzinski <ar...@datalingvo.com>
AuthorDate: Thu Sep 17 23:00:06 2020 -0700
WIP.
---
.../nlpcraft/model/intent/impl/NCIntentSolverEngine.scala | 13 +++++++++----
.../nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala | 10 ++++------
.../org/apache/nlpcraft/server/rest/NCRestModelSpec.scala | 10 +++++-----
3 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala
index 4a38cc1..dd7f876 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/intent/impl/NCIntentSolverEngine.scala
@@ -442,7 +442,7 @@ object NCIntentSolverEngine extends LazyLogging with NCOpenCensusTrace {
}
if (abort) {
- logger.info(s"Intent '$intentId' didn't match because of missing term $varStr.")
+ logger.info(s"Intent '$intentId' didn't match because of unmatched term $varStr.")
None
}
@@ -458,20 +458,25 @@ object NCIntentSolverEngine extends LazyLogging with NCOpenCensusTrace {
None
}
else if (!senToks.exists(tok ⇒ tok.used && !tok.conv)) {
- logger.info(s"Intent '$intentId' didn't match because all tokens came from STM $varStr.")
+ logger.info(s"Intent '$intentId' didn't match because all its matched tokens came from STM $varStr.")
None
}
else {
- val exactMatch = !(senToks ++ convToks).exists(tok ⇒ !tok.used && !tok.token.isFreeWord)
+ // Exact match calculation DOES NOT include tokens from conversation, if any.
+ val exactMatch = !senToks.exists(tok ⇒ !tok.used && !tok.token.isFreeWord)
- val mainWeight =
+ val mainWeight = {
+ // Best weight if the match is exact and conversation WAS NOT used.
if (exactMatch && convToks.isEmpty)
2
+ // Second best weight if the match is exact and conversation WAS used.
else if (exactMatch)
1
+ // Third best (i.e. worst) weight if match WAS NOT EXACT.
else
0
+ }
intentW.setWeight(0, mainWeight)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
index 8acee4d..6500bd4 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
@@ -223,7 +223,7 @@ object NCSuggestSynonymManager extends NCService {
if (ids.nonEmpty)
warns +=
- s"Following model intent have too few samples: ${ids.mkString(", ")}. " +
+ s"Following model intent have too few samples (${ids.mkString(", ")}). " +
s"It will negatively affect the quality of suggestions. " +
s"Try to increase overall sample count to at least $MIN_CNT_INTENT."
}
@@ -298,13 +298,11 @@ object NCSuggestSynonymManager extends NCService {
val allReqsCnt = allReqs.map(_._2.size).sum
val allSynsCnt = elemSyns.map(_._2.size).sum
- logger.trace(
- s"Request is going to execute on 'ctxword' server [" +
+ logger.trace(s"Request is going to execute on 'ctxword' server [" +
s"exs=${exs.size}, " +
s"syns=$allSynsCnt, " +
s"reqs=$allReqsCnt" +
- s"]"
- )
+ s"]")
if (allReqsCnt == 0)
onError(s"Suggestions cannot be generated for model: '$mdlId'")
@@ -369,7 +367,7 @@ object NCSuggestSynonymManager extends NCService {
cdl.await(Long.MaxValue, TimeUnit.MILLISECONDS)
if (err.get() != null)
- throw new NCE("Error during work with ContextWordServer", err.get())
+ throw new NCE("Error during work with 'ContextWordServer'.", err.get())
val allSynsStems = elemSyns.flatMap(_._2).flatten.map(_.stem).toSet
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
index 2c4ef04..bafe71d 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala
@@ -18,20 +18,20 @@
package org.apache.nlpcraft.server.rest
import org.apache.nlpcraft.NCTestEnvironment
-import org.apache.nlpcraft.examples.time.TimeModel
+import org.apache.nlpcraft.examples.alarm.AlarmModel
import org.junit.jupiter.api.Assertions._
import org.junit.jupiter.api.{Disabled, Test}
// Enable it and run if context word server started.
-@Disabled
-@NCTestEnvironment(model = classOf[TimeModel], startClient = false)
+//@Disabled
+@NCTestEnvironment(model = classOf[AlarmModel], startClient = false)
class NCRestModelSpec extends NCRestSpec {
@Test
def test(): Unit = {
- post("model/sugsyn", "mdlId" → "nlpcraft.time.ex")(
+ post("model/sugsyn", "mdlId" → "nlpcraft.alarm.ex")(
("$.status", (status: String) ⇒ assertEquals("API_OK", status))
)
- post("model/sugsyn", "mdlId" → "nlpcraft.time.ex", "minScore" → 0.5)(
+ post("model/sugsyn", "mdlId" → "nlpcraft.alarm.ex", "minScore" → 0.5)(
("$.status", (status: String) ⇒ assertEquals("API_OK", status))
)
}
[incubator-nlpcraft] 12/12: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 449f9652ca99a40b0ce3f7ce7606f608bb7b9919
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Fri Sep 18 09:57:37 2020 +0300
WIP.
---
.../apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
index 5fa6351..a22cd93 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
@@ -191,7 +191,7 @@ object NCSuggestSynonymManager extends NCService {
promise.success(
NCSuggestSynonymResult(
modelId = mdlId,
- minScore = 0,
+ minScore = minScore,
durationMs = System.currentTimeMillis() - now,
timestamp = now,
error = err,
@@ -326,8 +326,7 @@ object NCSuggestSynonymManager extends NCService {
GSON.toJson(
RestRequest(
sentences = batch.map(p ⇒ RestRequestSentence(p.sentence, Seq(p.index).asJava)).asJava,
- // 'ctxword'' server range is (0, 2), input range is (0, 1)
- minScore = minScore * 2,
+ minScore = 0,
limit = MAX_LIMIT
)
),
@@ -459,4 +458,4 @@ object NCSuggestSynonymManager extends NCService {
promise.future
}
-}
+}
\ No newline at end of file