You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2020/06/14 09:18:41 UTC
[incubator-nlpcraft] branch NLPCRAFT-70 updated (34f0f40 -> e088b63)
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a change to branch NLPCRAFT-70
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
from 34f0f40 WIP.
add 51faecf NLPCRAFT-67: Change json response/request schema
add 07da3df WIP on NLPCRAFT-80.
add 354b982 Merge branch 'master' into NLPCRAFT-67
new b7d3e5e Merge branch 'NLPCRAFT-67' into NLPCRAFT-70
new e088b63 WIP.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
nlpcraft/src/main/python/ctxword/README.md | 8 ++--
nlpcraft/src/main/python/ctxword/bertft/bertft.py | 5 ++-
nlpcraft/src/main/python/ctxword/bin/predict.sh | 2 +-
nlpcraft/src/main/python/ctxword/server.py | 11 +++++-
.../apache/nlpcraft/examples/sql/SqlModel.scala | 16 ++++++--
.../scala/org/apache/nlpcraft/model/NCIntent.java | 1 +
.../{NCIntentTerm.java => NCIntentExample.java} | 21 +++++-----
.../org/apache/nlpcraft/model/NCIntentRef.java | 1 +
.../org/apache/nlpcraft/model/NCIntentTerm.java | 1 +
.../model/intent/impl/NCIntentScanner.scala | 18 ++++-----
.../server/ctxword/NCContextWordManager.scala | 45 +++++++++-------------
.../enrichers/ctxword/NCContextWordEnricher.scala | 5 +--
.../nlpcraft/server/probe/NCProbeManager.scala | 2 +-
13 files changed, 76 insertions(+), 60 deletions(-)
copy nlpcraft/src/main/scala/org/apache/nlpcraft/model/{NCIntentTerm.java => NCIntentExample.java} (68%)
[incubator-nlpcraft] 01/02: Merge branch 'NLPCRAFT-67' into
NLPCRAFT-70
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-70
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit b7d3e5ecbfa99e9040eb91dadd5c35db37ee68a9
Merge: 34f0f40 354b982
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Sun Jun 14 12:05:52 2020 +0300
Merge branch 'NLPCRAFT-67' into NLPCRAFT-70
nlpcraft/src/main/python/ctxword/README.md | 8 ++++----
nlpcraft/src/main/python/ctxword/bertft/bertft.py | 5 ++++-
nlpcraft/src/main/python/ctxword/bin/predict.sh | 2 +-
nlpcraft/src/main/python/ctxword/server.py | 11 +++++++++--
.../org/apache/nlpcraft/examples/sql/SqlModel.scala | 16 ++++++++++++----
.../scala/org/apache/nlpcraft/model/NCIntent.java | 1 +
.../{NCIntentTerm.java => NCIntentExample.java} | 21 ++++++++++++---------
.../org/apache/nlpcraft/model/NCIntentRef.java | 1 +
.../org/apache/nlpcraft/model/NCIntentTerm.java | 1 +
.../model/intent/impl/NCIntentScanner.scala | 18 +++++++++---------
10 files changed, 54 insertions(+), 30 deletions(-)
[incubator-nlpcraft] 02/02: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-70
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit e088b633a24afb44673d5adc610f80566e4b4d99
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Sun Jun 14 12:18:32 2020 +0300
WIP.
---
.../server/ctxword/NCContextWordManager.scala | 45 +++++++++-------------
.../enrichers/ctxword/NCContextWordEnricher.scala | 5 +--
.../nlpcraft/server/probe/NCProbeManager.scala | 2 +-
3 files changed, 22 insertions(+), 30 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/ctxword/NCContextWordManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/ctxword/NCContextWordManager.scala
index 63b86bd..2940852 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/ctxword/NCContextWordManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/ctxword/NCContextWordManager.scala
@@ -39,6 +39,8 @@ import org.apache.nlpcraft.server.mdo.NCContextWordConfigMdo
import org.apache.nlpcraft.server.nlp.core.{NCNlpParser, NCNlpServerManager, NCNlpWord}
import org.apache.nlpcraft.server.opencensus.NCOpenCensusServerStats
+import java.util.{List ⇒ JList}
+
import scala.collection.JavaConverters._
import scala.util.control.Exception.catching
@@ -51,7 +53,7 @@ object NCContextWordManager extends NCService with NCOpenCensusServerStats with
}
private final val GSON = new Gson()
- private final val TYPE_RESP = new TypeToken[java.util.List[java.util.List[java.util.List[Any]]]]() {}.getType
+ private final val TYPE_RESP = new TypeToken[JList[JList[Suggestion]]]() {}.getType
private final val CLIENT = HttpClients.createDefault
private final val CONF_LIMIT = 20
@@ -62,9 +64,10 @@ object NCContextWordManager extends NCService with NCOpenCensusServerStats with
@volatile private var cache: IgniteCache[NCContextWordRequest, Seq[NCContextWordResponse]] = _
// We don't use directly bert and ftext indexes.
- case class Suggestion(word: String, totalScore: Double)
+ case class Suggestion(word: String, score: Double)
+ case class RestSentence(text: String, indexes: JList[Int])
case class RestRequest(
- sentences: java.util.List[java.util.List[Any]],
+ sentences: JList[RestSentence],
limit: Int,
min_score: Double,
min_ftext: Double,
@@ -83,20 +86,10 @@ object NCContextWordManager extends NCService with NCOpenCensusServerStats with
code match {
case 200 ⇒
- val data: java.util.List[java.util.List[java.util.List[Any]]] = GSON.fromJson(js, TYPE_RESP)
-
- data.asScala.map(p ⇒
- if (p.isEmpty)
- Seq.empty
- else
- // Skips header.
- p.asScala.tail.map(p ⇒
- Suggestion(
- word = p.get(0).asInstanceOf[String],
- totalScore = p.get(1).asInstanceOf[Double]
- )
- )
- )
+ val data: JList[JList[Suggestion]] = GSON.fromJson(js, TYPE_RESP)
+
+ // Skips header.
+ data.asScala.map(p ⇒ if (p.isEmpty) Seq.empty else p.asScala.tail)
case 400 ⇒ throw new RuntimeException(js)
case _ ⇒ throw new RuntimeException(s"Unexpected response [code=$code, response=$js]")
@@ -113,15 +106,17 @@ object NCContextWordManager extends NCService with NCOpenCensusServerStats with
val reqsSrv = res.filter { case (_, cachedWords) ⇒ cachedWords == null }.keys.toSeq
if (reqsSrv.nonEmpty) {
- val reqsSrvNorm: Seq[(Seq[String], Seq[Any])] =
+ val reqsSrvNorm: Seq[(Seq[String], RestSentence)] =
reqsSrv.groupBy(_.words).
- map { case (words, seq) ⇒ words → (Seq(words.mkString(" ")) ++ seq.map(_.wordIndex).sorted) }.toSeq
+ map {
+ case (words, seq) ⇒ words → RestSentence(words.mkString(" "), seq.map(_.wordIndex).sorted.asJava)
+ }.toSeq
- require(reqsSrv.size == reqsSrvNorm.map(_._2.size - 1).sum)
+ require(reqsSrv.size == reqsSrvNorm.map(_._2.indexes.size).sum)
val restReq =
RestRequest(
- sentences = reqsSrvNorm.map(_._2.asJava).asJava,
+ sentences = reqsSrvNorm.map { case (_, restSen) ⇒ restSen }.asJava,
limit = f.limit,
min_score = f.totalScore,
min_ftext = f.ftextScore,
@@ -140,7 +135,7 @@ object NCContextWordManager extends NCService with NCOpenCensusServerStats with
NCContextWordResponse(
word = p.word,
stem = NCNlpCoreManager.stemWord(p.word),
- totalScore = p.totalScore
+ totalScore = p.score
))
)
finally
@@ -149,9 +144,7 @@ object NCContextWordManager extends NCService with NCOpenCensusServerStats with
require(reqsSrv.size == resp.size)
val reqsSrvDenorm =
- reqsSrvNorm.flatMap {
- case (txt, vals) ⇒ vals.tail.map(idx ⇒ NCContextWordRequest(txt, idx.asInstanceOf[Int]))
- }
+ reqsSrvNorm.flatMap { case (txt, sen) ⇒ sen.indexes.asScala.map(NCContextWordRequest(txt, _)) }
def sort(seq: Seq[NCContextWordRequest]): Seq[NCContextWordRequest] =
seq.sortBy(p ⇒ (p.words.mkString(" "), p.wordIndex))
@@ -224,7 +217,7 @@ object NCContextWordManager extends NCService with NCOpenCensusServerStats with
}
@throws[NCE]
- def makeContextWordConfig(
+ def makeConfig(
mdlId: String,
ctxSyns: Map[String /*Element ID*/ , Map[String /*Value*/, Set[String] /*Values synonyms stems*/ ]],
examples: Set[String]
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
index 174c3ae..1138941 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
@@ -27,11 +27,10 @@ import org.apache.nlpcraft.server.nlp.enrichers.NCServerEnricher
import scala.collection.Map
object NCContextWordEnricher extends NCServerEnricher {
- // TODO: score
private final val MIN_SENTENCE_SCORE = 0.3
- private final val MIN_EXAMPLE_SCORE = 1
-
private final val MIN_SENTENCE_BERT = 0.3
+
+ private final val MIN_EXAMPLE_SCORE = 1
private final val MIN_EXAMPLE_BERT = 0.5
private final val LIMIT = 10
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index c576b23..71e0c5a 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -634,7 +634,7 @@ object NCProbeManager extends NCService {
enabledBuiltInTokens = m.enabledBuiltInTokens,
ctxWordsConfig =
if (m.ctxSynonyms.nonEmpty)
- Some(NCContextWordManager.makeContextWordConfig(m.id, m.ctxSynonyms, m.examples))
+ Some(NCContextWordManager.makeConfig(m.id, m.ctxSynonyms, m.examples))
else
None
)