You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2020/06/25 11:15:21 UTC
[incubator-nlpcraft] branch NLPCRAFT-70 updated (6014030 -> 700b516)
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a change to branch NLPCRAFT-70
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
from 6014030 Merge branch 'NLPCRAFT-67' into NLPCRAFT-70
new daad8b5 WIP.
new 700b516 WIP.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../classification/ClassificationModel.scala | 33 ++++++++
.../probe/mgrs/conn/NCConnectionManager.scala | 3 +-
.../server/ctxword/NCContextWordManager.scala | 16 +++-
.../server/ctxword/NCContextWordParameter.scala | 40 +--------
.../server/mdo/NCContextWordConfigMdo.scala | 8 +-
.../enrichers/ctxword/NCContextWordEnricher.scala | 47 ++++++-----
.../enrichers/ctxword/NCContextWordFactors.scala | 94 ++++++++++++++++++++++
.../nlpcraft/server/probe/NCProbeManager.scala | 14 ++--
8 files changed, 190 insertions(+), 65 deletions(-)
create mode 100644 nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordFactors.scala
[incubator-nlpcraft] 02/02: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-70
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 700b516b216b0278b23c00c4006ed04afecafa1d
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Thu Jun 25 14:15:14 2020 +0300
WIP.
---
.../classification/ClassificationModel.scala | 10 ++-
.../server/mdo/NCContextWordConfigMdo.scala | 5 +-
.../enrichers/ctxword/NCContextWordEnricher.scala | 56 ++++++-------
.../enrichers/ctxword/NCContextWordFactors.scala | 94 ++++++++++++++++++++++
.../nlpcraft/server/probe/NCProbeManager.scala | 2 +-
5 files changed, 135 insertions(+), 32 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/classification/ClassificationModel.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/classification/ClassificationModel.scala
index 6c5183e..f8d35fe 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/classification/ClassificationModel.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/classification/ClassificationModel.scala
@@ -43,9 +43,13 @@ class ClassificationModel extends NCModelFileAdapter("org/apache/nlpcraft/exampl
override def getMetadata: util.Map[String, AnyRef] = {
val md = super.getMetadata
- md.put("class:carBrand", mkMap(0.5, 0.5, 0.5, 0.5))
- md.put("class:animal", mkMap(0.5, 0.5, 0.5, 0.5))
- md.put("class:weather", mkMap(0.5, 0.5, 0.5, 0.5))
+ val factors = new java.util.HashMap[String, java.util.Map[String, Double]]()
+
+ factors.put("class:carBrand", mkMap(0.5, 0.5, 0.5, 0.5))
+ factors.put("class:animal", mkMap(0.5, 0.5, 0.5, 0.5))
+ factors.put("class:weather", mkMap(0.5, 0.5, 0.5, 0.5))
+
+ md.put("ctx.words.factors", factors)
md
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCContextWordConfigMdo.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCContextWordConfigMdo.scala
index 676a152..d464267 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCContextWordConfigMdo.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCContextWordConfigMdo.scala
@@ -35,4 +35,7 @@ case class NCContextWordConfigMdo(
@NCMdoField examples: Map[String /*Element ID*/, Seq[NCExampleMdo]/*Examples*/],
@NCMdoField poses: Set[String],/*All possible POSes of context words*/
@NCMdoField modelMeta: Map[String, AnyRef]
-)
\ No newline at end of file
+) {
+ require(synonyms.size == contextWords.size)
+ require(synonyms.size == examples.size)
+}
\ No newline at end of file
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
index 4e0f6a1..e3a9cf2 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
@@ -19,26 +19,15 @@ package org.apache.nlpcraft.server.nlp.enrichers.ctxword
import io.opencensus.trace.Span
import org.apache.nlpcraft.common.NCService
-import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote ⇒ Note, NCNlpSentenceToken ⇒ Token}
+import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote => Note, NCNlpSentenceToken => Token}
import org.apache.nlpcraft.server.ctxword.{NCContextWordManager, NCContextWordParameter, NCContextWordRequest, NCContextWordResponse}
-import org.apache.nlpcraft.server.mdo.{NCExampleMdo, NCContextWordConfigMdo ⇒ Config}
+import org.apache.nlpcraft.server.mdo.{NCExampleMdo, NCContextWordConfigMdo => Config}
import org.apache.nlpcraft.server.nlp.enrichers.NCServerEnricher
import org.jibx.schema.codegen.extend.DefaultNameConverter
-import scala.collection.Map
+import scala.collection._
object NCContextWordEnricher extends NCServerEnricher {
- // Configuration when we try to find context words for words nouns using initial sentence.
- private final val MIN_SENTENCE_SCORE = 0.5
- private final val MIN_SENTENCE_FTEXT = 0.5
-
- // Configuration when we try to find context words for words nouns using substituted examples.
- private final val MIN_EXAMPLE_SCORE = 0.8
- // Context words for all examples should satisfy it (not so strong)
- private final val MIN_EXAMPLE_ALL_FTEXT = 0.2
- // At least on context word with this score must be found.
- private final val MIN_EXAMPLE_BEST_FTEXT = 0.5
-
private final val CONVERTER = new DefaultNameConverter
private final val POS_PLURALS = Set("NNS", "NNPS")
@@ -94,13 +83,13 @@ object NCContextWordEnricher extends NCServerEnricher {
res
}
- private def trySentence(cfg: Config, toks: Seq[Token], ns: NCNlpSentence): Map[Token, Holder] = {
+ private def trySentence(cfg: Config, toks: Seq[Token], ns: NCNlpSentence, f: NCContextWordFactors): Map[Token, Holder] = {
val words = ns.tokens.map(_.origText)
val suggs =
NCContextWordManager.suggest(
toks.map(t ⇒ NCContextWordRequest(words, t.index)),
- NCContextWordParameter(totalScore = totalScore, ftextScore = ftextScore)
+ NCContextWordParameter(totalScore = f.getMinTotalSentence, ftextScore = f.getMinTotalExample)
)
require(toks.size == suggs.size)
@@ -110,7 +99,14 @@ object NCContextWordEnricher extends NCServerEnricher {
flatMap { case (tok, suggs) ⇒
suggs.sortBy(-_.totalScore).flatMap(sugg ⇒
cfg.contextWords.toStream.flatMap { case (elemId, stems) ⇒
- if (stems.contains(sugg.stem)) Some(tok → makeHolder(elemId, tok, sugg)) else None
+ if (
+ stems.contains(sugg.stem) &&
+ sugg.totalScore >= f.getTotalSentence(elemId) &&
+ sugg.ftextScore >= f.getFtextSentence(elemId)
+ )
+ Some(tok → makeHolder(elemId, tok, sugg))
+ else
+ None
}).headOption
}.toMap
@@ -119,7 +115,7 @@ object NCContextWordEnricher extends NCServerEnricher {
res
}
- private def tryExamples(cfg: Config, toks: Seq[Token], totalScore: Double, ftextScore: Double): Map[Token, Holder] = {
+ private def tryExamples(cfg: Config, toks: Seq[Token], f: NCContextWordFactors): Map[Token, Holder] = {
val examples = cfg.examples.toSeq
case class V(elementId: String, example: String, token: Token)
@@ -146,7 +142,7 @@ object NCContextWordEnricher extends NCServerEnricher {
val allSuggs =
NCContextWordManager.suggest(
allReqs.flatMap(_.requests),
- NCContextWordParameter(totalScore = totalScore, ftextScore = ftextScore / 2) // TODO:
+ NCContextWordParameter(totalScore = f.getMinTotalExample, ftextScore = f.getMinFtextExample / 2) // TODO:
)
require(allSuggs.size == allReqs.map(_.requests.size).sum)
@@ -166,12 +162,16 @@ object NCContextWordEnricher extends NCServerEnricher {
}
if (suggs.size == cfg.examples(elemId).size) {
- val best = suggs.toSeq.minBy(p ⇒ (-p.ftextScore, -p.totalScore))
-
- if (best.ftextScore >= ftextScore)
- Some(tok → makeHolder(elemId, tok, best))
- else
- None
+ suggs.toSeq.
+ filter(sugg ⇒
+ sugg.totalScore >= f.getTotalExample(elemId) &&
+ sugg.ftextScore >= f.getFtextExample(elemId)
+
+ ).
+ sortBy(p ⇒ (-p.ftextScore, -p.totalScore)).headOption match {
+ case Some(best) ⇒ Some(tok → makeHolder(elemId, tok, best))
+ case None ⇒ None
+ }
}
else
None
@@ -227,10 +227,12 @@ object NCContextWordEnricher extends NCServerEnricher {
def getOther: Seq[Token] = toks.filter(t ⇒ !m.contains(t))
if (m.size != toks.length) {
- m ++= trySentence(cfg, getOther, sen)
+ val factors = NCContextWordFactors(cfg.modelMeta, cfg.examples.keySet)
+
+ m ++= trySentence(cfg, getOther, sen, factors)
if (m.size != toks.length)
- m ++= tryExamples(cfg, getOther)
+ m ++= tryExamples(cfg, getOther, factors)
}
m.foreach {
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordFactors.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordFactors.scala
new file mode 100644
index 0000000..2c71e7e
--- /dev/null
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordFactors.scala
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.server.nlp.enrichers.ctxword
+
+import java.{lang, util}
+
+import com.typesafe.scalalogging.LazyLogging
+
+import scala.collection.JavaConverters._
+import scala.collection._
+
+object NCContextWordFactors {
+ // Configuration when we try to find context words for words nouns using initial sentence.
+ private final val MIN_SENTENCE_SCORE = 0.5
+ private final val MIN_SENTENCE_FTEXT = 0.5
+
+ // Configuration when we try to find context words for words nouns using substituted examples.
+ private final val MIN_EXAMPLE_SCORE = 0.8
+ private final val MIN_EXAMPLE_FTEXT = 0.5
+}
+
+import NCContextWordFactors._
+
+case class NCContextWordFactors(meta: Map[String, AnyRef], elementIds: Set[String]) extends LazyLogging {
+ private val (totalSen, totalExample, ftextSen, ftextExample) = initialize
+
+ private val minTotalSen: Double = minValue(totalSen)
+ private val minTotalExample: Double = minValue(totalExample)
+ private val minFtextSen: Double = minValue(ftextSen)
+ private val minFtextExample: Double = minValue(ftextExample)
+
+ private def initialize: (Map[String, Double], Map[String, Double], Map[String, Double], Map[String, Double]) = {
+ def mkMap(d: Double): mutable.Map[String, Double] =
+ mutable.HashMap.empty[String, Double] ++ elementIds.map(id ⇒ id → d).toMap
+
+ val totalSen = mkMap(MIN_SENTENCE_SCORE)
+ val totalExample = mkMap(MIN_EXAMPLE_SCORE)
+ val ftextSen = mkMap(MIN_SENTENCE_FTEXT)
+ val ftextExample = mkMap(MIN_EXAMPLE_FTEXT)
+
+ meta.get("ctx.words.factors") match {
+ case Some(v) ⇒
+ v.asInstanceOf[util.HashMap[String, util.Map[String, lang.Double]]].asScala.
+ foreach { case (elemId, factors) ⇒
+ if (elementIds.contains(elemId)) {
+ def set(name: String, m: mutable.Map[String, Double]): Unit = {
+ val v = factors.get(name)
+
+ if (v != null)
+ m += elemId → v
+ }
+
+ set("min.sentence.total.score", totalSen)
+ set("min.sentence.ftext.score", totalExample)
+ set("min.example.total.score", ftextSen)
+ set("min.example.ftext.score", ftextExample)
+ }
+ else
+ logger.warn(s"Unexpected element ID: '$elemId' data skipped.")
+ }
+
+ case None ⇒ // No-op.
+ }
+
+ (totalSen.toMap, totalExample.toMap, ftextSen.toMap, ftextExample.toMap)
+ }
+
+ private def minValue(m: Map[String, Double]): Double = m.values.min
+
+ def getMinTotalSentence: Double = minTotalSen
+ def getMinTotalExample: Double = minTotalExample
+ def getMinFtextSentence: Double = minFtextSen
+ def getMinFtextExample: Double = minFtextExample
+
+ def getTotalSentence(elemId: String): Double = totalSen(elemId)
+ def getTotalExample(elemId: String): Double = totalExample(elemId)
+ def getFtextSentence(elemId: String): Double = ftextSen(elemId)
+ def getFtextExample(elemId: String): Double = ftextExample(elemId)
+}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index b2bf652..fd3e6ac 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -614,7 +614,7 @@ object NCProbeManager extends NCService {
mlSyns.asScala.
map(p ⇒ p._1 → p._2.asScala.map(x ⇒ x._1 → x._2.asScala.toSet).toMap).toMap,
examples = examples.asScala.toSet,
- meta
+ meta.asScala.toMap
)
}.toSet
[incubator-nlpcraft] 01/02: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-70
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit daad8b5b83310030ee5666b391abdfd7aead3d15
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Thu Jun 25 00:36:38 2020 +0300
WIP.
---
.../classification/ClassificationModel.scala | 29 ++++++++++++++++
.../probe/mgrs/conn/NCConnectionManager.scala | 3 +-
.../server/ctxword/NCContextWordManager.scala | 16 +++++++--
.../server/ctxword/NCContextWordParameter.scala | 40 +++-------------------
.../server/mdo/NCContextWordConfigMdo.scala | 3 +-
.../enrichers/ctxword/NCContextWordEnricher.scala | 21 ++++++++----
.../nlpcraft/server/probe/NCProbeManager.scala | 14 +++++---
7 files changed, 74 insertions(+), 52 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/classification/ClassificationModel.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/classification/ClassificationModel.scala
index b8c445b..6c5183e 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/classification/ClassificationModel.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/classification/ClassificationModel.scala
@@ -17,10 +17,39 @@
package org.apache.nlpcraft.examples.cars
+import java.util
+
import org.apache.nlpcraft.model.{NCIntentTerm, _}
// TODO:
class ClassificationModel extends NCModelFileAdapter("org/apache/nlpcraft/examples/classification/classification_model.yaml") {
+ private def mkMap(
+ senTotalScore: Double,
+ senFtextScore: Double,
+ exampleTotalScore: Double,
+ exampleFtextScore: Double
+ ): java.util.Map[String, Double] ={
+ val m = new java.util.HashMap[String, Double]()
+
+ m.put("min.sentence.total.score", senTotalScore)
+ m.put("min.sentence.ftext.score", senFtextScore)
+ m.put("min.example.total.score", exampleTotalScore)
+ m.put("min.example.ftext.score", exampleFtextScore)
+
+ m
+ }
+
+ // Optional.
+ override def getMetadata: util.Map[String, AnyRef] = {
+ val md = super.getMetadata
+
+ md.put("class:carBrand", mkMap(0.5, 0.5, 0.5, 0.5))
+ md.put("class:animal", mkMap(0.5, 0.5, 0.5, 0.5))
+ md.put("class:weather", mkMap(0.5, 0.5, 0.5, 0.5))
+
+ md
+ }
+
@NCIntentRef("classification")
def onMatch(
@NCIntentTerm("brands") brands: Seq[NCToken],
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
index 15cfa3a..c1d83de 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
@@ -263,7 +263,8 @@ object NCConnectionManager extends NCService {
new util.HashMap[String, util.Map[String, util.Set[String]]](
ctxSyns.map(p ⇒ p._1 → p._2.map(x ⇒ x._1 → x._2.asJava).asJava).asJava
),
- new util.HashSet[String](mdl.getExamples)
+ new util.HashSet[String](mdl.getExamples),
+ mdl.getMetadata
)
})
), cryptoKey)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/ctxword/NCContextWordManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/ctxword/NCContextWordManager.scala
index aeedcb5..327cbbd 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/ctxword/NCContextWordManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/ctxword/NCContextWordManager.scala
@@ -35,7 +35,6 @@ import org.apache.nlpcraft.common.ascii.NCAsciiTable
import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.{NCE, NCService}
-import org.apache.nlpcraft.server.ctxword.NCContextWordParameter._
import org.apache.nlpcraft.server.ignite.NCIgniteInstance
import org.apache.nlpcraft.server.mdo.{NCContextWordConfigMdo, NCExampleMdo}
import org.apache.nlpcraft.server.nlp.core.{NCNlpParser, NCNlpServerManager, NCNlpWord}
@@ -49,6 +48,15 @@ import scala.util.control.Exception.catching
*
*/
object NCContextWordManager extends NCService with NCOpenCensusServerStats with NCIgniteInstance {
+ // Configuration request limit for each processed example.
+ private final val CONF_LIMIT = 1000
+ // Minimal score for requested words for each processed example.
+ private final val CONF_MIN_SCORE = 1
+ // If we have a lot of context words candidates, we choose top 50%.
+ private final val CONF_TOP_FACTOR = 0.5
+ // If we have small context words candidates count, we choose at least 3.
+ private final val CONF_TOP_MIN = 3
+
private object Config extends NCConfigurable {
lazy val url: Option[String] = getStringOpt("nlpcraft.server.ctxword.url")
}
@@ -248,7 +256,8 @@ object NCContextWordManager extends NCService with NCOpenCensusServerStats with
def makeConfig(
mdlId: String,
ctxSyns: Map[String /*Element ID*/ , Map[String /*Value*/, Set[String] /*Values synonyms stems*/ ]],
- examples: Set[String]
+ examples: Set[String],
+ modelMeta: Map[String, AnyRef]
): NCContextWordConfigMdo = {
val synonyms =
ctxSyns.map { case (elemId, map) ⇒ elemId → map.flatMap { case (value, syns) ⇒ syns.map(_ → value) } }
@@ -359,7 +368,8 @@ object NCContextWordManager extends NCService with NCOpenCensusServerStats with
synonyms,
groups.map { case (elemId, seq) ⇒ elemId → seq.map(_.group.word.stem).toSet },
examplesCfg.toMap,
- examplesCfg.values.flatten.flatMap(_.substitutions.map { case (_, pos) ⇒ pos } ).toSet
+ examplesCfg.values.flatten.flatMap(_.substitutions.map { case (_, pos) ⇒ pos } ).toSet,
+ modelMeta
)
}
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/ctxword/NCContextWordParameter.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/ctxword/NCContextWordParameter.scala
index edf42bb..35bcc74 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/ctxword/NCContextWordParameter.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/ctxword/NCContextWordParameter.scala
@@ -18,38 +18,6 @@
package org.apache.nlpcraft.server.ctxword
/**
- * All parameters.
- */
-object NCContextWordParameter {
- private final val DFLT_LIMIT = 20
- private final val DFLT_TOTAL_SCORE = 0
- private final val DFLT_FTEXT_SCORE = 0.25
- private final val DFLT_BERT_SCORE = 0
-
- // Configuration parameters.
- // Configuration request limit for each processed example.
- final val CONF_LIMIT = 1000
- // Minimal score for requested words for each processed example.
- final val CONF_MIN_SCORE = 1
- // If we have a lot of context words candidates, we choose top 50%.
- final val CONF_TOP_FACTOR = 0.5
- // If we have small context words candidates count, we choose at least 3.
- final val CONF_TOP_MIN = 3
-
- // Enricher parameters.
- // Configuration when we try to find context words for words nouns using initial sentence.
- final val MIN_SENTENCE_SCORE = 0.5
- final val MIN_SENTENCE_FTEXT = 0.5
-
- // Configuration when we try to find context words for words nouns using substituted examples.
- final val MIN_EXAMPLE_SCORE = 0.8
- // Context words for all examples should satisfy it (not so strong)
- final val MIN_EXAMPLE_ALL_FTEXT = 0.2
- // At least on context word with this score must be found.
- final val MIN_EXAMPLE_BEST_FTEXT = 0.5
-}
-
-/**
* Default ContextWord server parameters.
*
* @param limit
@@ -58,8 +26,8 @@ object NCContextWordParameter {
* @param bertScore
*/
case class NCContextWordParameter(
- limit: Int = NCContextWordParameter.DFLT_LIMIT,
- totalScore: Double = NCContextWordParameter.DFLT_TOTAL_SCORE,
- ftextScore: Double = NCContextWordParameter.DFLT_FTEXT_SCORE,
- bertScore: Double = NCContextWordParameter.DFLT_BERT_SCORE
+ limit: Int = 20,
+ totalScore: Double = 0,
+ ftextScore: Double = 0.25,
+ bertScore: Double = 0
)
\ No newline at end of file
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCContextWordConfigMdo.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCContextWordConfigMdo.scala
index d16b050..676a152 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCContextWordConfigMdo.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCContextWordConfigMdo.scala
@@ -33,5 +33,6 @@ case class NCContextWordConfigMdo(
@NCMdoField synonyms: Map[String /*Element ID*/, Map[String /*Synonym stem*/, String /*Value*/]],
@NCMdoField contextWords: Map[String /*Element ID*/, Set[String]/*Stems*/],
@NCMdoField examples: Map[String /*Element ID*/, Seq[NCExampleMdo]/*Examples*/],
- @NCMdoField poses: Set[String]/*All possible POSes of context words*/
+ @NCMdoField poses: Set[String],/*All possible POSes of context words*/
+ @NCMdoField modelMeta: Map[String, AnyRef]
)
\ No newline at end of file
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
index bdb9994..4e0f6a1 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/ctxword/NCContextWordEnricher.scala
@@ -23,13 +23,22 @@ import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote ⇒ Note
import org.apache.nlpcraft.server.ctxword.{NCContextWordManager, NCContextWordParameter, NCContextWordRequest, NCContextWordResponse}
import org.apache.nlpcraft.server.mdo.{NCExampleMdo, NCContextWordConfigMdo ⇒ Config}
import org.apache.nlpcraft.server.nlp.enrichers.NCServerEnricher
-import org.apache.nlpcraft.server.ctxword.NCContextWordParameter._
import org.jibx.schema.codegen.extend.DefaultNameConverter
-import org.apache.nlpcraft.server.ctxword.NCContextWordParameter._
import scala.collection.Map
object NCContextWordEnricher extends NCServerEnricher {
+ // Configuration when we try to find context words for words nouns using initial sentence.
+ private final val MIN_SENTENCE_SCORE = 0.5
+ private final val MIN_SENTENCE_FTEXT = 0.5
+
+ // Configuration when we try to find context words for words nouns using substituted examples.
+ private final val MIN_EXAMPLE_SCORE = 0.8
+ // Context words for all examples should satisfy it (not so strong)
+ private final val MIN_EXAMPLE_ALL_FTEXT = 0.2
+ // At least on context word with this score must be found.
+ private final val MIN_EXAMPLE_BEST_FTEXT = 0.5
+
private final val CONVERTER = new DefaultNameConverter
private final val POS_PLURALS = Set("NNS", "NNPS")
@@ -91,7 +100,7 @@ object NCContextWordEnricher extends NCServerEnricher {
val suggs =
NCContextWordManager.suggest(
toks.map(t ⇒ NCContextWordRequest(words, t.index)),
- NCContextWordParameter(totalScore = MIN_SENTENCE_SCORE, ftextScore = MIN_SENTENCE_FTEXT)
+ NCContextWordParameter(totalScore = totalScore, ftextScore = ftextScore)
)
require(toks.size == suggs.size)
@@ -110,7 +119,7 @@ object NCContextWordEnricher extends NCServerEnricher {
res
}
- private def tryExamples(cfg: Config, toks: Seq[Token]): Map[Token, Holder] = {
+ private def tryExamples(cfg: Config, toks: Seq[Token], totalScore: Double, ftextScore: Double): Map[Token, Holder] = {
val examples = cfg.examples.toSeq
case class V(elementId: String, example: String, token: Token)
@@ -137,7 +146,7 @@ object NCContextWordEnricher extends NCServerEnricher {
val allSuggs =
NCContextWordManager.suggest(
allReqs.flatMap(_.requests),
- NCContextWordParameter(totalScore = MIN_EXAMPLE_SCORE, ftextScore = MIN_EXAMPLE_ALL_FTEXT)
+ NCContextWordParameter(totalScore = totalScore, ftextScore = ftextScore / 2) // TODO:
)
require(allSuggs.size == allReqs.map(_.requests.size).sum)
@@ -159,7 +168,7 @@ object NCContextWordEnricher extends NCServerEnricher {
if (suggs.size == cfg.examples(elemId).size) {
val best = suggs.toSeq.minBy(p ⇒ (-p.ftextScore, -p.totalScore))
- if (best.ftextScore >= MIN_EXAMPLE_BEST_FTEXT)
+ if (best.ftextScore >= ftextScore)
Some(tok → makeHolder(elemId, tok, best))
else
None
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index 71e0c5a..b2bf652 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -582,7 +582,8 @@ object NCProbeManager extends NCService {
version: String,
enabledBuiltInTokens: Set[String],
ctxSynonyms: Map[String, Map[String, Set[String]]],
- examples: Set[String]
+ examples: Set[String],
+ meta: Map[String, AnyRef]
)
val models =
@@ -592,7 +593,8 @@ object NCProbeManager extends NCService {
String,
java.util.Set[String],
java.util.Map[String, java.util.Map[String, java.util.Set[String]]],
- java.util.Set[String]
+ java.util.Set[String],
+ java.util.Map[String, AnyRef]
)]]("PROBE_MODELS").
map { case (
mdlId,
@@ -600,7 +602,8 @@ object NCProbeManager extends NCService {
mdlVer,
enabledBuiltInToks,
mlSyns,
- examples
+ examples,
+ meta
) ⇒
ProbeModel(
id = mdlId,
@@ -610,7 +613,8 @@ object NCProbeManager extends NCService {
ctxSynonyms =
mlSyns.asScala.
map(p ⇒ p._1 → p._2.asScala.map(x ⇒ x._1 → x._2.asScala.toSet).toMap).toMap,
- examples = examples.asScala.toSet
+ examples = examples.asScala.toSet,
+ meta
)
}.toSet
@@ -634,7 +638,7 @@ object NCProbeManager extends NCService {
enabledBuiltInTokens = m.enabledBuiltInTokens,
ctxWordsConfig =
if (m.ctxSynonyms.nonEmpty)
- Some(NCContextWordManager.makeConfig(m.id, m.ctxSynonyms, m.examples))
+ Some(NCContextWordManager.makeConfig(m.id, m.ctxSynonyms, m.examples, m.meta))
else
None
)