You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2021/07/27 13:32:52 UTC
[incubator-nlpcraft] 01/02: Merge branch 'master' into
NLPCRAFT-70_NEW
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-70_NEW
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 41011383913c2fc6e113e7df09aa3f764414c93a
Merge: 26fbef2 251364a
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Tue Jul 27 16:29:20 2021 +0300
Merge branch 'master' into NLPCRAFT-70_NEW
# Conflicts:
# nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
# nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala
# nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
# nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
hs_err_pid98656.log | 3856 ++++++++++++++++++++
.../apache/nlpcraft/examples/alarm/AlarmModel.java | 4 -
.../alarm/src/main/resources/alarm_samples.txt | 2 +-
.../src/main/resources/lightswitch_model.yaml | 2 +
.../nlpcraft/example/minecraft/MinecraftModel.kt | 17 +-
.../minecraft/src/main/resources/minecraft.yaml | 2 +
.../sql/src/main/resources/sql_model.yaml | 2 +
.../apache/nlpcraft/examples/time/TimeModel.java | 2 -
.../nlpcraft/examples/weather/WeatherModel.java | 4 -
.../org/apache/nlpcraft/common/NCService.scala | 17 +-
.../nlpcraft/common/config/NCConfigurable.scala | 2 +-
.../apache/nlpcraft/common/debug/NCLogHolder.scala | 5 +-
.../org/apache/nlpcraft/common/util/NCUtils.scala | 4 +
.../apache/nlpcraft/model/NCDialogFlowItem.java | 13 -
.../org/apache/nlpcraft/model/NCIntentMatch.java | 15 -
.../org/apache/nlpcraft/model/NCModelView.java | 5 +
.../scala/org/apache/nlpcraft/model/NCToken.java | 9 +
.../scala/org/apache/nlpcraft/model/NCVariant.java | 8 +-
.../apache/nlpcraft/model/intent/NCIdlIntent.scala | 9 +-
.../nlpcraft/model/intent/NCIdlIntentOptions.scala | 40 +
.../apache/nlpcraft/model/intent/NCIdlTerm.scala | 2 +-
.../model/intent/compiler/NCIdlCompiler.scala | 55 +-
.../nlpcraft/model/intent/compiler/antlr4/NCIdl.g4 | 8 +-
.../model/intent/compiler/antlr4/NCIdl.interp | 8 +-
.../model/intent/compiler/antlr4/NCIdl.tokens | 4 +-
.../intent/compiler/antlr4/NCIdlBaseListener.java | 18 +-
.../model/intent/compiler/antlr4/NCIdlLexer.interp | 8 +-
.../model/intent/compiler/antlr4/NCIdlLexer.java | 16 +-
.../model/intent/compiler/antlr4/NCIdlLexer.tokens | 4 +-
.../intent/compiler/antlr4/NCIdlListener.java | 22 +-
.../model/intent/compiler/antlr4/NCIdlParser.java | 240 +-
.../model/intent/solver/NCIntentSolver.scala | 1 -
.../model/intent/solver/NCIntentSolverEngine.scala | 143 +-
.../model/intent/solver/NCIntentSolverResult.scala | 6 +-
.../nlpcraft/model/tools/cmdline/NCCli.scala | 114 +-
.../nlpcraft/model/tools/cmdline/NCCliBase.scala | 9 +-
.../model/tools/cmdline/NCCliCommands.scala | 150 +-
.../model/tools/cmdline/NCCliRestSpec.scala | 19 +
.../model/tools/test/NCTestClientBuilder.java | 22 +-
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 247 +-
.../probe/mgrs/conn/NCConnectionManager.scala | 7 +-
.../probe/mgrs/conversation/NCConversation.scala | 48 +-
.../probe/mgrs/deploy/NCDeployManager.scala | 8 +-
.../mgrs/dialogflow/NCDialogFlowManager.scala | 58 +-
.../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 12 +-
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 18 +-
.../server/ignite/NCIgniteCacheStore.scala | 2 +-
.../nlpcraft/server/mdo/NCProbeModelMdo.scala | 1 +
.../opencensus/NCOpenCensusServerStats.scala | 7 +-
.../nlpcraft/server/probe/NCProbeManager.scala | 162 +-
.../nlpcraft/server/query/NCQueryManager.scala | 2 +-
.../nlpcraft/server/rest/NCBasicRestApi.scala | 229 +-
.../server/sugsyn/NCSuggestSynonymManager.scala | 38 +-
.../nlpcraft/server/user/NCUserManager.scala | 2 +-
nlpcraft/src/test/resources/log4j2.xml | 2 +-
.../scala/org/apache/nlpcraft/NCTestContext.scala | 7 +-
.../scala/org/apache/nlpcraft/NCTestElement.scala | 20 +-
.../nlpcraft/model/NCTrueIntentIdlSpec.scala | 2 +-
.../intent/idl/compiler/NCIdlCompilerSpec.scala | 71 +-
.../model/intent/idl/options/NCFreeWordsSpec.scala | 75 +
.../model/intent/idl/options/NCOrderedSpec.scala | 75 +
.../model/intent/idl/options/NCStmOnlySpec.scala | 130 +
.../model/intent/idl/options/NCSysTokensSpec.scala | 75 +
.../intent/idl/options/NCUserTokensSpec.scala | 75 +
.../model/properties/NCTokensPropertiesSpec.scala | 68 +-
.../mgrs/nlp/enrichers/NCEnricherBaseSpec.scala | 4 +-
.../nlpcraft/server/rest/NCRestModelSpec.scala | 75 +-
.../apache/nlpcraft/server/rest/NCRestSpec.scala | 1 -
.../nlpcraft/server/rest/RestTestModel.scala | 5 +-
openapi/nlpcraft_swagger.yml | 128 +-
70 files changed, 5912 insertions(+), 609 deletions(-)
diff --cc nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
index cdcb7a4,d2a4619..6a7cffb
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala
@@@ -31,11 -30,11 +30,10 @@@ import org.apache.nlpcraft.probe.mgrs.m
import java.io.{EOFException, IOException, InterruptedIOException}
import java.net.{InetAddress, NetworkInterface}
- import java.{lang, util}
+ import java.util
import java.util.concurrent.CountDownLatch
- import java.util.{Collections, Properties, TimeZone}
+ import java.util.{Properties, TimeZone}
import scala.collection.mutable
- import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, SeqHasAsJava, SetHasAsJava, SetHasAsScala}
-import scala.jdk.CollectionConverters.{SetHasAsJava, SetHasAsScala}
/**
* Probe down/up link connection manager.
@@@ -269,9 -223,7 +267,10 @@@ object NCConnectionManager extends NCSe
mdl.getName,
mdl.getVersion,
new util.HashSet[String](mdl.getEnabledBuiltInTokens),
- new util.HashSet[String](mdl.getElements.asScala.map(_.getId).asJava)
++ new util.HashSet[String](mdl.getElements.asScala.map(_.getId).asJava),
+ singleValues,
+ corpus,
+ categoriesElements
)
})
), cryptoKey)
diff --cc nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala
index 8a538ab,1b6001b..a727912
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala
@@@ -36,7 -28,7 +36,8 @@@ case class NCProbeModelMdo
@NCMdoField name: String,
@NCMdoField version: String,
@NCMdoField enabledBuiltInTokens: Set[String],
- @NCMdoField elementIds: Set[String]
++ @NCMdoField elementIds: Set[String],
+ @NCMdoField ctxWordConfig: Option[NCCtxWordCategoriesConfigMdo]
) extends NCAnnotatedMdo[NCProbeModelMdo] {
override def hashCode(): Int = s"$id$name".hashCode()
diff --cc nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index a18ad80,92238a2..3abd5cb
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@@ -619,9 -621,7 +626,10 @@@ object NCProbeManager extends NCServic
String,
String,
java.util.Set[String],
- java.util.Set[String]
++ java.util.Set[String],
+ java.util.Map[String, java.util.Map[String, java.util.Set[String]]],
+ java.util.Set[String],
+ java.util.Map[String, Double]
)]]("PROBE_MODELS").
map {
case (
@@@ -629,41 -629,20 +637,44 @@@
mdlName,
mdlVer,
enabledBuiltInToks,
- elmIds
++ elmIds,
+ singleValues,
+ corpus,
+ categoriesElements
) =>
require(mdlId != null)
require(mdlName != null)
require(mdlVer != null)
require(enabledBuiltInToks != null)
+ require(elmIds != null)
+ require(singleValues.isEmpty && corpus.isEmpty || !singleValues.isEmpty && !corpus.isEmpty)
NCProbeModelMdo(
id = mdlId,
name = mdlName,
version = mdlVer,
enabledBuiltInTokens = enabledBuiltInToks.asScala.toSet,
- elementIds = elmIds.asScala.toSet
++ elementIds = elmIds.asScala.toSet,
+ ctxWordConfig =
+ if (!singleValues.isEmpty) {
+ Some(
+ NCCtxWordCategoriesConfigMdo(
+ probeId = probeId,
+ modelId = mdlId,
+ singleValues = singleValues.asScala.map {
+ case (elemId, map) =>
+ elemId ->
+ map.asScala.map {
+ case (value, syns) => value -> syns.asScala.toSet
+ }.toMap
+ }.toMap,
+ corpus = corpus.asScala.toSet,
+ elements = categoriesElements.asScala.toMap
+ )
+ )
+ }
+ else
+ None
)
}.toSet
@@@ -714,7 -693,24 +725,24 @@@
}
}
}
-
+
+ /**
+ *
+ * @param probeMsg
+ * @param m
+ */
+ private def processJavaMetaMessage(probeMsg: NCProbeMessage, m: ConcurrentHashMap[String, Promise[JavaMeta]]): Unit = {
+ val p = m.remove(probeMsg.data[String]("reqGuid"))
+
+ if (p != null)
+ probeMsg.dataOpt[String]("resp") match {
+ case Some(resp) => p.success(GSON.fromJson(resp, TYPE_JAVA_META))
+ case None => p.failure(new NCE(probeMsg.data[String]("error")))
+ }
+ else
+ logger.warn(s"Message ignored: $probeMsg")
+ }
-
++
/**
* Processes the messages received from the probe.
*
diff --cc nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala
index e9d6b43,a12a4e8..301bc59
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala
@@@ -270,9 -270,9 +270,9 @@@ object NCQueryManager extends NCServic
tbl += (s"${b("Server Request ID")}", m(srvReqId))
tbl += (s"${b("Data")}", U.prettyJson(data.orNull).split("\n").toSeq)
- logger.info(s"New request received:\n$tbl")
+ logger.info(s"New user request received:\n$tbl")
- val enabledBuiltInToks = NCProbeManager.getModel(mdlId, span).enabledBuiltInTokens
+ val mdl = NCProbeManager.getModel(mdlId, span)
@throws[NCE]
def unzipProperties(gzipOpt: Option[String]): Option[JavaMeta] =