You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2021/05/30 11:14:51 UTC
[incubator-nlpcraft] branch kamov/NLPCRAFT-296 created (now 679db5a)
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a change to branch kamov/NLPCRAFT-296
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
at 679db5a WIP.
This branch includes the following new commits:
new 679db5a WIP.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-nlpcraft] 01/01: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch kamov/NLPCRAFT-296
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 679db5af91174d3e2a454df8cb1c9533a530c284
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Sun May 30 14:14:41 2021 +0300
WIP.
---
nlpcraft/pom.xml | 8 +++----
.../org/apache/nlpcraft/common/util/NCUtils.scala | 2 +-
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 2 +-
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 2 +-
.../probe/mgrs/sentence/NCSentenceManager.scala | 10 +++++---
.../enrichers/stopword/NCStopWordEnricher.scala | 27 ++++++++++++++--------
.../server/sugsyn/NCSuggestSynonymManager.scala | 4 ++--
.../nlpcraft/common/ascii/NCAsciiTableSpec.scala | 1 +
.../compiler/functions/NCIdlFunctionsDate.scala | 1 +
.../apache/nlpcraft/server/rest/NCRestSpec.scala | 2 +-
pom.xml | 20 ++++++++--------
11 files changed, 47 insertions(+), 32 deletions(-)
diff --git a/nlpcraft/pom.xml b/nlpcraft/pom.xml
index 779a4ba..4d8c292 100644
--- a/nlpcraft/pom.xml
+++ b/nlpcraft/pom.xml
@@ -190,15 +190,15 @@
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
- <artifactId>akka-http_2.12</artifactId>
+ <artifactId>akka-http_2.13</artifactId>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
- <artifactId>akka-stream_2.12</artifactId>
+ <artifactId>akka-stream_2.13</artifactId>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
- <artifactId>akka-http-spray-json_2.12</artifactId>
+ <artifactId>akka-http-spray-json_2.13</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
@@ -210,7 +210,7 @@
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_2.12</artifactId>
+ <artifactId>jackson-module-scala_2.13</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
index 3e7b046..c698e10 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
@@ -1016,7 +1016,7 @@ object NCUtils extends LazyLogging {
*/
def asFuture[T](
body: Unit => T,
- onFailure: Throwable => Unit = _ => Unit,
+ onFailure: Throwable => Unit = _ => (),
onSuccess: T => Unit = (_: T) => ())(implicit ec: ExecutionContext): Future[T] = {
val fut = Future {
body(())
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
index 8a88008..1f96350 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
@@ -101,7 +101,7 @@ object NCCommandManager extends NCService {
txt = msg.data[String]("txt"),
nlpSens = msg.data[JList[NCNlpSentence]]("nlpSens").asScala,
usrId = msg.data[Long]("userId"),
- senMeta = msg.data[java.util.Map[String, JSerializable]]("senMeta").asScala,
+ senMeta = msg.data[java.util.Map[String, JSerializable]]("senMeta").asScala.toMap,
mdlId = msg.data[String]("mdlId"),
enableLog = msg.data[Boolean]("enableLog"),
span
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
index 6b92656..1b10224 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
@@ -286,7 +286,7 @@ object NCModelEnricher extends NCProbeEnricher {
elem = mdl.elements.getOrElse(elemId, throw new NCE(s"Custom model parser returned unknown element ID: $elemId")),
toks = matchedToks,
direct = true,
- metaOpt = Some(e.getMetadata.asScala)
+ metaOpt = Some(e.getMetadata.asScala.toMap)
)
})
}
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala
index 4c49d1d..63913ac 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala
@@ -28,6 +28,7 @@ import java.io.{Serializable => JSerializable}
import java.util
import java.util.{List => JList}
import scala.collection.mutable
+import scala.collection.parallel.CollectionConverters.ImmutableIterableIsParallelizable
import scala.jdk.CollectionConverters.{ListHasAsScala, SeqHasAsJava, SetHasAsJava}
import scala.language.implicitConversions
@@ -655,7 +656,7 @@ object NCSentenceManager extends NCService {
distinct
// Optimization. Deletes all wholly swallowed notes.
- val links = getLinks(sen.flatten)
+ val links = getLinks(sen.tokens.flatMap(p => p))
val swallowed =
delCombs.
@@ -690,13 +691,16 @@ object NCSentenceManager extends NCService {
groupBy { case (idx, _) => idx }.
map { case (_, seq) => seq.map { case (_, note) => note }.toSet }.
toSeq.sortBy(-_.size)
+
+ def findCombinations(): Seq[Seq[NCNlpSentenceNote]] =
+ NCSentenceHelper.findCombinations(toksByIdx.map(_.asJava).asJava, pool).asScala.map(_.asScala.toSeq)
val seqSens =
combCache.
getOrElseUpdate(sen.srvReqId, mutable.HashMap.empty[CacheKey, CacheValue]).
getOrElseUpdate(
toksByIdx,
- NCSentenceHelper.findCombinations(toksByIdx.map(_.asJava).asJava, pool).asScala.map(_.asScala)
+ findCombinations()
).par.
flatMap(delComb => {
val nsClone = sen.clone()
@@ -721,7 +725,7 @@ object NCSentenceManager extends NCService {
Holder(
// We have to delete some keys to have possibility to compare sentences.
- notes.map(_.clone().filter { case (name, _) => name != "direct" }),
+ notes.map(_.clone().filter { case (name, _) => name != "direct" }).toSeq,
sen,
notes.filter(_.isNlp).map(p => if (p.isDirect) 0 else 1).sum
)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/stopword/NCStopWordEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/stopword/NCStopWordEnricher.scala
index 0762677..725a737 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/stopword/NCStopWordEnricher.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/stopword/NCStopWordEnricher.scala
@@ -232,15 +232,24 @@ object NCStopWordEnricher extends NCServerEnricher {
}
}
- def mkEntry[T](f: WordForm, mkT: Unit => T, isExc: Boolean):((Boolean, WordForm), T) = (isExc, f) -> mkT(())
- def mkMap[T](mkT: Unit => T): Map[(Boolean, WordForm), T] =
- WordForm
- .values
- .toSet
- .flatMap(f => Map(
- mkEntry(f, mkT, isExc = true),
- mkEntry(f, mkT, isExc = false)
- )).toMap
+ type Key = (Boolean, WordForm)
+ def mkMap[T](mkT: Unit => T): Map[Key, T] = {
+ val m = mutable.Map.empty[Key, T]
+
+ def add(f: WordForm, mkT: Unit => T, isExc: Boolean): Unit = {
+ val tuple: (Key, T) = (isExc, f) -> mkT(())
+
+ m += tuple._1 → tuple._2
+ }
+
+ WordForm.values.foreach(f => {
+ add(f, mkT, isExc = true)
+ add(f, mkT, isExc = false)
+
+ })
+
+ m.toMap
+ }
// Prepares collections.
val mHash = mkMap(_ => new Condition[Word]())
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
index cd3777b..47f8171 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala
@@ -38,7 +38,7 @@ import java.util.concurrent._
import java.util.concurrent.atomic.{AtomicInteger, AtomicReference}
import scala.collection.mutable
import scala.concurrent.{ExecutionContext, Future, Promise}
-import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsScala, SeqHasAsJava}
+import scala.jdk.CollectionConverters._
import scala.util.{Failure, Success}
/**
@@ -411,7 +411,7 @@ object NCSuggestSynonymManager extends NCService {
m
}).asJava
- }.asJava
+ }.toMap.asJava
promise.success(
NCSuggestSynonymResult(
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/common/ascii/NCAsciiTableSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/common/ascii/NCAsciiTableSpec.scala
index a3bf00f..f914eaa 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/common/ascii/NCAsciiTableSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/common/ascii/NCAsciiTableSpec.scala
@@ -17,6 +17,7 @@
package org.apache.nlpcraft.common.ascii
+import org.apache.nlpcraft.common.U
import org.apache.nlpcraft.common.ansi.NCAnsi._
import org.junit.jupiter.api.Test
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/compiler/functions/NCIdlFunctionsDate.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/compiler/functions/NCIdlFunctionsDate.scala
index 0bac73e..97a149b 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/compiler/functions/NCIdlFunctionsDate.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/compiler/functions/NCIdlFunctionsDate.scala
@@ -17,6 +17,7 @@
package org.apache.nlpcraft.model.intent.idl.compiler.functions
+import org.apache.nlpcraft.common.U
import org.junit.jupiter.api.Test
import java.time.temporal.IsoFields
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestSpec.scala
index 948e5ee..03bf811 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestSpec.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestSpec.scala
@@ -130,7 +130,7 @@ import org.apache.nlpcraft.server.rest.NCRestSpec._
class NCRestSpec extends NCTestContext {
type ResponseContent = java.util.Map[String, Object]
type ResponseList = JList[ResponseContent]
- type JList[T] = JList[T]
+ type JList[T] = java.util.List[T]
protected var tkn: String = _
/**
diff --git a/pom.xml b/pom.xml
index bf77506..be9d907 100644
--- a/pom.xml
+++ b/pom.xml
@@ -128,9 +128,9 @@
<maven.clean.plugin.ver>3.1.0</maven.clean.plugin.ver>
<apache.rat.plugin.ver>0.13</apache.rat.plugin.ver>
<org.antlr4.ver>4.9</org.antlr4.ver>
- <akka.http.2.12.ver>10.2.0</akka.http.2.12.ver>
- <akka.stream.2.12.ver>2.6.8</akka.stream.2.12.ver>
- <akka.http.spray.json.2.12.ver>10.2.0</akka.http.spray.json.2.12.ver>
+ <akka.http.2.13.ver>10.2.4</akka.http.2.13.ver>
+ <akka.stream.2.13.ver>2.6.14</akka.stream.2.13.ver>
+ <akka.http.spray.json.2.13.ver>10.2.4</akka.http.spray.json.2.13.ver>
<google.guava.ver>29.0-jre</google.guava.ver>
<diff.utils.ver>4.7</diff.utils.ver>
<vertical.blank.ver>1.0.1</vertical.blank.ver>
@@ -409,20 +409,20 @@
<dependency>
<groupId>com.typesafe.akka</groupId>
- <artifactId>akka-http_2.12</artifactId>
- <version>${akka.http.2.12.ver}</version>
+ <artifactId>akka-http_2.13</artifactId>
+ <version>${akka.http.2.13.ver}</version>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
- <artifactId>akka-stream_2.12</artifactId>
- <version>${akka.stream.2.12.ver}</version>
+ <artifactId>akka-stream_2.13</artifactId>
+ <version>${akka.stream.2.13.ver}</version>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
- <artifactId>akka-http-spray-json_2.12</artifactId>
- <version>${akka.http.spray.json.2.12.ver}</version>
+ <artifactId>akka-http-spray-json_2.13</artifactId>
+ <version>${akka.http.spray.json.2.13.ver}</version>
</dependency>
<dependency>
@@ -439,7 +439,7 @@
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_2.12</artifactId>
+ <artifactId>jackson-module-scala_2.13</artifactId>
<version>${jackson.ver}</version>
</dependency>