You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2021/12/28 19:40:31 UTC
[incubator-nlpcraft] branch NLPCRAFT-471 updated (2b497ae -> 56a5894)
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a change to branch NLPCRAFT-471
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
from 2b497ae WIP.
add fe73434 WIP
add f97fa9a WIP.
add 2793cdf Update NCUtils.scala
new 957a4b6 Merge branch 'master' into NLPCRAFT-471
new 56a5894 WIP.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../scala/org/apache/nlpcraft/NCLifecycle.java | 2 +-
.../scala/org/apache/nlpcraft/NCModelClient.java | 18 ++---
.../scala/org/apache/nlpcraft/NCModelConfig.java | 2 +-
.../org/apache/nlpcraft/NCModelConfigAdapter.java | 86 ++++++++++++++++------
.../main/scala/org/apache/nlpcraft/NCRequest.java | 10 +--
.../main/scala/org/apache/nlpcraft/NCToken.java | 8 +-
.../parser/opennlp/NCOpenNlpEntityParser.java | 13 +++-
.../opennlp/impl/NCOpenNlpEntityParserImpl.scala | 4 +-
.../token/enricher/NCEnBracketsTokenEnricher.java | 4 +-
.../enricher/NCEnDictionaryTokenEnricher.java | 4 +-
.../token/enricher/NCEnLanguageTokenEnricher.java | 4 +-
.../token/enricher/NCEnQuotesTokenEnricher.java | 4 +-
.../enricher/NCEnSwearWordsTokenEnricher.java | 4 +-
.../nlp/token/enricher/impl/NCEnBracketsImpl.scala | 6 +-
.../token/enricher/impl/NCEnDictionaryImpl.scala | 2 +-
.../enricher/impl/NCEnLanguageWordsImpl.scala | 2 +-
.../nlp/token/enricher/impl/NCEnQuotesImpl.scala | 2 +-
.../token/enricher/impl/NCEnSwearWordsImpl.scala | 2 +-
.../parser/opennlp/NCEnOpenNlpTokenParser.java | 4 +-
.../parser/opennlp/impl/NCEnOpenNlpImpl.scala | 10 +--
.../parser/opennlp/impl/NCEnStopWordsFinder.scala | 8 +-
.../apache/nlpcraft/internal/util/NCUtils.scala | 4 +-
.../opennlp/NCEnOpenNlpTokenParserBenchmark.java | 2 +-
.../parser/opennlp/NCOpenNlpEntityParserSpec.scala | 1 -
.../opennlp/NCEnOpenNlpTokenParserSpec.scala | 4 +-
.../nlpcraft/internal/nlp/util/NCTestRequest.scala | 3 +-
.../nlpcraft/internal/nlp/util/NCTestToken.scala | 3 +-
.../nlpcraft/internal/nlp/util/NCTestUtils.scala | 14 ++--
28 files changed, 127 insertions(+), 103 deletions(-)
[incubator-nlpcraft] 01/02: Merge branch 'master' into NLPCRAFT-471
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-471
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 957a4b6418cdf835617f42f7b4325c80ae918209
Merge: 2b497ae 2793cdf
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Tue Dec 28 22:34:26 2021 +0300
Merge branch 'master' into NLPCRAFT-471
# Conflicts:
# nlpcraft/src/main/scala/org/apache/nlpcraft/NCModelClient.java
# nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestRequest.scala
# nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala
.../scala/org/apache/nlpcraft/NCLifecycle.java | 2 +-
.../scala/org/apache/nlpcraft/NCModelClient.java | 10 +--
.../scala/org/apache/nlpcraft/NCModelConfig.java | 2 +-
.../org/apache/nlpcraft/NCModelConfigAdapter.java | 86 ++++++++++++++++------
.../main/scala/org/apache/nlpcraft/NCRequest.java | 10 +--
.../main/scala/org/apache/nlpcraft/NCToken.java | 8 +-
.../token/enricher/NCEnBracketsTokenEnricher.java | 4 +-
.../enricher/NCEnDictionaryTokenEnricher.java | 4 +-
.../token/enricher/NCEnLanguageTokenEnricher.java | 4 +-
.../token/enricher/NCEnQuotesTokenEnricher.java | 4 +-
.../enricher/NCEnSwearWordsTokenEnricher.java | 4 +-
.../nlp/token/enricher/impl/NCEnBracketsImpl.scala | 6 +-
.../token/enricher/impl/NCEnDictionaryImpl.scala | 2 +-
.../enricher/impl/NCEnLanguageWordsImpl.scala | 2 +-
.../nlp/token/enricher/impl/NCEnQuotesImpl.scala | 2 +-
.../token/enricher/impl/NCEnSwearWordsImpl.scala | 2 +-
.../parser/opennlp/NCEnOpenNlpTokenParser.java | 4 +-
.../parser/opennlp/impl/NCEnOpenNlpImpl.scala | 10 +--
.../parser/opennlp/impl/NCEnStopWordsFinder.scala | 8 +-
.../apache/nlpcraft/internal/util/NCUtils.scala | 4 +-
.../opennlp/NCEnOpenNlpTokenParserBenchmark.java | 2 +-
.../opennlp/NCEnOpenNlpTokenParserSpec.scala | 4 +-
.../nlpcraft/internal/nlp/util/NCTestRequest.scala | 3 +-
.../nlpcraft/internal/nlp/util/NCTestToken.scala | 3 +-
.../nlpcraft/internal/nlp/util/NCTestUtils.scala | 11 ++-
25 files changed, 111 insertions(+), 90 deletions(-)
diff --cc nlpcraft/src/main/scala/org/apache/nlpcraft/NCModelClient.java
index 8dfc3f7,7d9b5dc..b70b2d1
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCModelClient.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCModelClient.java
@@@ -43,49 -43,24 +43,49 @@@ public class NCModelClient implements N
// TODO:
}
- private static void start(List<? extends NCLifecycle> list, NCModelConfig cfg) {
+ private static void start(ExecutorService s, List<? extends NCLifecycle> list) {
+ assert s != null;
+
if (list != null)
- list.forEach(p -> s.execute(() -> p.start()));
- list.forEach(p -> p.start(cfg));
++ list.forEach(p -> s.execute(() -> p.start(cfg)));
}
- private static void stop(List<? extends NCLifecycle> list) {
+ private static void stop(ExecutorService s, List<? extends NCLifecycle> list) {
+ assert s != null;
+
if (list != null)
- list.forEach(p -> p.stop());
+ list.forEach(p -> s.execute(() -> p.stop()));
+ }
+
+ private static void stopExecutorService(ExecutorService s) {
+ try {
+ s.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ }
+ catch (InterruptedException e) {
+ throw new NCException("Thread interrupted.", e);
+ }
+ }
+
+ private static ExecutorService getExecutorService() {
+ return Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
}
@Override
- public void start() {
+ public void start(NCModelConfig cfg) {
verify();
- start(cfg.getTokenParsers(), cfg);
- start(cfg.getEntityParsers(), cfg);
- start(cfg.getEntityEnrichers(), cfg);
- start(cfg.getTokenEnrichers(), cfg);
+ NCModelConfig cfg = mdl.getConfig();
+ ExecutorService s = getExecutorService();
+
+ try {
+ s.execute(() -> cfg.getTokenParser().start());
- start(s, cfg.getEntityParsers());
- start(s, cfg.getEntityEnrichers());
- start(s, cfg.getTokenEnrichers());
++ start(s, cfg.getEntityParsers(), cfg);
++ start(s, cfg.getEntityEnrichers(), cfg);
++ start(s, cfg.getTokenEnrichers(), cfg);
+ }
+ finally {
+ stopExecutorService(s);
+ }
}
@Override
diff --cc nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/impl/NCEnOpenNlpImpl.scala
index f12936c,097981f..07b4e3a
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/impl/NCEnOpenNlpImpl.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/impl/NCEnOpenNlpImpl.scala
@@@ -156,10 -156,9 +156,9 @@@ class NCEnOpenNlpImpl
(lemma, idx) => fixes.getOrElse(idx, lemma)
}
- val res: Seq[NCToken] = holders.zip(posTags).zip(lemmas).toIndexedSeq.map { case ((h, pos), lemma) =>
+ val res: Seq[NCToken] = holders.zip(posTags).zip(lemmas).toIndexedSeq.zipWithIndex.map { case (((h, pos), lemma), idx) =>
new NCPropertyMapAdapter with NCToken:
- override def getOriginalText: String = h.origin
- override def getNormalizedText: String = h.normalized
+ override def getText: String = h.origin
override def getLemma: String = lemma
override def getStem: String = stemmer.stem(h.normalized)
override def getPos: String = pos
diff --cc nlpcraft/src/main/scala/org/apache/nlpcraft/internal/util/NCUtils.scala
index 77bb9d7,b756e63..12875e4
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/util/NCUtils.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/util/NCUtils.scala
@@@ -870,9 -870,9 +870,9 @@@ object NCUtils extends LazyLogging
def readResource(res: String, enc: String = "UTF-8", log: Logger = logger): List[String] =
val list =
try
- Using.resource(Source.fromInputStream(getStream(res), enc))(_.getLines()).toList
+ Using.resource(Source.fromInputStream(getStream(res), enc))(_.getLines().toSeq).toList
catch
- case e: IOException => throw new NCException(s"Failed to read stream.", e)
+ case e: IOException => throw new NCException(s"Failed to read stream: $res", e)
log.trace(s"Loaded resource: $res")
diff --cc nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/benchmark/token/parser/opennlp/NCEnOpenNlpTokenParserBenchmark.java
index 974e826,193dd00..a91dc81
--- a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/benchmark/token/parser/opennlp/NCEnOpenNlpTokenParserBenchmark.java
+++ b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/benchmark/token/parser/opennlp/NCEnOpenNlpTokenParserBenchmark.java
@@@ -53,9 -52,13 +53,9 @@@ public class NCEnOpenNlpTokenParserBenc
* @return
*/
private static NCEnOpenNlpTokenParser prepareParser() {
- NCEnOpenNlpTokenParser p = new NCEnOpenNlpTokenParser(
- "opennlp/en-token.bin",
- "opennlp/en-pos-maxent.bin",
- "opennlp/en-lemmatizer.dict"
- );
+ NCEnOpenNlpTokenParser p = NCTestUtils.mkEnParser();
- p.start();
+ p.start(null); // TODO: fix it.
return p;
}
diff --cc nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala
index 25bb543,06accfd..6ef2feb
--- a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala
+++ b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala
@@@ -27,27 -25,44 +27,26 @@@ import scala.jdk.CollectionConverters.
*/
object NCTestUtils:
/**
- *
* @param toks
- * @param props
*/
- def printTokens(toks: Seq[NCToken], props: String*): Unit =
+ def printTokens(toks: Seq[NCToken]): Unit =
val tbl = new NCAsciiTable()
- tbl #= ("Origin", "Normalized", "POS", "Stem", "Lemma", "Start", "End", "Length", "Stopword", "Properties")
- if props.isEmpty
- then tbl #= ("Text", "POS", "Stem", "Lemma", "Start", "End", "Length", "Stopword")
- else tbl #= ("Text", "POS", "Stem", "Lemma", "Start", "End", "Length", "Stopword", "Properties")
-
- toks.foreach(t =>
- if props.isEmpty then
- tbl += (
- t.getText,
- t.getPos,
- t.getStem,
- t.getLemma,
- t.getStartCharIndex,
- t.getEndCharIndex,
- t.getLength,
- t.isStopWord
- )
- else
- tbl += (
- t.getText,
- t.getPos,
- t.getStem,
- t.getLemma,
- t.getStartCharIndex,
- t.getEndCharIndex,
- t.getLength,
- t.isStopWord,
- props.map(p => s"$p=${t.get[Any](p)}").mkString("{", ", ", "}")
- )
- )
++ tbl #= ("Text", "POS", "Stem", "Lemma", "Start", "End", "Length", "Stopword", "Properties")
+ for (t <- toks)
+ tbl += (
- t.getOriginalText,
- t.getNormalizedText,
++ t.getText,
+ t.getPos,
+ t.getStem,
+ t.getLemma,
+ t.getStartCharIndex,
+ t.getEndCharIndex,
+ t.getLength,
+ t.isStopWord,
+ t.keysSet().asScala.map(p => s"$p=${t.get[Any](p)}").mkString("[", ", ", "]")
+ )
- println(s"Request: ${toks.map(_.getOriginalText).mkString(" ")}")
+ println(s"Request: ${toks.map(_.getText).mkString(" ")}")
println(tbl.toString)
/**
@@@ -82,16 -78,6 +81,16 @@@
val t = make
val started = now()
- t.start()
+ t.start(null) // TODO: fix it.
println(s"'${t.getClass.getSimpleName}' created with time=${started - start} ms and started=${now() - started} ms.")
t
+
+ /**
- *
++ *
+ * @return
+ */
+ def mkEnParser: NCEnOpenNlpTokenParser = new NCEnOpenNlpTokenParser(
+ "opennlp/en-token.bin",
+ "opennlp/en-pos-maxent.bin",
+ "opennlp/en-lemmatizer.dict"
+ )
[incubator-nlpcraft] 02/02: WIP.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-471
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 56a5894f4acaf9d90ccf990b9bc3b9981b1a0b29
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Tue Dec 28 22:40:20 2021 +0300
WIP.
---
.../src/main/scala/org/apache/nlpcraft/NCModelClient.java | 8 +++-----
.../nlp/entity/parser/opennlp/NCOpenNlpEntityParser.java | 13 +++++++++----
.../parser/opennlp/impl/NCOpenNlpEntityParserImpl.scala | 4 ++--
.../entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala | 1 -
.../token/parser/opennlp/NCEnOpenNlpTokenParserSpec.scala | 2 +-
.../org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala | 5 +++--
6 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCModelClient.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCModelClient.java
index b70b2d1..603a481 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/NCModelClient.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/NCModelClient.java
@@ -43,7 +43,7 @@ public class NCModelClient implements NCLifecycle {
// TODO:
}
- private static void start(ExecutorService s, List<? extends NCLifecycle> list) {
+ private static void start(ExecutorService s, List<? extends NCLifecycle> list, NCModelConfig cfg) {
assert s != null;
if (list != null)
@@ -74,11 +74,10 @@ public class NCModelClient implements NCLifecycle {
public void start(NCModelConfig cfg) {
verify();
- NCModelConfig cfg = mdl.getConfig();
ExecutorService s = getExecutorService();
try {
- s.execute(() -> cfg.getTokenParser().start());
+ start(s, cfg.getTokenParsers(), cfg);
start(s, cfg.getEntityParsers(), cfg);
start(s, cfg.getEntityEnrichers(), cfg);
start(s, cfg.getTokenEnrichers(), cfg);
@@ -97,8 +96,7 @@ public class NCModelClient implements NCLifecycle {
stop(s, cfg.getTokenEnrichers());
stop(s, cfg.getEntityEnrichers());
stop(s, cfg.getEntityParsers());
- s.execute(() -> cfg.getTokenParser().stop());
-
+ stop(s, cfg.getTokenEnrichers());
}
finally {
stopExecutorService(s);
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/entity/parser/opennlp/NCOpenNlpEntityParser.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/entity/parser/opennlp/NCOpenNlpEntityParser.java
index 3425af8..b02a6f8 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/entity/parser/opennlp/NCOpenNlpEntityParser.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/entity/parser/opennlp/NCOpenNlpEntityParser.java
@@ -17,11 +17,16 @@
package org.apache.nlpcraft.internal.nlp.entity.parser.opennlp;
-import org.apache.nlpcraft.*;
+import org.apache.nlpcraft.NCEntity;
+import org.apache.nlpcraft.NCEntityParser;
+import org.apache.nlpcraft.NCModelConfig;
+import org.apache.nlpcraft.NCRequest;
+import org.apache.nlpcraft.NCToken;
import org.apache.nlpcraft.internal.nlp.entity.parser.opennlp.impl.NCOpenNlpEntityParserImpl;
import java.io.File;
-import java.util.*;
+import java.util.List;
+import java.util.Objects;
/**
* Generates entities with
@@ -58,8 +63,8 @@ public class NCOpenNlpEntityParser implements NCEntityParser {
}
@Override
- public void start() {
- impl.start();
+ public void start(NCModelConfig cfg) {
+ impl.start(cfg);
}
@Override
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/entity/parser/opennlp/impl/NCOpenNlpEntityParserImpl.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/entity/parser/opennlp/impl/NCOpenNlpEntityParserImpl.scala
index 588923a..70a59bf 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/entity/parser/opennlp/impl/NCOpenNlpEntityParserImpl.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/internal/nlp/entity/parser/opennlp/impl/NCOpenNlpEntityParserImpl.scala
@@ -46,7 +46,7 @@ class NCOpenNlpEntityParserImpl(is: InputStream, res: String) extends NCEntityPa
private case class Holder(start: Int, end: Int, name: String, probability: Double)
- override def start(): Unit =
+ override def start(cfg: NCModelConfig): Unit =
finder = new NameFinderME(new TokenNameFinderModel(NCUtils.getStream(res)))
logger.trace(s"Loaded resource: $res")
@@ -63,7 +63,7 @@ class NCOpenNlpEntityParserImpl(is: InputStream, res: String) extends NCEntityPa
override def parse(req: NCRequest, cfg: NCModelConfig, toks: JList[NCToken]): JList[NCEntity] =
val toksSeq = toks.asScala
- find(toksSeq.map(_.getOriginalText).toArray).flatMap(h =>
+ find(toksSeq.map(_.getText).toArray).flatMap(h =>
def calcIndex(getHolderIndex: Holder => Int): Int =
toksSeq.find(_.getIndex == getHolderIndex(h)) match
case Some(t) => t.getIndex
diff --git a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala
index 0192085..f4762cc 100644
--- a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala
+++ b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/entity/parser/opennlp/NCOpenNlpEntityParserSpec.scala
@@ -62,7 +62,6 @@ class NCOpenNlpEntityParserSpec:
require(resSeq.size == 1)
val res = resSeq.head
-
NCTestUtils.printEntities(txt, res)
require(res.exists(_.getOpt(s"opennlp:$expected:probability").isPresent))
diff --git a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/NCEnOpenNlpTokenParserSpec.scala b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/NCEnOpenNlpTokenParserSpec.scala
index 6eae9d3..3ba6869 100644
--- a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/NCEnOpenNlpTokenParserSpec.scala
+++ b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/token/parser/opennlp/NCEnOpenNlpTokenParserSpec.scala
@@ -87,5 +87,5 @@ class NCEnOpenNlpTokenParserSpec:
test(
// Nested brackets.
"< < [ A ] > >",
- toks => require(!toks.find(_.getText == "a").get.isStopWord)
+ toks => require(!toks.find(_.getText == "A").get.isStopWord)
)
diff --git a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala
index 6ef2feb..0afda8a 100644
--- a/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala
+++ b/nlpcraft/src/test/java/org/apache/nlpcraft/internal/nlp/util/NCTestUtils.scala
@@ -32,10 +32,11 @@ object NCTestUtils:
def printTokens(toks: Seq[NCToken]): Unit =
val tbl = new NCAsciiTable()
- tbl #= ("Text", "POS", "Stem", "Lemma", "Start", "End", "Length", "Stopword", "Properties")
+ tbl #= ("Text", "Index", "POS", "Stem", "Lemma", "Start", "End", "Length", "Stopword", "Properties")
for (t <- toks)
tbl += (
t.getText,
+ t.getIndex,
t.getPos,
t.getStem,
t.getLemma,
@@ -61,7 +62,7 @@ object NCTestUtils:
for (e <- ents)
tbl += (
e.getId,
- e.getTokens.asScala.map(_.getOriginalText).mkString("|"),
+ e.getTokens.asScala.map(_.getText).mkString("|"),
e.keysSet().asScala.map(p => s"$p=${e.get[Any](p)}").mkString("{", ", ", "}")
)