You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by ar...@apache.org on 2020/03/19 20:34:26 UTC
[incubator-nlpcraft] branch NLPCRAFT-15 updated: WIP on NLPCRAFT-21.
This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a commit to branch NLPCRAFT-15
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-15 by this push:
new d106dad WIP on NLPCRAFT-21.
d106dad is described below
commit d106dad306b4909b95c6790bbd1fa465e194d0fa
Author: Aaron Radzinzski <ar...@datalingvo.com>
AuthorDate: Thu Mar 19 13:34:11 2020 -0700
WIP on NLPCRAFT-21.
---
.../nlpcraft/common/nlp/core/NCNlpCoreManager.scala | 2 --
.../nlpcraft/common/nlp/core/NCNlpTokenizer.scala | 4 +---
.../common/nlp/core/opennlp/NCOpenNlpTokenizer.scala | 19 ++++---------------
.../nlp/core/stanford/NCStanfordTokenizer.scala | 3 +--
.../scala/org/apache/nlpcraft/probe/NCProbeBoot.scala | 4 +++-
.../nlpcraft/probe/mgrs/deploy/NCDeployManager.scala | 2 ++
.../nlp/core/opennlp/NCOpenNlpNerEnricher.scala | 9 +--------
.../server/nlp/core/opennlp/NCOpenNlpParser.scala | 7 -------
8 files changed, 12 insertions(+), 38 deletions(-)
diff --git a/src/main/scala/org/apache/nlpcraft/common/nlp/core/NCNlpCoreManager.scala b/src/main/scala/org/apache/nlpcraft/common/nlp/core/NCNlpCoreManager.scala
index 0896af7..6dedbbd 100644
--- a/src/main/scala/org/apache/nlpcraft/common/nlp/core/NCNlpCoreManager.scala
+++ b/src/main/scala/org/apache/nlpcraft/common/nlp/core/NCNlpCoreManager.scala
@@ -57,8 +57,6 @@ object NCNlpCoreManager extends NCService {
case _ ⇒ throw new AssertionError(s"Unexpected engine: ${Config.engine}")
}
- tokenizer.start()
-
super.start()
}
diff --git a/src/main/scala/org/apache/nlpcraft/common/nlp/core/NCNlpTokenizer.scala b/src/main/scala/org/apache/nlpcraft/common/nlp/core/NCNlpTokenizer.scala
index 5e551b3..3391ab2 100644
--- a/src/main/scala/org/apache/nlpcraft/common/nlp/core/NCNlpTokenizer.scala
+++ b/src/main/scala/org/apache/nlpcraft/common/nlp/core/NCNlpTokenizer.scala
@@ -17,12 +17,10 @@
package org.apache.nlpcraft.common.nlp.core
-import org.apache.nlpcraft.common.NCService
-
/**
* NLP tokenizer.
*/
-trait NCNlpTokenizer extends NCService {
+trait NCNlpTokenizer {
/**
* Tokenizes the sentence.
*
diff --git a/src/main/scala/org/apache/nlpcraft/common/nlp/core/opennlp/NCOpenNlpTokenizer.scala b/src/main/scala/org/apache/nlpcraft/common/nlp/core/opennlp/NCOpenNlpTokenizer.scala
index 042d245..6818dca 100644
--- a/src/main/scala/org/apache/nlpcraft/common/nlp/core/opennlp/NCOpenNlpTokenizer.scala
+++ b/src/main/scala/org/apache/nlpcraft/common/nlp/core/opennlp/NCOpenNlpTokenizer.scala
@@ -19,10 +19,9 @@ package org.apache.nlpcraft.common.nlp.core.opennlp
import java.io.BufferedInputStream
-import io.opencensus.trace.Span
import opennlp.tools.tokenize.{Tokenizer, TokenizerME, TokenizerModel}
import org.apache.nlpcraft.common.nlp.core.{NCNlpCoreToken, NCNlpTokenizer}
-import org.apache.nlpcraft.common.{NCService, _}
+import org.apache.nlpcraft.common._
import resource.managed
import scala.language.{implicitConversions, postfixOps}
@@ -30,21 +29,11 @@ import scala.language.{implicitConversions, postfixOps}
/**
* OpenNLP tokenizer implementation.
*/
-object NCOpenNlpTokenizer extends NCService with NCNlpTokenizer {
+object NCOpenNlpTokenizer extends NCNlpTokenizer {
private final val MODEL_PATH = "opennlp/en-token.bin"
- @volatile private var tokenizer: Tokenizer = _
-
- override def start(parent: Span = null): NCService = startScopedSpan("start", parent, "model" → MODEL_PATH) { _ ⇒
- tokenizer = managed(new BufferedInputStream(U.getStream(MODEL_PATH))) acquireAndGet { in ⇒
- new TokenizerME(new TokenizerModel(in))
- }
-
- super.start()
- }
-
- override def stop(parent: Span = null): Unit = startScopedSpan("stop", parent) { _ ⇒
- super.stop()
+ private val tokenizer: Tokenizer = managed(new BufferedInputStream(U.getStream(MODEL_PATH))) acquireAndGet { in ⇒
+ new TokenizerME(new TokenizerModel(in))
}
override def tokenize(sen: String): Seq[NCNlpCoreToken] =
diff --git a/src/main/scala/org/apache/nlpcraft/common/nlp/core/stanford/NCStanfordTokenizer.scala b/src/main/scala/org/apache/nlpcraft/common/nlp/core/stanford/NCStanfordTokenizer.scala
index 5c4cd3a..55e233c 100644
--- a/src/main/scala/org/apache/nlpcraft/common/nlp/core/stanford/NCStanfordTokenizer.scala
+++ b/src/main/scala/org/apache/nlpcraft/common/nlp/core/stanford/NCStanfordTokenizer.scala
@@ -20,14 +20,13 @@ package org.apache.nlpcraft.common.nlp.core.stanford
import java.io.StringReader
import edu.stanford.nlp.process.PTBTokenizer
-import org.apache.nlpcraft.common.NCService
import org.apache.nlpcraft.common.nlp.core.{NCNlpCoreToken, NCNlpTokenizer}
import scala.collection.JavaConverters._
/**
* Stanford tokenizer implementation.
*/
-object NCStanfordTokenizer extends NCService with NCNlpTokenizer {
+object NCStanfordTokenizer extends NCNlpTokenizer {
override def tokenize(sen: String): Seq[NCNlpCoreToken] = {
PTBTokenizer.newPTBTokenizer(new StringReader(sen)).
tokenize().
diff --git a/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala b/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
index a81cbbb..15093a5 100644
--- a/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
+++ b/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
@@ -216,8 +216,10 @@ private [probe] object NCProbeBoot extends LazyLogging with NCOpenCensusTrace {
started = false
- if (probeThread != null)
+ if (probeThread != null) {
probeThread.interrupt()
+ probeThread.join()
+ }
logger.info("Embedded probe shutdown OK.")
}
diff --git a/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala b/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
index d23ca79..050879a 100644
--- a/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
+++ b/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala
@@ -265,6 +265,8 @@ object NCDeployManager extends NCService with DecorateAsScala {
if (isStarted)
modelFactory.terminate()
+ models.clear()
+
super.stop()
}
diff --git a/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpNerEnricher.scala b/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpNerEnricher.scala
index 5b4e96c..c5f8c94 100644
--- a/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpNerEnricher.scala
+++ b/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpNerEnricher.scala
@@ -68,18 +68,11 @@ object NCOpenNlpNerEnricher extends NCService with NCNlpNerEnricher with NCIgnit
catching(wrapIE) {
cache = ignite.cache[String, Array[String]]("opennlp-cache")
}
-
- // Should be started even another NLP engine configured.
- if (!NCOpenNlpTokenizer.isStarted)
- NCOpenNlpTokenizer.start()
-
+
super.start()
}
override def stop(parent: Span = null): Unit = startScopedSpan("stop", parent) { _ ⇒
- if (NCOpenNlpTokenizer.isStarted)
- NCOpenNlpTokenizer.stop(parent)
-
cache = null
super.stop()
diff --git a/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpParser.scala b/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpParser.scala
index ff5b4db..b04cf93 100644
--- a/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpParser.scala
+++ b/src/main/scala/org/apache/nlpcraft/server/nlp/core/opennlp/NCOpenNlpParser.scala
@@ -61,17 +61,10 @@ object NCOpenNlpParser extends NCService with NCNlpParser with NCIgniteInstance
cache = ignite.cache[String, Array[String]]("opennlp-cache")
}
- // Should be started even another NLP engine configured.
- if (!NCOpenNlpTokenizer.isStarted)
- NCOpenNlpTokenizer.start()
-
super.start()
}
override def stop(parent: Span = null): Unit = startScopedSpan("stop", parent) { _ ⇒
- if (NCOpenNlpTokenizer.isStarted)
- NCOpenNlpTokenizer.stop(parent)
-
cache = null
super.stop()