You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by ar...@apache.org on 2020/09/18 07:20:49 UTC
[incubator-nlpcraft] branch NLPCRAFT-126 updated (d7a8522 ->
7341569)
This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a change to branch NLPCRAFT-126
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
from d7a8522 Update NCProbeEnrichmentManager.scala
add f3c305c WIP.
add 892f811 WIP.
add 74b573e WIP.
add 60c3501 Update NCSuggestSynonymManager.scala
add bdfe2bc WIP.
add 4a9b645 WIP.
add cd7340f Merge branch 'master' into NLPCRAFT-124
add 750b3e9 WIP.
add c92cf62 WIP.
add 3e8056c Merge branch 'master' into NLPCRAFT-124
add bed81f1 Sort enricher fixes.
add 22edf82 Merge branch 'master' into NLPCRAFT-41
add 2026f46 Fix for NLPCRAFT-130.
add 180d0b4 Improved logging for test framework (ANSI colored JSON).
add 1d49198 Merge branch 'master' into NLPCRAFT-41
add d36a992 WIP.
add 461fd78 WIP.
add 4271471 Merge branch 'NLPCRAFT-41' into NLPCRAFT-117
add 3a21be0 WIP.
add 13b26e8 Merge branch 'master' into NLPCRAFT-117
add e93767b WIP.
add 3a2f270 Merge branch 'NLPCRAFT-117' into NLPCRAFT-41
add 67c7505 WIP.
add 449f965 WIP.
new 7052140 Merge branch 'master' into NLPCRAFT-126
new 7341569 WIP.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../nlp/core/stanford/NCStanfordCoreManager.scala | 14 +-
.../nlp/core/stanford/NCStanfordNerEnricher.scala | 19 ++-
.../nlp/core/stanford/NCStanfordParser.scala | 13 +-
.../org/apache/nlpcraft/common/NCService.scala | 53 ++++---
.../common/extcfg/NCExternalConfigManager.scala | 15 +-
.../nlpcraft/common/makro/NCMacroParser.scala | 2 +-
.../apache/nlpcraft/common/nlp/NCNlpSentence.scala | 22 +--
.../common/nlp/core/NCNlpCoreManager.scala | 21 ++-
.../nlp/core/opennlp/NCOpenNlpTokenizer.scala | 25 +++-
.../common/nlp/dict/NCDictionaryManager.scala | 12 +-
.../common/nlp/numeric/NCNumericManager.scala | 10 +-
.../org/apache/nlpcraft/common/util/NCUtils.scala | 86 +++++++++--
.../examples/misc/darksky/DarkSkyService.java | 4 +-
.../model/intent/impl/NCIntentSolver.scala | 4 +-
.../model/intent/impl/NCIntentSolverEngine.scala | 31 +++-
.../model/intent/impl/NCIntentSolverVariant.scala | 6 +-
.../model/tools/test/NCTestClientBuilder.java | 13 +-
.../test/impl/NCTestAutoModelValidatorImpl.scala | 5 +-
.../org/apache/nlpcraft/probe/NCProbeBoot.scala | 2 +-
.../nlpcraft/probe/mgrs/NCProbeSynonym.scala | 5 +-
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 18 +++
.../probe/mgrs/conn/NCConnectionManager.scala | 4 +-
.../mgrs/conversation/NCConversationManager.scala | 13 +-
.../probe/mgrs/deploy/NCDeployManager.scala | 159 ++++++++++-----------
.../mgrs/dialogflow/NCDialogFlowManager.scala | 13 +-
.../probe/mgrs/lifecycle/NCLifecycleManager.scala | 22 ++-
.../nlpcraft/probe/mgrs/model/NCModelManager.scala | 32 +++--
.../nlpcraft/probe/mgrs/nlp/NCProbeEnricher.scala | 3 +-
.../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 39 ++---
.../dictionary/NCDictionaryEnricher.scala | 23 +--
.../mgrs/nlp/enrichers/limit/NCLimitEnricher.scala | 16 ++-
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 13 +-
.../enrichers/relation/NCRelationEnricher.scala | 14 +-
.../mgrs/nlp/enrichers/sort/NCSortEnricher.scala | 140 ++++++++++--------
.../enrichers/stopword/NCStopWordEnricher.scala | 37 +++--
.../suspicious/NCSuspiciousNounsEnricher.scala | 13 +-
.../mgrs/nlp/validate/NCValidateManager.scala | 17 ++-
.../nlpcraft/server/company/NCCompanyManager.scala | 17 ++-
.../server/feedback/NCFeedbackManager.scala | 15 +-
.../apache/nlpcraft/server/geo/NCGeoManager.scala | 26 ++--
.../geo/tools/NCGeoSyntheticNamesGenerator.scala | 3 +-
.../lifecycle/NCServerLifecycleManager.scala | 21 ++-
.../server/nlp/core/NCNlpNerEnricher.scala | 4 +-
.../server/nlp/core/NCNlpServerManager.scala | 13 +-
.../nlp/core/google/NCGoogleNerEnricher.scala | 23 ++-
.../nlp/core/opennlp/NCOpenNlpNerEnricher.scala | 25 +++-
.../server/nlp/core/opennlp/NCOpenNlpParser.scala | 19 ++-
.../server/nlp/core/spacy/NCSpaCyNerEnricher.scala | 21 ++-
.../nlp/enrichers/NCServerEnrichmentManager.scala | 14 +-
.../nlp/enrichers/basenlp/NCBaseNlpEnricher.scala | 26 +++-
.../coordinate/NCCoordinatesEnricher.scala | 19 ++-
.../server/nlp/enrichers/date/NCDateEnricher.scala | 131 +++++++++--------
.../server/nlp/enrichers/geo/NCGeoEnricher.scala | 21 ++-
.../nlp/enrichers/numeric/NCNumericEnricher.scala | 23 ++-
.../nlp/enrichers/quote/NCQuoteEnricher.scala | 23 ++-
.../enrichers/stopword/NCStopWordEnricher.scala | 16 ++-
.../server/nlp/preproc/NCPreProcessManager.scala | 17 ++-
.../server/nlp/spell/NCSpellCheckManager.scala | 19 ++-
.../server/nlp/wordnet/NCWordNetManager.scala | 16 ++-
.../nlpcraft/server/probe/NCProbeManager.scala | 18 +--
.../server/proclog/NCProcessLogManager.scala | 13 +-
.../nlpcraft/server/query/NCQueryManager.scala | 21 ++-
.../nlpcraft/server/rest/NCBasicRestApi.scala | 16 +--
.../nlpcraft/server/rest/NCRestManager.scala | 15 +-
.../apache/nlpcraft/server/sql/NCSqlManager.scala | 16 ++-
.../server/sugsyn/NCSuggestSynonymManager.scala | 64 ++++++---
.../apache/nlpcraft/server/tx/NCTxManager.scala | 29 ++--
.../nlpcraft/server/user/NCUserManager.scala | 13 +-
.../scala/org/apache/nlpcraft/NCTestContext.scala | 116 +++++++++++++++
.../org/apache/nlpcraft/NCTestEnvironment.java} | 32 ++---
.../nlpcraft/examples/sql/NCSqlExampleSpec.scala | 36 ++---
.../nlpcraft/examples/sql/NCSqlModelSpec.scala | 27 ++--
.../nlpcraft/examples/time/NCTimeModelSpec.scala | 33 +----
.../model/conversation/NCConversationSpec.scala | 33 +----
.../nlpcraft/models/stm/NCStmTestModelSpec.scala | 34 +----
.../mgrs/nlp/enrichers/NCEnricherBaseSpec.scala | 45 ++----
.../nlp/enrichers/limit/NCEnricherLimitSpec.scala | 7 +-
.../model/NCEnricherNestedModelSpec.scala | 34 ++++-
.../nlp/enrichers/model/NCNestedTestModel.scala | 47 ------
.../relation/NCEnricherRelationSpec.scala | 7 +-
.../nlp/enrichers/sort/NCEnricherSortSpec.scala | 9 +-
.../nlpcraft/server/rest/NCRestAskSpec.scala | 10 +-
.../nlpcraft/server/rest/NCRestClearSpec.scala | 10 +-
.../nlpcraft/server/rest/NCRestModelSpec.scala | 17 +--
.../nlpcraft/server/rest/NCRestProbeSpec.scala | 14 +-
.../apache/nlpcraft/server/rest/NCRestSpec.scala | 3 +-
86 files changed, 1340 insertions(+), 829 deletions(-)
create mode 100644 nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala
copy nlpcraft/src/{main/scala/org/apache/nlpcraft/server/mdo/impl/NCMdoEntity.java => test/scala/org/apache/nlpcraft/NCTestEnvironment.java} (61%)
delete mode 100644 nlpcraft/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCNestedTestModel.scala
[incubator-nlpcraft] 02/02: WIP.
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a commit to branch NLPCRAFT-126
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 73415696507dc2e112c0d82a7a2e5e2a5570c0cc
Author: Aaron Radzinski <ar...@datalingvo.com>
AuthorDate: Fri Sep 18 00:20:38 2020 -0700
WIP.
---
.../org/apache/nlpcraft/common/util/NCUtils.scala | 21 ++++++++++++++++++++-
.../examples/misc/darksky/DarkSkyService.java | 4 ++--
2 files changed, 22 insertions(+), 3 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
index 13e00e2..1aa87a1 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
@@ -26,7 +26,7 @@ import java.nio.file.attribute.BasicFileAttributes
import java.sql.Timestamp
import java.text.{DecimalFormat, DecimalFormatSymbols}
import java.time.{Instant, ZoneId, ZonedDateTime}
-import java.util.concurrent.{ExecutorService, TimeUnit}
+import java.util.concurrent.{ExecutorService, Executors, LinkedBlockingQueue, RejectedExecutionHandler, ThreadPoolExecutor, TimeUnit}
import java.util.jar.JarFile
import java.util.stream.Collectors
import java.util.zip.{ZipInputStream, GZIPInputStream ⇒ GIS, GZIPOutputStream ⇒ GOS}
@@ -1447,6 +1447,25 @@ object NCUtils extends LazyLogging {
}
/**
+ *
+ * @param threadNum
+ * @return
+ */
+ def mkThreadPool(threadNum: Int = Runtime.getRuntime.availableProcessors()): ThreadPoolExecutor =
+ new ThreadPoolExecutor(
+ 1,
+ threadNum,
+ 0L,
+ TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue[Runnable],
+ Executors.defaultThreadFactory,
+ new RejectedExecutionHandler() {
+ // Ignore rejections.
+ override def rejectedExecution(r: Runnable, exec: ThreadPoolExecutor): Unit = ()
+ }
+ )
+
+ /**
* Unzips file.
*
* @param zipFile Zip file.
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/misc/darksky/DarkSkyService.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/misc/darksky/DarkSkyService.java
index c4ae11d..ef8d353 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/misc/darksky/DarkSkyService.java
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/examples/misc/darksky/DarkSkyService.java
@@ -25,6 +25,7 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
+import org.apache.nlpcraft.common.util.NCUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +40,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -71,7 +71,7 @@ public class DarkSkyService {
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss").withZone(ZoneId.systemDefault());
// Can be configured.
- private final ExecutorService pool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
+ private final ExecutorService pool = NCUtils.mkThreadPool(Runtime.getRuntime().availableProcessors());
/**
*
[incubator-nlpcraft] 01/02: Merge branch 'master' into NLPCRAFT-126
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a commit to branch NLPCRAFT-126
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 7052140ad2c6df6417cb9d45cbe4acd231d24d1a
Merge: d7a8522 449f965
Author: Aaron Radzinski <ar...@datalingvo.com>
AuthorDate: Fri Sep 18 00:20:33 2020 -0700
Merge branch 'master' into NLPCRAFT-126
.../nlp/core/stanford/NCStanfordCoreManager.scala | 14 +-
.../nlp/core/stanford/NCStanfordNerEnricher.scala | 19 ++-
.../nlp/core/stanford/NCStanfordParser.scala | 13 +-
.../org/apache/nlpcraft/common/NCService.scala | 53 ++++---
.../common/extcfg/NCExternalConfigManager.scala | 15 +-
.../nlpcraft/common/makro/NCMacroParser.scala | 2 +-
.../apache/nlpcraft/common/nlp/NCNlpSentence.scala | 22 +--
.../common/nlp/core/NCNlpCoreManager.scala | 21 ++-
.../nlp/core/opennlp/NCOpenNlpTokenizer.scala | 25 +++-
.../common/nlp/dict/NCDictionaryManager.scala | 12 +-
.../common/nlp/numeric/NCNumericManager.scala | 10 +-
.../org/apache/nlpcraft/common/util/NCUtils.scala | 65 +++++++--
.../model/intent/impl/NCIntentSolver.scala | 4 +-
.../model/intent/impl/NCIntentSolverEngine.scala | 31 +++-
.../model/intent/impl/NCIntentSolverVariant.scala | 6 +-
.../model/tools/test/NCTestClientBuilder.java | 13 +-
.../test/impl/NCTestAutoModelValidatorImpl.scala | 5 +-
.../org/apache/nlpcraft/probe/NCProbeBoot.scala | 2 +-
.../nlpcraft/probe/mgrs/NCProbeSynonym.scala | 5 +-
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 18 +++
.../probe/mgrs/conn/NCConnectionManager.scala | 4 +-
.../mgrs/conversation/NCConversationManager.scala | 13 +-
.../probe/mgrs/deploy/NCDeployManager.scala | 159 ++++++++++-----------
.../mgrs/dialogflow/NCDialogFlowManager.scala | 13 +-
.../probe/mgrs/lifecycle/NCLifecycleManager.scala | 22 ++-
.../nlpcraft/probe/mgrs/model/NCModelManager.scala | 32 +++--
.../nlpcraft/probe/mgrs/nlp/NCProbeEnricher.scala | 3 +-
.../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 39 ++---
.../dictionary/NCDictionaryEnricher.scala | 23 +--
.../mgrs/nlp/enrichers/limit/NCLimitEnricher.scala | 16 ++-
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 13 +-
.../enrichers/relation/NCRelationEnricher.scala | 14 +-
.../mgrs/nlp/enrichers/sort/NCSortEnricher.scala | 140 ++++++++++--------
.../enrichers/stopword/NCStopWordEnricher.scala | 37 +++--
.../suspicious/NCSuspiciousNounsEnricher.scala | 13 +-
.../mgrs/nlp/validate/NCValidateManager.scala | 17 ++-
.../nlpcraft/server/company/NCCompanyManager.scala | 17 ++-
.../server/feedback/NCFeedbackManager.scala | 15 +-
.../apache/nlpcraft/server/geo/NCGeoManager.scala | 26 ++--
.../geo/tools/NCGeoSyntheticNamesGenerator.scala | 3 +-
.../lifecycle/NCServerLifecycleManager.scala | 21 ++-
.../server/nlp/core/NCNlpNerEnricher.scala | 4 +-
.../server/nlp/core/NCNlpServerManager.scala | 13 +-
.../nlp/core/google/NCGoogleNerEnricher.scala | 23 ++-
.../nlp/core/opennlp/NCOpenNlpNerEnricher.scala | 25 +++-
.../server/nlp/core/opennlp/NCOpenNlpParser.scala | 19 ++-
.../server/nlp/core/spacy/NCSpaCyNerEnricher.scala | 21 ++-
.../nlp/enrichers/NCServerEnrichmentManager.scala | 14 +-
.../nlp/enrichers/basenlp/NCBaseNlpEnricher.scala | 26 +++-
.../coordinate/NCCoordinatesEnricher.scala | 19 ++-
.../server/nlp/enrichers/date/NCDateEnricher.scala | 131 +++++++++--------
.../server/nlp/enrichers/geo/NCGeoEnricher.scala | 21 ++-
.../nlp/enrichers/numeric/NCNumericEnricher.scala | 23 ++-
.../nlp/enrichers/quote/NCQuoteEnricher.scala | 23 ++-
.../enrichers/stopword/NCStopWordEnricher.scala | 16 ++-
.../server/nlp/preproc/NCPreProcessManager.scala | 17 ++-
.../server/nlp/spell/NCSpellCheckManager.scala | 19 ++-
.../server/nlp/wordnet/NCWordNetManager.scala | 16 ++-
.../nlpcraft/server/probe/NCProbeManager.scala | 18 +--
.../server/proclog/NCProcessLogManager.scala | 13 +-
.../nlpcraft/server/query/NCQueryManager.scala | 21 ++-
.../nlpcraft/server/rest/NCBasicRestApi.scala | 16 +--
.../nlpcraft/server/rest/NCRestManager.scala | 15 +-
.../apache/nlpcraft/server/sql/NCSqlManager.scala | 16 ++-
.../server/sugsyn/NCSuggestSynonymManager.scala | 64 ++++++---
.../apache/nlpcraft/server/tx/NCTxManager.scala | 29 ++--
.../nlpcraft/server/user/NCUserManager.scala | 13 +-
.../scala/org/apache/nlpcraft/NCTestContext.scala | 116 +++++++++++++++
.../org/apache/nlpcraft/NCTestEnvironment.java} | 40 ++++--
.../nlpcraft/examples/sql/NCSqlExampleSpec.scala | 36 ++---
.../nlpcraft/examples/sql/NCSqlModelSpec.scala | 27 ++--
.../nlpcraft/examples/time/NCTimeModelSpec.scala | 33 +----
.../model/conversation/NCConversationSpec.scala | 33 +----
.../nlpcraft/models/stm/NCStmTestModelSpec.scala | 34 +----
.../mgrs/nlp/enrichers/NCEnricherBaseSpec.scala | 45 ++----
.../nlp/enrichers/limit/NCEnricherLimitSpec.scala | 7 +-
.../model/NCEnricherNestedModelSpec.scala | 34 ++++-
.../nlp/enrichers/model/NCNestedTestModel.scala | 47 ------
.../relation/NCEnricherRelationSpec.scala | 7 +-
.../nlp/enrichers/sort/NCEnricherSortSpec.scala | 9 +-
.../nlpcraft/server/rest/NCRestAskSpec.scala | 10 +-
.../nlpcraft/server/rest/NCRestClearSpec.scala | 10 +-
.../nlpcraft/server/rest/NCRestModelSpec.scala | 17 +--
.../nlpcraft/server/rest/NCRestProbeSpec.scala | 14 +-
.../apache/nlpcraft/server/rest/NCRestSpec.scala | 3 +-
85 files changed, 1328 insertions(+), 824 deletions(-)
diff --cc nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
index b22eaa9,52630b8..28cb6a8
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
@@@ -89,27 -94,18 +94,20 @@@ object NCProbeEnrichmentManager extend
override def start(parent: Span = null): NCService = startScopedSpan("start", parent) { _ ⇒
embeddedCbs = mutable.HashSet.empty[EMBEDDED_CB]
- pool = new ThreadPoolExecutor(
- 1,
- Runtime.getRuntime.availableProcessors() * 8,
- 0L,
- TimeUnit.MILLISECONDS,
- new LinkedBlockingQueue[Runnable],
- Executors.defaultThreadFactory,
- new RejectedExecutionHandler() {
- override def rejectedExecution(r: Runnable, executor: ThreadPoolExecutor): Unit =
- if (isStarted)
- logger.warn("Task was rejected")
- }
- )
- pool = Executors.newFixedThreadPool(8 * Runtime.getRuntime.availableProcessors())
++ pool = U.mkThreadPool()
+
executor = ExecutionContext.fromExecutor(pool)
- super.start()
+ ackStart()
}
-
+
+ /**
+ *
+ * @param parent Optional parent span.
+ */
override def stop(parent: Span = null): Unit = startScopedSpan("stop", parent) { _ ⇒
+ super.stop()
+
mux.synchronized {
if (embeddedCbs != null)
embeddedCbs.clear()