You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2022/01/16 07:49:22 UTC
[incubator-nlpcraft] branch NLPCRAFT-472 updated: Git ignore file updated.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-472
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-472 by this push:
new a827652 Git ignore file updated.
a827652 is described below
commit a827652fe74f251eb899b04c4e6eaf15f4c492ab
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Sun Jan 16 10:48:44 2022 +0300
Git ignore file updated.
---
.../token/enricher/en/impl/NCStopWordsImpl.scala | 130 ++++++++++-----------
.../enricher/en/impl/NCStopWordsImplSpec.scala | 64 ++++++++++
2 files changed, 127 insertions(+), 67 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/token/enricher/en/impl/NCStopWordsImpl.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/token/enricher/en/impl/NCStopWordsImpl.scala
index 1aa3dcc..3fd36de 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/token/enricher/en/impl/NCStopWordsImpl.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/nlp/token/enricher/en/impl/NCStopWordsImpl.scala
@@ -98,6 +98,69 @@ object NCStopWordsImpl:
"percent"
)
+ private def isQuote(t: NCToken): Boolean = Q_POS.contains(t.getPos)
+ private def toLemmaKey(toks: Seq[NCToken]): String = toks.map(_.getLemma).mkString(" ")
+ private def toValueKey(toks: Seq[NCToken]): String = toks.map(_.getText.toLowerCase).mkString(" ")
+ private def toOriginalKey(toks: Seq[NCToken]): String = toks.map(_.getText).mkString(" ")
+ private def isStopWord(t: NCToken): Boolean = t.getOpt[Boolean]("stopword").orElse(false)
+
+ /**
+ * Gets all sequential permutations of tokens in this NLP sentence.
+ * This method is like a 'tokenMix', but with all combinations of stop-words (with and without)
+ *
+ * @param tokens Tokens.
+ * @param maxLen Maximum number of tokens in the sequence.
+ */
+ private[impl] def tokenMixWithStopWords(tokens: Seq[NCToken], maxLen: Int = Integer.MAX_VALUE): Seq[Seq[NCToken]] =
+ /**
+ * Gets all combinations for sequence of mandatory tokens with stop-words and without.
+ *
+ * Example:
+ * 'A (stop), B, C(stop) -> [A, B, C]; [A, B]; [B, C], [B]
+ * 'A, B(stop), C(stop) -> [A, B, C]; [A, B]; [A, C], [A].
+ *
+ * @param toks Tokens.
+ */
+ def permutations(toks: Seq[NCToken]): Seq[Seq[NCToken]] =
+ def multiple(seq: Seq[Seq[Option[NCToken]]], t: NCToken): Seq[Seq[Option[NCToken]]] =
+ if seq.isEmpty then
+ if isStopWord(t) then IndexedSeq(IndexedSeq(Option(t)), IndexedSeq(None)) else IndexedSeq(IndexedSeq(Option(t)))
+ else
+ (for (subSeq <- seq) yield subSeq :+ Option(t)) ++ (if isStopWord(t) then for (subSeq <- seq) yield subSeq :+ None else Seq.empty)
+
+ var res: Seq[Seq[Option[NCToken]]] = Seq.empty
+ for (t <- toks) res = multiple(res, t) // TODO: are we missing '+=' operator instead?
+ res.map(_.flatten).filter(_.nonEmpty)
+
+ tokenMix(tokens, maxLen).
+ flatMap(permutations).
+ filter(_.nonEmpty).
+ distinct.
+ sortBy(seq => (-seq.length, seq.head.getIndex))
+
+ /**
+ * Gets all sequential permutations of tokens in this NLP sentence.
+ *
+ * For example, if NLP sentence contains "a, b, c, d" tokens, then
+ * this function will return the sequence of following token sequences in this order:
+ * "a b c d"
+ * "a b c"
+ * "b c d"
+ * "a b"
+ * "b c"
+ * "c d"
+ * "a"
+ * "b"
+ * "c"
+ * "d"
+ *
+ * NOTE: this method will not return any permutations with a quoted token.
+ *
+ * @param toks Tokens.
+ * @param maxLen Maximum number of tokens in the sequence.
+ */
+ private def tokenMix(toks: Seq[NCToken], maxLen: Int = Integer.MAX_VALUE): Seq[Seq[NCToken]] =
+ (for (n <- toks.length until 0 by -1 if n <= maxLen) yield toks.sliding(n)).flatten
import org.apache.nlpcraft.nlp.token.enricher.en.impl.NCStopWordsImpl.*
@@ -120,14 +183,8 @@ class NCStopWordsImpl(addStopsSet: JSet[String], exclStopsSet: JSet[String]) ext
init()
private def read(path: String): Set[String] = NCUtils.readTextGzipResource(path, "UTF-8", logger).toSet
-
private def stem(s: String): String = stemmer.stem(s.toLowerCase)
- private def isQuote(t: NCToken): Boolean = Q_POS.contains(t.getPos)
private def toStemKey(toks: Seq[NCToken]): String = toks.map(_.getText).map(stem).mkString(" ")
- private def toLemmaKey(toks: Seq[NCToken]): String = toks.map(_.getLemma).mkString(" ")
- private def toValueKey(toks: Seq[NCToken]): String = toks.map(_.getText.toLowerCase).mkString(" ")
- private def toOriginalKey(toks: Seq[NCToken]): String = toks.map(_.getText).mkString(" ")
- private def isStopWord(t: NCToken): Boolean = t.getOpt[Boolean]("stopword").orElse(false)
/**
* Stop words holder, used for hash search.
@@ -218,67 +275,6 @@ class NCStopWordsImpl(addStopsSet: JSet[String], exclStopsSet: JSet[String]) ext
wildcardsOrigins.matches(toOriginalKey(toks), posOpt)
/**
- * Gets all sequential permutations of tokens in this NLP sentence.
- * This method is like a 'tokenMix', but with all combinations of stop-words (with and without)
- *
- * @param tokens Tokens.
- * @param maxLen Maximum number of tokens in the sequence.
- */
- private def tokenMixWithStopWords(tokens: Seq[NCToken], maxLen: Int = Integer.MAX_VALUE): Seq[Seq[NCToken]] =
- /**
- * Gets all combinations for sequence of mandatory tokens with stop-words and without.
- *
- * Example:
- * 'A (stop), B, C(stop) -> [A, B, C]; [A, B]; [B, C], [B]
- * 'A, B(stop), C(stop) -> [A, B, C]; [A, B]; [A, C], [A].
- *
- * @param toks Tokens.
- */
- def permutations(toks: Seq[NCToken]): Seq[Seq[NCToken]] =
- def multiple(seq: Seq[Seq[Option[NCToken]]], t: NCToken): Seq[Seq[Option[NCToken]]] =
- if seq.isEmpty then
- if isStopWord(t) then IndexedSeq(IndexedSeq(Option(t)), IndexedSeq(None)) else IndexedSeq(IndexedSeq(Option(t)))
- else
- (for (subSeq <- seq) yield subSeq :+ Option(t)) ++ (if isStopWord(t) then for (subSeq <- seq) yield subSeq :+ None else Seq.empty)
-
- var res: Seq[Seq[Option[NCToken]]] = Seq.empty
- for (t <- toks) res = multiple(res, t) // TODO: are we missing '+=' operator instead?
- res.map(_.flatten).filter(_.nonEmpty)
-
- tokenMix(tokens, stopWords = true, maxLen).
- flatMap(permutations).
- filter(_.nonEmpty).
- distinct.
- sortBy(seq => (-seq.length, seq.head.getIndex))
-
- /**
- * Gets all sequential permutations of tokens in this NLP sentence.
- *
- * For example, if NLP sentence contains "a, b, c, d" tokens, then
- * this function will return the sequence of following token sequences in this order:
- * "a b c d"
- * "a b c"
- * "b c d"
- * "a b"
- * "b c"
- * "c d"
- * "a"
- * "b"
- * "c"
- * "d"
- *
- * NOTE: this method will not return any permutations with a quoted token.
- *
- * @param tokens Tokens.
- * @param stopWords Whether or not include tokens marked as stop words.
- * @param maxLen Maximum number of tokens in the sequence.
- */
- private def tokenMix(tokens: Seq[NCToken], stopWords: Boolean = false, maxLen: Int = Integer.MAX_VALUE): Seq[Seq[NCToken]] =
- val toks = tokens.filter(t => stopWords || (!stopWords && !isStopWord(t)))
-
- (for (n <- toks.length until 0 by -1 if n <= maxLen) yield toks.sliding(n)).flatten
-
- /**
*
*/
private def init(): Unit =
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/en/impl/NCStopWordsImplSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/en/impl/NCStopWordsImplSpec.scala
new file mode 100644
index 0000000..ca4eaee
--- /dev/null
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/token/enricher/en/impl/NCStopWordsImplSpec.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.nlp.token.enricher.en.impl
+
+import org.apache.nlpcraft.*
+import org.apache.nlpcraft.nlp.token.enricher.en.*
+import org.apache.nlpcraft.nlp.token.enricher.en.impl.NCStopWordsImpl
+import org.apache.nlpcraft.nlp.util.*
+import org.apache.nlpcraft.nlp.util.opennlp.*
+import org.junit.jupiter.api.*
+
+import scala.jdk.CollectionConverters.*
+
+
+/**
+ *
+ */
+class NCStopWordsImplSpec:
+ case class W(text: String, stop: Boolean)
+
+ /**
+ *
+ * @param words
+ * @param expected
+ */
+ private def test0(words: Seq[W], expected: Seq[String]): Unit =
+ val toksList = EN_PIPELINE.getTokenParser.tokenize(words.map(_.text).mkString(" "))
+ require(toksList.size == words.size)
+ val toks = toksList.asScala
+
+ toks.zip(words).foreach { (t, w) => t.put("stopword", w.stop) }
+
+ val mix = NCStopWordsImpl.tokenMixWithStopWords(toks)
+
+ val resSorted = mix.map(_.map(_.getText).mkString).sorted
+ val expectedSorted = expected.sorted
+
+ require(expectedSorted == resSorted, s"Expected=$expectedSorted, result=$resSorted")
+
+ @Test
+ def testPermute(): Unit =
+ test0(
+ Seq(W("A", false), W("B", true), W("C", true)),
+ Seq("ABC", "AB", "AC", "BC", "A", "B", "C")
+ )
+ test0(
+ Seq(W("A", false), W("B", false), W("C", false)),
+ Seq("ABC", "AB", "BC", "A", "B", "C")
+ )