You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2020/05/19 16:40:24 UTC
[incubator-nlpcraft] branch NLPCRAFT-30-tmp created (now a61c6c9)
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a change to branch NLPCRAFT-30-tmp
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
at a61c6c9 Utilities methods refactoring.
This branch includes the following new commits:
new a61c6c9 Utilities methods refactoring.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-nlpcraft] 01/01: Utilities methods refactoring.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-30-tmp
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit a61c6c9aeefc90bce3f453f5e9660e690e6f7a83
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Tue May 19 19:39:39 2020 +0300
Utilities methods refactoring.
---
.../apache/nlpcraft/common/nlp/NCNlpSentence.scala | 635 ++++++++++++++++-
.../nlpcraft/common/nlp/NCNlpSentenceNote.scala | 45 +-
.../apache/nlpcraft/model/impl/NCValueImpl.java | 50 --
.../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 22 +-
.../probe/mgrs/nlp/enrichers/NCEnricherUtils.scala | 779 ---------------------
.../mgrs/nlp/enrichers/limit/NCLimitEnricher.scala | 28 +-
.../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 11 +-
.../enrichers/relation/NCRelationEnricher.scala | 28 +-
.../mgrs/nlp/enrichers/sort/NCSortEnricher.scala | 25 +-
.../probe/mgrs/nlp/impl/NCVariantsCreator.scala | 79 +++
10 files changed, 825 insertions(+), 877 deletions(-)
diff --git a/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentence.scala b/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentence.scala
index b88a472..e20295b 100644
--- a/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentence.scala
+++ b/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentence.scala
@@ -17,19 +17,438 @@
package org.apache.nlpcraft.common.nlp
-import scala.collection._
+import java.util
+import java.util.Collections
+
+import org.apache.nlpcraft.common.NCE
+import org.apache.nlpcraft.common.nlp.pos.NCPennTreebank
+
+import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
+import scala.collection.{Map, Seq, Set, mutable}
import scala.language.implicitConversions
+object NCNlpSentence {
+ implicit def toTokens(x: NCNlpSentence): ArrayBuffer[NCNlpSentenceToken] = x.tokens
+
+ /**
+ *
+ * @param ns
+ * @param idxs
+ * @param notesType
+ * @param note
+ * @return
+ */
+ private def checkRelation(ns: NCNlpSentence, idxs: Seq[Int], notesType: String, note: NCNlpSentenceNote): Boolean = {
+ val types =
+ idxs.flatMap(idx ⇒ {
+ val types = ns(idx).map(p ⇒ p).filter(!_.isNlp).map(_.noteType)
+
+ types.size match {
+ case 0 ⇒ None
+ case 1 ⇒ Some(types.head)
+ case _ ⇒ throw new AssertionError(s"Unexpected tokes: ${ns(idx)}")
+ }
+ }).distinct
+
+ /**
+ * Example:
+ *1. Sentence 'maximum x' (single element related function)
+ *- maximum is aggregate function linked to date element.
+ *- x defined as 2 elements: date and num.
+ * So, the variant 'maximum x (as num)' should be excluded.
+ * *
+ *2. Sentence 'compare x and y' (multiple elements related function)
+ *- compare is relation function linked to date element.
+ *- x an y defined as 2 elements: date and num.
+ * So, variants 'x (as num) and x (as date)' and 'x (as date) and x (as num)'
+ * should't be excluded, but invalid relation should be deleted for these combinations.
+ */
+
+ types.size match {
+ case 0 ⇒ throw new AssertionError(s"Unexpected empty types [notesType=$notesType]")
+ case 1 ⇒ types.head == notesType
+ case _ ⇒
+ // Equal elements should be processed together with function element.
+ if (types.size == 1)
+ false
+ else {
+ ns.removeNote(note)
+
+ true
+ }
+ }
+ }
+
+ /**
+ * Fixes notes with references to other notes indexes.
+ * Note that 'idxsField' is 'indexes' and 'noteField' is 'note' for all kind of references.
+ *
+ * @param noteType Note type.
+ * @param ns Sentence.
+ * @param history Indexes transformation history.
+ * @return Valid flag.
+ */
+ private def fixIndexesReferences(noteType: String, ns: NCNlpSentence, history: Seq[(Int, Int)]): Boolean = {
+ ns.filter(_.isTypeOf(noteType)).foreach(tok ⇒
+ tok.getNoteOpt(noteType, "indexes") match {
+ case Some(n) ⇒
+ val idxs: Seq[Int] = n.data[java.util.List[Int]]("indexes").asScala
+ var fixed = idxs
+
+ history.foreach { case (idxOld, idxNew) ⇒ fixed = fixed.map(i ⇒ if (i == idxOld) idxNew else i) }
+
+ fixed = fixed.distinct
+
+ if (idxs != fixed)
+ ns.fixNote(n, "indexes" → fixed.asJava.asInstanceOf[java.io.Serializable])
+ case None ⇒ // No-op.
+ }
+ )
+
+ ns.flatMap(_.getNotes(noteType)).forall(
+ n ⇒ checkRelation(ns, n.data[java.util.List[Int]]("indexes").asScala, n.data[String]("note"), n)
+ )
+ }
+
+ /**
+ * Copies token.
+ *
+ * @param ns Sentence.
+ * @param history Indexes transformation history.
+ * @param toksCopy Copied tokens.
+ * @param i Index.
+ */
+ private def simpleCopy(
+ ns: NCNlpSentence,
+ history: mutable.ArrayBuffer[(Int, Int)],
+ toksCopy: NCNlpSentence, i: Int
+ ): Seq[NCNlpSentenceToken] = {
+ val tokCopy = toksCopy(i)
+
+ history += tokCopy.index → ns.size
+
+ ns += tokCopy.clone(ns.size)
+ }
+
+ /**
+ * Glues stop words.
+ *
+ * @param ns Sentence.
+ * @param userNoteTypes Notes types.
+ * @param history Indexes transformation history.
+ */
+ private def unionStops(
+ ns: NCNlpSentence,
+ userNoteTypes: Seq[String],
+ history: mutable.ArrayBuffer[(Int, Int)]
+ ): Unit = {
+ // Java collection used because using scala collections (mutable.Buffer.empty[mutable.Buffer[Token]]) is reason
+ // Of compilation errors which seems as scala compiler internal error.
+ val bufs = new util.ArrayList[mutable.Buffer[NCNlpSentenceToken]]()
+
+ def last[T](l: util.List[T]): T = l.get(l.size() - 1)
+
+ ns.filter(t ⇒ t.isStopWord && !t.isBracketed).foreach(t ⇒
+ if (!bufs.isEmpty && last(bufs).last.index + 1 == t.index)
+ last(bufs) += t
+ else
+ bufs.add(mutable.Buffer.empty[NCNlpSentenceToken] :+ t)
+ )
+
+ val idxsSeq = bufs.asScala.filter(_.lengthCompare(1) > 0).map(_.map(_.index))
+
+ if (idxsSeq.nonEmpty) {
+ val nsCopyToks = ns.clone()
+ ns.clear()
+
+ val buf = mutable.Buffer.empty[Int]
+
+ for (i ← nsCopyToks.indices)
+ idxsSeq.find(_.contains(i)) match {
+ case Some(idxs) ⇒
+ if (!buf.contains(idxs.head)) {
+ buf += idxs.head
+
+ ns += mkCompound(ns, nsCopyToks, idxs, stop = true, ns.size, None, history)
+ }
+ case None ⇒ simpleCopy(ns, history, nsCopyToks, i)
+ }
+
+ fixIndexes(ns, userNoteTypes)
+ }
+ }
+
+ /**
+ * Fixes indexes for all notes after recreating tokens.
+ *
+ * @param ns Sentence.
+ * @param userNoteTypes Notes types.
+ */
+ private def fixIndexes(ns: NCNlpSentence, userNoteTypes: Seq[String]) {
+ // Replaces other notes indexes.
+ for (t ← userNoteTypes :+ "nlpcraft:nlp"; note ← ns.getNotes(t)) {
+ val toks = ns.filter(_.contains(note)).sortBy(_.index)
+
+ val newNote = note.clone(toks.map(_.index), toks.flatMap(_.wordIndexes).sorted)
+
+ toks.foreach(t ⇒ {
+ t.remove(note)
+ t.add(newNote)
+ })
+ }
+
+ // Special case - field index of core NLP note.
+ ns.zipWithIndex.foreach { case (tok, idx) ⇒ ns.fixNote(tok.getNlpNote, "index" → idx) }
+ }
+
+ /**
+ * Zip notes with same type.
+ *
+ * @param ns Sentence.
+ * @param nType Notes type.
+ * @param userNotesTypes Notes types.
+ * @param history Indexes transformation history.
+ */
+ private def zipNotes(
+ ns: NCNlpSentence,
+ nType: String,
+ userNotesTypes: Seq[String],
+ history: mutable.ArrayBuffer[(Int, Int)]
+ ): Unit = {
+ val nts = ns.getNotes(nType).filter(n ⇒ n.tokenFrom != n.tokenTo).sortBy(_.tokenFrom)
+
+ val overlapped =
+ nts.flatMap(n ⇒ n.tokenFrom to n.tokenTo).map(ns(_)).exists(
+ t ⇒ userNotesTypes.map(pt ⇒ t.getNotes(pt).size).sum > 1
+ )
+
+ if (nts.nonEmpty && !overlapped) {
+ val nsCopyToks = ns.clone()
+ ns.clear()
+
+ val buf = mutable.ArrayBuffer.empty[Int]
+
+ for (i ← nsCopyToks.indices)
+ nts.find(_.tokenIndexes.contains(i)) match {
+ case Some(n) ⇒
+ if (!buf.contains(n.tokenFrom)) {
+ buf += n.tokenFrom
+
+ ns += mkCompound(ns, nsCopyToks, n.tokenIndexes, stop = false, ns.size, Some(n), history)
+ }
+ case None ⇒ simpleCopy(ns, history, nsCopyToks, i)
+ }
+
+ fixIndexes(ns, userNotesTypes)
+ }
+ }
+
+ /**
+ * Makes compound note.
+ *
+ * @param ns Sentence.
+ * @param nsCopyToks Tokens.
+ * @param indexes Indexes.
+ * @param stop Flag.
+ * @param idx Index.
+ * @param commonNote Common note.
+ * @param history Indexes transformation history.
+ */
+ private def mkCompound(
+ ns: NCNlpSentence,
+ nsCopyToks: Seq[NCNlpSentenceToken],
+ indexes: Seq[Int],
+ stop: Boolean,
+ idx: Int,
+ commonNote: Option[NCNlpSentenceNote],
+ history: mutable.ArrayBuffer[(Int, Int)]
+ ): NCNlpSentenceToken = {
+ val t = NCNlpSentenceToken(idx)
+
+ // Note, it adds stop-words too.
+ val content = nsCopyToks.zipWithIndex.filter(p ⇒ indexes.contains(p._2)).map(_._1)
+
+ content.foreach(t ⇒ history += t.index → idx)
+
+ def mkValue(get: NCNlpSentenceToken ⇒ String): String = {
+ val buf = mutable.Buffer.empty[String]
+
+ val n = content.size - 1
+
+ content.zipWithIndex.foreach(p ⇒ {
+ val t = p._1
+ val idx = p._2
+
+ buf += get(t)
+
+ if (idx < n && t.endCharIndex != content(idx + 1).startCharIndex)
+ buf += " "
+ })
+
+ buf.mkString
+ }
+
+ val origText = mkValue((t: NCNlpSentenceToken) ⇒ t.origText)
+
+ val idxs = Seq(idx)
+ val wordIdxs = content.flatMap(_.wordIndexes).sorted
+
+ val direct =
+ commonNote match {
+ case Some(n) if n.isUser ⇒ n.isDirect
+ case _ ⇒ content.forall(_.isDirect)
+ }
+
+ val params = Seq(
+ "index" → idx,
+ "pos" → NCPennTreebank.SYNTH_POS,
+ "posDesc" → NCPennTreebank.SYNTH_POS_DESC,
+ "lemma" → mkValue((t: NCNlpSentenceToken) ⇒ t.lemma),
+ "origText" → origText,
+ "normText" → mkValue((t: NCNlpSentenceToken) ⇒ t.normText),
+ "stem" → mkValue((t: NCNlpSentenceToken) ⇒ t.stem),
+ "start" → content.head.startCharIndex,
+ "end" → content.last.endCharIndex,
+ "charLength" → origText.length,
+ "quoted" → false,
+ "stopWord" → stop,
+ "bracketed" → false,
+ "direct" → direct,
+ "dict" → (if (nsCopyToks.size == 1) nsCopyToks.head.getNlpNote.data[Boolean]("dict") else false),
+ "english" → nsCopyToks.forall(_.getNlpNote.data[Boolean]("english")),
+ "swear" → nsCopyToks.exists(_.getNlpNote.data[Boolean]("swear"))
+ )
+
+ val nlpNote = NCNlpSentenceNote(idxs, wordIdxs, "nlpcraft:nlp", params: _*)
+
+ t.add(nlpNote)
+
+ // Adds processed note with fixed indexes.
+ commonNote match {
+ case Some(n) ⇒
+ ns.removeNote(n)
+ t.add(n.clone(idxs, wordIdxs))
+ case None ⇒ // No-op.
+ }
+
+ t
+ }
+
+
+ /**
+ * Fixes notes with references list to other notes indexes.
+ *
+ * @param noteType Note type.
+ * @param idxsField Indexes field.
+ * @param noteField Note field.
+ * @param ns Sentence.
+ * @param history Indexes transformation history.
+ * @return Valid flag.
+ */
+ private def fixIndexesReferencesList(
+ noteType: String,
+ idxsField: String,
+ noteField: String,
+ ns: NCNlpSentence,
+ history: Seq[(Int, Int)]
+ ): Boolean = {
+ var ok = true
+
+ for (tok ← ns.filter(_.isTypeOf(noteType)) if ok)
+ tok.getNoteOpt(noteType, idxsField) match {
+ case Some(n) ⇒
+ val idxs: Seq[Seq[Int]] = n.data[java.util.List[java.util.List[Int]]](idxsField).asScala.map(_.asScala)
+ var fixed = idxs
+
+ history.foreach { case (idxOld, idxNew) ⇒ fixed = fixed.map(_.map(i ⇒ if (i == idxOld) idxNew else i).distinct) }
+
+ if (fixed.forall(_.size == 1))
+ // Fix double dimension array to one dimension,
+ // so it should be called always in spite of 'fixIndexesReferences' method.
+ ns.fixNote(n, idxsField → fixed.map(_.head).asJava.asInstanceOf[java.io.Serializable])
+ else
+ ok = false
+ case None ⇒ // No-op.
+ }
+
+ ok &&
+ ns.flatMap(_.getNotes(noteType)).forall(rel ⇒
+ rel.dataOpt[java.util.List[Int]](idxsField) match {
+ case Some(idxsList) ⇒
+ val notesTypes = rel.data[util.List[String]](noteField)
+
+ require(idxsList.size() == notesTypes.size())
+
+ idxsList.asScala.zip(notesTypes.asScala).forall {
+ case (idxs, notesType) ⇒ checkRelation(ns, Seq(idxs), notesType, rel)
+ }
+ case None ⇒ true
+ }
+ )
+ }
+
+ /**
+ * Fixes tokens positions.
+ *
+ * @param ns Sentence.
+ * @param notNlpTypes Token types.
+ */
+ private def collapseSentence(ns: NCNlpSentence, notNlpTypes: Seq[String]): Boolean = {
+ ns.
+ filter(!_.isNlp).
+ filter(_.isStopWord).
+ flatten.
+ filter(_.isNlp).
+ foreach(n ⇒ ns.fixNote(n, "stopWord" → false))
+
+ val nsNotes: Map[String, Seq[Int]] = ns.tokens.flatten.map(p ⇒ p.noteType → p.tokenIndexes).toMap
+
+ for (
+ t ← ns.tokens;
+ stopReason ← t.stopsReasons
+ if nsNotes.getOrElse(stopReason.noteType, Seq.empty) == stopReason.tokenIndexes
+ )
+ ns.fixNote(t.getNlpNote, "stopWord" → true)
+
+ val history = mutable.ArrayBuffer.empty[(Int, Int)]
+
+ notNlpTypes.foreach(typ ⇒ zipNotes(ns, typ, notNlpTypes, history))
+
+ unionStops(ns, notNlpTypes, history)
+
+ val res =
+ Seq("nlpcraft:relation", "nlpcraft:limit").forall(t ⇒ fixIndexesReferences(t, ns, history)) &&
+ fixIndexesReferencesList("nlpcraft:sort", "subjindexes", "subjnotes", ns, history) &&
+ fixIndexesReferencesList("nlpcraft:sort", "byindexes", "bynotes", ns, history)
+
+ if (res)
+ // Validation (all indexes calculated well)
+ require(
+ !ns.flatten.
+ exists(n ⇒ ns.filter(_.wordIndexes.exists(n.wordIndexes.contains)).exists(t ⇒ !t.contains(n))),
+ s"Invalid sentence:\n" +
+ ns.map(t ⇒
+ // Human readable invalid sentence for debugging.
+ s"${t.origText}{index:${t.index}}[${t.map(n ⇒ s"${n.noteType}, {range:${n.tokenFrom}-${n.tokenTo}}").mkString("|")}]"
+ ).mkString("\n")
+ )
+
+ res
+ }
+}
+
+import org.apache.nlpcraft.common.nlp.NCNlpSentence._
+
/**
* Parsed NLP sentence is a collection of tokens. Each token is a collection of notes and
* each note is a collection of KV pairs.
*
- * @param srvReqId Server request ID.
- * @param text Normalized text.
- * @param weight Weight.
+ * @param srvReqId Server request ID.
+ * @param text Normalized text.
+ * @param weight Weight.
* @param enabledBuiltInToks Enabled built-in tokens.
- * @param tokens Initial buffer.
+ * @param tokens Initial buffer.
*/
class NCNlpSentence(
val srvReqId: String,
@@ -54,7 +473,7 @@ class NCNlpSentence(
* @param noteType Note type.
*/
def getNotes(noteType: String): Seq[NCNlpSentenceNote] = this.flatMap(_.getNotes(noteType)).distinct
-
+
/**
* Utility method that removes note with given ID from all tokens in this sentence.
* No-op if such note wasn't found.
@@ -82,6 +501,206 @@ class NCNlpSentence(
hash = null
}
+ /**
+ * This collapser handles several tasks:
+ * - "overall" collapsing after all other individual collapsers had their turn.
+ * - Special further enrichment of tokens like linking, etc.
+ *
+ * In all cases of overlap (full or partial) - the "longest" note wins. In case of overlap and equal
+ * lengths - the winning note is chosen based on this priority.
+ *
+ */
+ @throws[NCE]
+ def collapse(): Seq[NCNlpSentence] = {
+ // Always deletes `similar` notes.
+ // Some words with same note type can be detected various ways.
+ // We keep only one variant - with `best` direct and sparsity parameters,
+ // other variants for these words are redundant.
+ val redundant: Seq[NCNlpSentenceNote] =
+ this.flatten.filter(!_.isNlp).distinct.
+ groupBy(_.getKey()).
+ map(p ⇒ p._2.sortBy(p ⇒
+ (
+ // System notes don't have such flags.
+ if (p.isUser) {
+ if (p.isDirect) 0 else 1
+ }
+ else
+ 0,
+ if (p.isUser) p.sparsity else 0
+ )
+ )).
+ flatMap(_.drop(1)).
+ toSeq
+
+ redundant.foreach(this.removeNote)
+
+ def getNotNlpNotes(toks: Seq[NCNlpSentenceToken]): Seq[NCNlpSentenceNote] =
+ toks.flatten.filter(!_.isNlp).distinct
+
+ val delCombs: Seq[NCNlpSentenceNote] =
+ getNotNlpNotes(this).
+ flatMap(note ⇒ getNotNlpNotes(this.slice(note.tokenFrom, note.tokenTo + 1)).filter(_ != note)).
+ distinct
+
+ val toksByIdx: Seq[Seq[NCNlpSentenceNote]] =
+ delCombs.flatMap(note ⇒ note.wordIndexes.map(_ → note)).
+ groupBy { case (idx, _) ⇒ idx }.
+ map { case (_, seq) ⇒ seq.map { case (_, note) ⇒ note } }.
+ toSeq.sortBy(-_.size)
+
+ val minDelSize = if (toksByIdx.isEmpty) 1 else toksByIdx.map(_.size).max - 1
+
+ val sens =
+ if (delCombs.nonEmpty) {
+ val deleted = mutable.ArrayBuffer.empty[Seq[NCNlpSentenceNote]]
+
+ val sens =
+ (minDelSize to delCombs.size).
+ flatMap(i ⇒
+ delCombs.combinations(i).
+ filter(delComb ⇒ !toksByIdx.exists(_.count(note ⇒ !delComb.contains(note)) > 1))
+ ).
+ sortBy(_.size).
+ flatMap(delComb ⇒
+ // Already processed with less subset of same deleted tokens.
+ if (!deleted.exists(_.forall(delComb.contains))) {
+ val nsClone = this.clone()
+
+ delComb.foreach(nsClone.removeNote)
+
+ // Has overlapped notes for some tokens.
+ require(!nsClone.exists(_.count(!_.isNlp) > 1))
+
+ deleted += delComb
+
+ val notNlpTypes = getNotNlpNotes(nsClone).map(_.noteType).distinct
+
+ if (collapseSentence(nsClone, notNlpTypes)) Some(nsClone) else None
+ }
+ else
+ None
+ )
+
+ // It removes sentences which have only one difference - 'direct' flag of their user tokens.
+ // `Direct` sentences have higher priority.
+ case class Key(
+ sysNotes: Seq[Map[String, java.io.Serializable]],
+ userNotes: Seq[Map[String, java.io.Serializable]]
+ )
+ case class Value(sentence: NCNlpSentence, directCount: Int)
+
+ val m = mutable.HashMap.empty[Key, Value]
+
+ sens.map(sen ⇒ {
+ val notes = sen.flatten
+
+ val sysNotes = notes.filter(_.isSystem)
+ val nlpNotes = notes.filter(_.isNlp)
+ val userNotes = notes.filter(_.isUser)
+
+ def get(seq: Seq[NCNlpSentenceNote], keys2Skip: String*): Seq[Map[String, java.io.Serializable]] =
+ seq.map(p ⇒
+ // We have to delete some keys to have possibility to compare sentences.
+ p.clone().filter(_._1 != "direct")
+ )
+
+ (Key(get(sysNotes), get(userNotes)), sen, nlpNotes.map(p ⇒ if (p.isDirect) 0 else 1).sum)
+ }).
+ foreach { case (key, sen, directCnt) ⇒
+ m.get(key) match {
+ case Some(v) ⇒
+ // Best sentence is sentence with `direct` synonyms.
+ if (v.directCount > directCnt)
+ m += key → Value(sen, directCnt)
+ case None ⇒ m += key → Value(sen, directCnt)
+ }
+ }
+
+ m.values.map(_.sentence).toSeq
+ }
+ else {
+ if (collapseSentence(this, getNotNlpNotes(this).map(_.noteType).distinct)) Seq(this) else Seq.empty
+ }.distinct
+
+ sens.foreach(sen ⇒
+ sen.foreach(tok ⇒
+ tok.size match {
+ case 1 ⇒ require(tok.head.isNlp, s"Unexpected non-'nlpcraft:nlp' token: $tok")
+ case 2 ⇒ require(tok.head.isNlp ^ tok.last.isNlp, s"Unexpected token notes: $tok")
+ case _ ⇒ require(requirement = false, s"Unexpected token notes count: $tok")
+ }
+ )
+ )
+
+ // Drops similar sentences (with same tokens structure).
+ // Among similar sentences we prefer one with minimal free words count.
+ sens.groupBy(_.flatten.filter(!_.isNlp).map(_.getKey(withIndexes = false))).
+ map { case (_, seq) ⇒ seq.minBy(_.filter(p ⇒ p.isNlp && !p.isStopWord).map(_.wordIndexes.length).sum) }.
+ toSeq
+ }
+
+ /**
+ *
+ * @param n1
+ * @param n2
+ * @return
+ */
+ def notesEqualOrSimilar(n1: NCNlpSentenceNote, n2: NCNlpSentenceNote): Boolean =
+ if (n1.noteType != n2.noteType)
+ false
+ else {
+ val stopIdxs = this.filter(_.isStopWord).map(_.index)
+
+ // One possible difference - stopwords indexes.
+ def wordsEqualOrSimilar0(n1: NCNlpSentenceNote, n2: NCNlpSentenceNote): Boolean = {
+ val set1 = n1.wordIndexes.toSet
+ val set2 = n2.wordIndexes.toSet
+
+ set1 == set2 || set1.subsetOf(set2) && set2.diff(set1).forall(stopIdxs.contains)
+ }
+
+ def wordsEqualOrSimilar(n1: NCNlpSentenceNote, n2: NCNlpSentenceNote): Boolean =
+ wordsEqualOrSimilar0(n1, n2) || wordsEqualOrSimilar0(n2, n1)
+
+ def tokensEqualOrSimilar0(set1: Set[NCNlpSentenceToken], set2: Set[NCNlpSentenceToken]): Boolean =
+ set1 == set2 || set1.subsetOf(set2) && set2.diff(set1).forall(_.isStopWord)
+
+ def tokensEqualOrSimilar(set1: Set[NCNlpSentenceToken], set2: Set[NCNlpSentenceToken]): Boolean =
+ tokensEqualOrSimilar0(set1, set2) || tokensEqualOrSimilar0(set2, set1)
+
+ def getList(n: NCNlpSentenceNote, refIdxName: String): Set[NCNlpSentenceToken] =
+ n.getOrElse(refIdxName, Collections.emptyList).asInstanceOf[java.util.List[Int]].asScala.
+ map(this (_)).toSet
+
+ def getListList(n: NCNlpSentenceNote, refIdxName: String): Set[NCNlpSentenceToken] =
+ n.getOrElse(refIdxName, Collections.emptyList).asInstanceOf[java.util.List[java.util.List[Int]]].asScala.
+ flatMap(_.asScala.map(this (_))).toSet
+
+ def referencesEqualOrSimilar0(n1: NCNlpSentenceNote, n2: NCNlpSentenceNote): Boolean = {
+ require(n1.noteType == n2.noteType)
+
+ n1.noteType match {
+ case "nlpcraft:sort" ⇒
+ tokensEqualOrSimilar(getListList(n1, "subjindexes"), getListList(n2, "subjindexes")) &&
+ tokensEqualOrSimilar(getListList(n1, "byindexes"), getListList(n2, "byindexes"))
+ case "nlpcraft:limit" ⇒
+ tokensEqualOrSimilar(getList(n1, "indexes"), getList(n2, "indexes"))
+ case "nlpcraft:reference" ⇒
+ tokensEqualOrSimilar(getList(n1, "indexes"), getList(n2, "indexes"))
+
+ case _ ⇒ true
+ }
+ }
+
+ def referencesEqualOrSimilar(n1: NCNlpSentenceNote, n2: NCNlpSentenceNote): Boolean =
+ referencesEqualOrSimilar0(n1, n2) || referencesEqualOrSimilar0(n2, n1)
+
+ def getUniqueKey0(n: NCNlpSentenceNote): Seq[Any] = n.getKey(withIndexes = false, withReferences = false)
+
+ getUniqueKey0(n1) == getUniqueKey0(n2) && wordsEqualOrSimilar(n1, n2) && referencesEqualOrSimilar(n1, n2)
+ }
+
override def equals(obj: Any): Boolean = obj match {
case x: NCNlpSentence ⇒
tokens == x.tokens &&
@@ -91,7 +710,3 @@ class NCNlpSentence(
case _ ⇒ false
}
}
-
-object NCNlpSentence {
- implicit def toTokens(x: NCNlpSentence): ArrayBuffer[NCNlpSentenceToken] = x.tokens
-}
\ No newline at end of file
diff --git a/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceNote.scala b/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceNote.scala
index d707000..a52afce 100644
--- a/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceNote.scala
+++ b/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceNote.scala
@@ -17,11 +17,13 @@
package org.apache.nlpcraft.common.nlp
+import java.util.Collections
+
import org.apache.nlpcraft.common._
import org.apache.nlpcraft.common.ascii._
import scala.collection.JavaConverters._
-import scala.collection.mutable
+import scala.collection.{Seq, Set, mutable}
import scala.language.implicitConversions
/**
@@ -127,6 +129,47 @@ class NCNlpSentenceNote(private val values: Map[String, java.io.Serializable]) e
/**
*
+ * @param withIndexes
+ * @param withReferences
+ * @return
+ */
+ def getKey(withIndexes: Boolean = true, withReferences: Boolean = true): Seq[Any] = {
+ def addRefs(names: String*): Seq[String] = if (withReferences) names else Seq.empty
+
+ val names: Seq[String] =
+ if (isUser)
+ Seq.empty
+ else
+ noteType match {
+ case "nlpcraft:continent" ⇒ Seq("continent")
+ case "nlpcraft:subcontinent" ⇒ Seq("continent", "subcontinent")
+ case "nlpcraft:country" ⇒ Seq("continent", "subcontinent", "country")
+ case "nlpcraft:region" ⇒ Seq("continent", "subcontinent", "country", "region")
+ case "nlpcraft:city" ⇒ Seq("continent", "subcontinent", "country", "region", "city")
+ case "nlpcraft:metro" ⇒ Seq("metro")
+ case "nlpcraft:date" ⇒ Seq("from", "to")
+ case "nlpcraft:relation" ⇒ Seq("type", "note") ++ addRefs("indexes")
+ case "nlpcraft:sort" ⇒ Seq("asc", "subjnotes", "bynotes") ++ addRefs("subjindexes", "byindexes")
+ case "nlpcraft:limit" ⇒ Seq("limit", "note") ++ addRefs("indexes", "asc") // Asc flag has sense only with references for limit.
+ case "nlpcraft:coordinate" ⇒ Seq("latitude", "longitude")
+ case "nlpcraft:num" ⇒ Seq("from", "to", "unit", "unitType")
+ case x if x.startsWith("google:") ⇒ Seq("meta", "mentionsBeginOffsets", "mentionsContents", "mentionsTypes")
+ case x if x.startsWith("stanford:") ⇒ Seq("nne")
+ case x if x.startsWith("opennlp:") ⇒ Seq.empty
+ case x if x.startsWith("spacy:") ⇒ Seq("vector")
+
+ case _ ⇒ throw new AssertionError(s"Unexpected note type: $noteType")
+ }
+
+ val seq1 = if (withIndexes) Seq(wordIndexes, noteType) else Seq(noteType)
+ val seq2 = names.map(name ⇒ this.getOrElse(name, null))
+
+ seq1 ++ seq2
+ }
+
+
+ /**
+ *
* @return
*/
override def toString: String =
diff --git a/src/main/scala/org/apache/nlpcraft/model/impl/NCValueImpl.java b/src/main/scala/org/apache/nlpcraft/model/impl/NCValueImpl.java
deleted file mode 100644
index 348369b..0000000
--- a/src/main/scala/org/apache/nlpcraft/model/impl/NCValueImpl.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.model.impl;
-
-import org.apache.nlpcraft.model.NCValue;
-
-import java.util.List;
-
-/**
- * Default value implementation.
- */
-public class NCValueImpl implements NCValue {
- private final String name;
- private final List<String> synonyms;
-
- /**
- *
- * @param name
- * @param synonyms
- */
- public NCValueImpl(String name, List<String> synonyms) {
- this.name = name;
- this.synonyms = synonyms;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public List<String> getSynonyms() {
- return synonyms;
- }
-}
diff --git a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
index f7cb78d..693626a 100644
--- a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
+++ b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
@@ -30,7 +30,7 @@ import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.common.debug.NCLogHolder
import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote}
import org.apache.nlpcraft.model._
-import org.apache.nlpcraft.model.impl.{NCModelImpl, NCTokenLogger, NCVariantImpl}
+import org.apache.nlpcraft.model.impl.{NCModelImpl, NCTokenLogger}
import org.apache.nlpcraft.model.intent.impl.NCIntentSolverInput
import org.apache.nlpcraft.model.opencensus.stats.NCOpenCensusModelStats
import org.apache.nlpcraft.probe.embedded.NCEmbeddedResult
@@ -39,7 +39,6 @@ import org.apache.nlpcraft.probe.mgrs.conn.NCConnectionManager
import org.apache.nlpcraft.probe.mgrs.conversation.NCConversationManager
import org.apache.nlpcraft.probe.mgrs.dialogflow.NCDialogFlowManager
import org.apache.nlpcraft.probe.mgrs.model.NCModelManager
-import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.NCEnricherUtils
import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.dictionary.NCDictionaryEnricher
import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.limit.NCLimitEnricher
import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.model.NCModelEnricher
@@ -47,7 +46,7 @@ import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.relation.NCRelationEnricher
import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.sort.NCSortEnricher
import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.stopword.NCStopWordEnricher
import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.suspicious.NCSuspiciousNounsEnricher
-import org.apache.nlpcraft.probe.mgrs.nlp.impl._
+import org.apache.nlpcraft.probe.mgrs.nlp.impl.{NCVariantsCreator, _}
import org.apache.nlpcraft.probe.mgrs.nlp.validate._
import scala.collection.JavaConverters._
@@ -395,7 +394,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats {
val diff = notes2.filter(n ⇒ !notes1.contains(n))
val diffRedundant = diff.flatMap(n2 ⇒
- notes1.find(n1 ⇒ NCEnricherUtils.equalOrSimilar(n1, n2, nlpSen)) match {
+ notes1.find(n1 ⇒ nlpSen.notesEqualOrSimilar(n1, n2)) match {
case Some(similar) ⇒ Some(n2 → similar)
case None ⇒ None
}
@@ -436,7 +435,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats {
logger.info(s"Enrichment finished [step=$step]")
}
- NCEnricherUtils.collapse(mdlDec, nlpSen.clone(), span).
+ nlpSen.clone().collapse().
// Sorted to support deterministic logs.
sortBy(p ⇒
p.map(p ⇒ {
@@ -474,19 +473,18 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats {
}
val meta = mutable.HashMap.empty[String, Any] ++ senMeta
- val varsNlp = sensSeq.map(_.toSeq)
val req = NCRequestImpl(meta, srvReqId)
- var senVars = NCEnricherUtils.convert(mdlDec, srvReqId, varsNlp)
+ var senVars = NCVariantsCreator.makeVariants(mdlDec, srvReqId, sensSeq)
// Sentence variants can be filtered by model.
- val fltSenVars: Seq[(Seq[NCToken], Int)] =
+ val fltSenVars: Seq[(NCVariant, Int)] =
senVars.
zipWithIndex.
- flatMap { case (variant, i) ⇒ if (mdlDec.model.onParsedVariant(new NCVariantImpl(variant.asJava))) Some(variant, i) else None }
+ flatMap { case (variant, i) ⇒ if (mdlDec.model.onParsedVariant(variant)) Some(variant, i) else None }
senVars = fltSenVars.map(_._1)
- val allVars = senVars.flatten
+ val allVars = senVars.flatMap(_.asScala)
// Prints here only filtered variants.
val fltIdxs = fltSenVars.map { case (_, i) ⇒ i }
@@ -499,7 +497,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats {
zipWithIndex.
flatMap { case (sen, i) ⇒ if (fltIdxs.contains(i)) Some(sen) else None }.
zipWithIndex.foreach { case (sen, i) ⇒
- NCTokenLogger.prepareTable(sen).
+ NCTokenLogger.prepareTable(sen.asScala).
info(
logger,
Some(s"Parsing variant #${i + 1} for: $txt")
@@ -528,7 +526,7 @@ object NCProbeEnrichmentManager extends NCService with NCOpenCensusModelStats {
}
override def isOwnerOf(tok: NCToken): Boolean = allVars.contains(tok)
- override def getVariants: util.Collection[_ <: NCVariant] = senVars.map(s ⇒ new NCVariantImpl(s.asJava)).asJava
+ override def getVariants: util.Collection[_ <: NCVariant] = senVars.asJava
}
if (logHldr != null) {
diff --git a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCEnricherUtils.scala b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCEnricherUtils.scala
deleted file mode 100644
index 2103e63..0000000
--- a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/NCEnricherUtils.scala
+++ /dev/null
@@ -1,779 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.probe.mgrs.nlp.enrichers
-
-import java.io.Serializable
-import java.util
-import java.util.Collections
-
-import com.typesafe.scalalogging.LazyLogging
-import io.opencensus.trace.Span
-import org.apache.nlpcraft.common.nlp.pos.NCPennTreebank
-import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote, NCNlpSentenceToken}
-import org.apache.nlpcraft.common.{NCE, TOK_META_ALIASES_KEY}
-import org.apache.nlpcraft.model.NCToken
-import org.apache.nlpcraft.model.impl.NCTokenImpl
-import org.apache.nlpcraft.probe.mgrs.NCModelDecorator
-
-import scala.collection.JavaConverters._
-import scala.collection.{Map, Seq, Set, mutable}
-
-/**
- *
- */
-object NCEnricherUtils extends LazyLogging {
- /**
- *
- * @param note
- * @param withIndexes
- * @param withReferences
- * @return
- */
- def getKey(note: NCNlpSentenceNote, withIndexes: Boolean = true, withReferences: Boolean = true): Seq[Any] = {
- def addRefs(names: String*): Seq[String] = if (withReferences) names else Seq.empty
-
- val names: Seq[String] =
- if (note.isUser)
- Seq.empty
- else
- note.noteType match {
- case "nlpcraft:continent" ⇒ Seq("continent")
- case "nlpcraft:subcontinent" ⇒ Seq("continent", "subcontinent")
- case "nlpcraft:country" ⇒ Seq("continent", "subcontinent", "country")
- case "nlpcraft:region" ⇒ Seq("continent", "subcontinent", "country", "region")
- case "nlpcraft:city" ⇒ Seq("continent", "subcontinent", "country", "region", "city")
- case "nlpcraft:metro" ⇒ Seq("metro")
- case "nlpcraft:date" ⇒ Seq("from", "to")
- case "nlpcraft:relation" ⇒ Seq("type", "note") ++ addRefs("indexes")
- case "nlpcraft:sort" ⇒ Seq("asc", "subjnotes", "bynotes") ++ addRefs("subjindexes", "byindexes")
- case "nlpcraft:limit" ⇒ Seq("limit", "note") ++ addRefs("indexes", "asc") // Asc flag has sense only with references for limit.
- case "nlpcraft:coordinate" ⇒ Seq("latitude", "longitude")
- case "nlpcraft:num" ⇒ Seq("from", "to", "unit", "unitType")
- case x if x.startsWith("google:") ⇒ Seq("meta", "mentionsBeginOffsets", "mentionsContents", "mentionsTypes")
- case x if x.startsWith("stanford:") ⇒ Seq("nne")
- case x if x.startsWith("opennlp:") ⇒ Seq.empty
- case x if x.startsWith("spacy:") ⇒ Seq("vector")
-
- case _ ⇒ throw new AssertionError(s"Unexpected note type: ${note.noteType}")
- }
-
- val seq1 = if (withIndexes) Seq(note.wordIndexes, note.noteType) else Seq(note.noteType)
- val seq2 = names.map(name ⇒ note.getOrElse(name, null))
-
- seq1 ++ seq2
- }
-
- /**
- * Fixes tokens positions.
- *
- * @param ns Sentence.
- * @param notNlpTypes Token types.
- */
- private def collapse(ns: NCNlpSentence, notNlpTypes: Seq[String]): Boolean = {
- ns.
- filter(!_.isNlp).
- filter(_.isStopWord).
- flatten.
- filter(_.isNlp).
- foreach(n ⇒ ns.fixNote(n, "stopWord" → false))
-
- val nsNotes: Map[String, Seq[Int]] = ns.tokens.flatten.map(p ⇒ p.noteType → p.tokenIndexes).toMap
-
- for (
- t ← ns.tokens;
- stopReason ← t.stopsReasons
- if nsNotes.getOrElse(stopReason.noteType, Seq.empty) == stopReason.tokenIndexes
- )
- ns.fixNote(t.getNlpNote, "stopWord" → true)
-
- val history = mutable.ArrayBuffer.empty[(Int, Int)]
-
- notNlpTypes.foreach(typ ⇒ zipNotes(ns, typ, notNlpTypes, history))
-
- unionStops(ns, notNlpTypes, history)
-
- val res =
- Seq("nlpcraft:relation", "nlpcraft:limit").forall(t ⇒ fixIndexesReferences(t, ns, history)) &&
- fixIndexesReferencesList("nlpcraft:sort", "subjindexes", "subjnotes", ns, history) &&
- fixIndexesReferencesList("nlpcraft:sort", "byindexes", "bynotes", ns, history)
-
- if (res)
- // Validation (all indexes calculated well)
- require(
- !ns.flatten.
- exists(n ⇒ ns.filter(_.wordIndexes.exists(n.wordIndexes.contains)).exists(t ⇒ !t.contains(n))),
- s"Invalid sentence:\n" +
- ns.map(t ⇒
- // Human readable invalid sentence for debugging.
- s"${t.origText}{index:${t.index}}[${t.map(n ⇒ s"${n.noteType}, {range:${n.tokenFrom}-${n.tokenTo}}").mkString("|")}]"
- ).mkString("\n")
- )
- else
- logger.trace(s"Invalid notes combination skipped: [${nsNotes.map(p ⇒ s"${p._1} → [${p._2.mkString(",")}]").mkString("|")}]")
-
- res
- }
-
- /**
- *
- * @param ns
- * @param idxs
- * @param notesType
- * @param note
- * @return
- */
- private def checkRelation(ns: NCNlpSentence, idxs: Seq[Int], notesType: String, note: NCNlpSentenceNote): Boolean = {
- val types =
- idxs.flatMap(idx ⇒ {
- val types = ns(idx).map(p ⇒ p).filter(!_.isNlp).map(_.noteType)
-
- types.size match {
- case 0 ⇒ None
- case 1 ⇒ Some(types.head)
- case _ ⇒ throw new AssertionError(s"Unexpected tokes: ${ns(idx)}")
- }
- }).distinct
-
-
- /**
- Example:
- 1. Sentence 'maximum x' (single element related function)
- - maximum is aggregate function linked to date element.
- - x defined as 2 elements: date and num.
- So, the variant 'maximum x (as num)' should be excluded.
-
- 2. Sentence 'compare x and y' (multiple elements related function)
- - compare is relation function linked to date element.
- - x an y defined as 2 elements: date and num.
- So, variants 'x (as num) and x (as date)' and 'x (as date) and x (as num)'
- should't be excluded, but invalid relation should be deleted for these combinations.
- */
-
- types.size match {
- case 0 ⇒ throw new AssertionError(s"Unexpected empty types [notesType=$notesType]")
- case 1 ⇒ types.head == notesType
- case _ ⇒
- // Equal elements should be processed together with function element.
- if (types.size == 1)
- false
- else {
- ns.removeNote(note)
-
- true
- }
- }
- }
-
- /**
- * Fixes notes with references to other notes indexes.
- * Note that 'idxsField' is 'indexes' and 'noteField' is 'note' for all kind of references.
- *
- * @param noteType Note type.
- * @param ns Sentence.
- * @param history Indexes transformation history.
- * @return Valid flag.
- */
- private def fixIndexesReferences(noteType: String, ns: NCNlpSentence, history: Seq[(Int, Int)]): Boolean = {
- ns.filter(_.isTypeOf(noteType)).foreach(tok ⇒
- tok.getNoteOpt(noteType, "indexes") match {
- case Some(n) ⇒
- val idxs: Seq[Int] = n.data[java.util.List[Int]]("indexes").asScala
- var fixed = idxs
-
- history.foreach { case (idxOld, idxNew) ⇒ fixed = fixed.map(i ⇒ if (i == idxOld) idxNew else i) }
-
- fixed = fixed.distinct
-
- if (idxs != fixed) {
- ns.fixNote(n, "indexes" → fixed.asJava.asInstanceOf[java.io.Serializable])
-
- def x(seq: Seq[Int]): String = s"[${seq.mkString(", ")}]"
-
- logger.trace(s"`$noteType` note `indexes` fixed [old=${x(idxs)}}, new=${x(fixed)}]")
- }
- case None ⇒ // No-op.
- }
- )
-
- ns.flatMap(_.getNotes(noteType)).forall(
- n ⇒ checkRelation(ns, n.data[java.util.List[Int]]("indexes").asScala, n.data[String]("note"), n)
- )
- }
-
- /**
- * Fixes notes with references list to other notes indexes.
- *
- * @param noteType Note type.
- * @param idxsField Indexes field.
- * @param noteField Note field.
- * @param ns Sentence.
- * @param history Indexes transformation history.
- * @return Valid flag.
- */
- private def fixIndexesReferencesList(
- noteType: String,
- idxsField: String,
- noteField: String,
- ns: NCNlpSentence,
- history: Seq[(Int, Int)]
- ): Boolean = {
- var ok = true
-
- for (tok ← ns.filter(_.isTypeOf(noteType)) if ok)
- tok.getNoteOpt(noteType, idxsField) match {
- case Some(n) ⇒
- val idxs: Seq[Seq[Int]] = n.data[java.util.List[java.util.List[Int]]](idxsField).asScala.map(_.asScala)
- var fixed = idxs
-
- history.foreach { case (idxOld, idxNew) ⇒ fixed = fixed.map(_.map(i ⇒ if (i == idxOld) idxNew else i).distinct) }
-
- if (fixed.forall(_.size == 1)) {
- // Fix double dimension array to one dimension,
- // so it should be called always in spite of 'fixIndexesReferences' method.
- ns.fixNote(n, idxsField → fixed.map(_.head).asJava.asInstanceOf[java.io.Serializable])
-
- def x(seq: Seq[Seq[Int]]): String = s"[${seq.map(p ⇒ s"[${p.mkString(",")}]").mkString(", ")}]"
-
- logger.trace(s"`$noteType` note `indexes` fixed [old=${x(idxs)}}, new=${x(fixed)}]")
- }
- else
- ok = false
- case None ⇒ // No-op.
- }
-
- ok &&
- ns.flatMap(_.getNotes(noteType)).forall(rel ⇒
- rel.dataOpt[java.util.List[Int]](idxsField) match {
- case Some(idxsList) ⇒
- val notesTypes = rel.data[util.List[String]](noteField)
-
- require(idxsList.size() == notesTypes.size())
-
- idxsList.asScala.zip(notesTypes.asScala).forall {
- case (idxs, notesType) ⇒ checkRelation(ns, Seq(idxs), notesType, rel)
- }
- case None ⇒ true
- }
- )
- }
-
- /**
- * Zip notes with same type.
- *
- * @param ns Sentence.
- * @param nType Notes type.
- * @param userNotesTypes Notes types.
- * @param history Indexes transformation history.
- */
- private def zipNotes(
- ns: NCNlpSentence,
- nType: String,
- userNotesTypes: Seq[String],
- history: mutable.ArrayBuffer[(Int, Int)]
- ): Unit = {
- val nts = ns.getNotes(nType).filter(n ⇒ n.tokenFrom != n.tokenTo).sortBy(_.tokenFrom)
-
- val overlapped =
- nts.flatMap(n ⇒ n.tokenFrom to n.tokenTo).map(ns(_)).exists(
- t ⇒ userNotesTypes.map(pt ⇒ t.getNotes(pt).size).sum > 1
- )
-
- if (nts.nonEmpty && !overlapped) {
- val nsCopyToks = ns.clone()
- ns.clear()
-
- val buf = mutable.ArrayBuffer.empty[Int]
-
- for (i ← nsCopyToks.indices)
- nts.find(_.tokenIndexes.contains(i)) match {
- case Some(n) ⇒
- if (!buf.contains(n.tokenFrom)) {
- buf += n.tokenFrom
-
- ns += mkCompound(ns, nsCopyToks, n.tokenIndexes, stop = false, ns.size, Some(n), history)
- }
- case None ⇒ simpleCopy(ns, history, nsCopyToks, i)
- }
-
- fixIndexes(ns, userNotesTypes)
- }
- }
-
- /**
- * Glues stop words.
- *
- * @param ns Sentence.
- * @param userNoteTypes Notes types.
- * @param history Indexes transformation history.
- */
- private def unionStops(
- ns: NCNlpSentence,
- userNoteTypes: Seq[String],
- history: mutable.ArrayBuffer[(Int, Int)]
- ): Unit = {
- // Java collection used because using scala collections (mutable.Buffer.empty[mutable.Buffer[Token]]) is reason
- // Of compilation errors which seems as scala compiler internal error.
- val bufs = new util.ArrayList[mutable.Buffer[NCNlpSentenceToken]]()
-
- def last[T](l: util.List[T]): T = l.get(l.size() - 1)
-
- ns.filter(t ⇒ t.isStopWord && !t.isBracketed).foreach(t ⇒
- if (!bufs.isEmpty && last(bufs).last.index + 1 == t.index)
- last(bufs) += t
- else
- bufs.add(mutable.Buffer.empty[NCNlpSentenceToken] :+ t)
- )
-
- val idxsSeq = bufs.asScala.filter(_.lengthCompare(1) > 0).map(_.map(_.index))
-
- if (idxsSeq.nonEmpty) {
- val nsCopyToks = ns.clone()
- ns.clear()
-
- val buf = mutable.Buffer.empty[Int]
-
- for (i ← nsCopyToks.indices)
- idxsSeq.find(_.contains(i)) match {
- case Some(idxs) ⇒
- if (!buf.contains(idxs.head)) {
- buf += idxs.head
-
- ns += mkCompound(ns, nsCopyToks, idxs, stop = true, ns.size, None, history)
- }
- case None ⇒ simpleCopy(ns, history, nsCopyToks, i)
- }
-
- fixIndexes(ns, userNoteTypes)
- }
- }
-
- /**
- * Copies token.
- *
- * @param ns Sentence.
- * @param history Indexes transformation history.
- * @param toksCopy Copied tokens.
- * @param i Index.
- */
- private def simpleCopy(ns: NCNlpSentence, history: mutable.ArrayBuffer[(Int, Int)], toksCopy: NCNlpSentence, i: Int): Seq[NCNlpSentenceToken] = {
- val tokCopy = toksCopy(i)
-
- history += tokCopy.index → ns.size
-
- ns += tokCopy.clone(ns.size)
- }
-
- /**
- * Fixes indexes for all notes after recreating tokens.
- *
- * @param ns Sentence.
- * @param userNoteTypes Notes types.
- */
- private def fixIndexes(ns: NCNlpSentence, userNoteTypes: Seq[String]) {
- // Replaces other notes indexes.
- for (t ← userNoteTypes :+ "nlpcraft:nlp"; note ← ns.getNotes(t)) {
- val toks = ns.filter(_.contains(note)).sortBy(_.index)
-
- val newNote = note.clone(toks.map(_.index), toks.flatMap(_.wordIndexes).sorted)
-
- toks.foreach(t ⇒ {
- t.remove(note)
- t.add(newNote)
- })
- }
-
- // Special case - field index of core NLP note.
- ns.zipWithIndex.foreach { case (tok, idx) ⇒ ns.fixNote(tok.getNlpNote, "index" → idx) }
- }
-
- /**
- * Makes compound note.
- *
- * @param ns Sentence.
- * @param nsCopyToks Tokens.
- * @param indexes Indexes.
- * @param stop Flag.
- * @param idx Index.
- * @param commonNote Common note.
- * @param history Indexes transformation history.
- */
- private def mkCompound(
- ns: NCNlpSentence,
- nsCopyToks: Seq[NCNlpSentenceToken],
- indexes: Seq[Int],
- stop: Boolean,
- idx: Int,
- commonNote: Option[NCNlpSentenceNote],
- history: mutable.ArrayBuffer[(Int, Int)]
- ): NCNlpSentenceToken = {
- val t = NCNlpSentenceToken(idx)
-
- // Note, it adds stop-words too.
- val content = nsCopyToks.zipWithIndex.filter(p ⇒ indexes.contains(p._2)).map(_._1)
-
- content.foreach(t ⇒ history += t.index → idx)
-
- def mkValue(get: NCNlpSentenceToken ⇒ String): String = {
- val buf = mutable.Buffer.empty[String]
-
- val n = content.size - 1
-
- content.zipWithIndex.foreach(p ⇒ {
- val t = p._1
- val idx = p._2
-
- buf += get(t)
-
- if (idx < n && t.endCharIndex != content(idx + 1).startCharIndex)
- buf += " "
- })
-
- buf.mkString
- }
-
- val origText = mkValue((t: NCNlpSentenceToken) ⇒ t.origText)
-
- val idxs = Seq(idx)
- val wordIdxs = content.flatMap(_.wordIndexes).sorted
-
- val direct =
- commonNote match {
- case Some(n) if n.isUser ⇒ n.isDirect
- case _ ⇒ content.forall(_.isDirect)
- }
-
- val params = Seq(
- "index" → idx,
- "pos" → NCPennTreebank.SYNTH_POS,
- "posDesc" → NCPennTreebank.SYNTH_POS_DESC,
- "lemma" → mkValue((t: NCNlpSentenceToken) ⇒ t.lemma),
- "origText" → origText,
- "normText" → mkValue((t: NCNlpSentenceToken) ⇒ t.normText),
- "stem" → mkValue((t: NCNlpSentenceToken) ⇒ t.stem),
- "start" → content.head.startCharIndex,
- "end" → content.last.endCharIndex,
- "charLength" → origText.length,
- "quoted" → false,
- "stopWord" → stop,
- "bracketed" → false,
- "direct" → direct,
- "dict" → (if (nsCopyToks.size == 1) nsCopyToks.head.getNlpNote.data[Boolean]("dict") else false),
- "english" → nsCopyToks.forall(_.getNlpNote.data[Boolean]("english")),
- "swear" → nsCopyToks.exists(_.getNlpNote.data[Boolean]("swear"))
- )
-
- val nlpNote = NCNlpSentenceNote(idxs, wordIdxs, "nlpcraft:nlp", params: _*)
-
- t.add(nlpNote)
-
- // Adds processed note with fixed indexes.
- commonNote match {
- case Some(n) ⇒
- ns.removeNote(n)
- t.add(n.clone(idxs, wordIdxs))
- case None ⇒ // No-op.
- }
-
- t
- }
-
- /**
- * This collapser handles several tasks:
- * - "overall" collapsing after all other individual collapsers had their turn.
- * - Special further enrichment of tokens like linking, etc.
- *
- * In all cases of overlap (full or partial) - the "longest" note wins. In case of overlap and equal
- * lengths - the winning note is chosen based on this priority.
- *
- * @param mdl
- * @param ns
- * @param parent Optional parent span.
- * @return
- */
- @throws[NCE]
- def collapse(mdl: NCModelDecorator, ns: NCNlpSentence, parent: Span = null): Seq[NCNlpSentence] = {
- // Always deletes `similar` notes.
- // Some words with same note type can be detected various ways.
- // We keep only one variant - with `best` direct and sparsity parameters,
- // other variants for these words are redundant.
- val redundant: Seq[NCNlpSentenceNote] =
- ns.flatten.filter(!_.isNlp).distinct.
- groupBy(p ⇒ getKey(p)).
- map(p ⇒ p._2.sortBy(p ⇒
- (
- // System notes don't have such flags.
- if (p.isUser) {
- if (p.isDirect) 0 else 1
- }
- else
- 0,
- if (p.isUser) p.sparsity else 0
- )
- )).
- flatMap(_.drop(1)).
- toSeq
-
- redundant.foreach(ns.removeNote)
-
- def getNotNlpNotes(toks: Seq[NCNlpSentenceToken]): Seq[NCNlpSentenceNote] =
- toks.flatten.filter(!_.isNlp).distinct
-
- val delCombs: Seq[NCNlpSentenceNote] =
- getNotNlpNotes(ns).
- flatMap(note ⇒ getNotNlpNotes(ns.slice(note.tokenFrom, note.tokenTo + 1)).filter(_ != note)).
- distinct
-
- val toksByIdx: Seq[Seq[NCNlpSentenceNote]] =
- delCombs.flatMap(note ⇒ note.wordIndexes.map(_ → note)).
- groupBy { case (idx, _) ⇒ idx }.
- map { case (_, seq) ⇒ seq.map { case (_, note) ⇒ note } }.
- toSeq.sortBy(-_.size)
-
- val minDelSize = if (toksByIdx.isEmpty) 1 else toksByIdx.map(_.size).max - 1
-
- val sens =
- if (delCombs.nonEmpty) {
- val deleted = mutable.ArrayBuffer.empty[Seq[NCNlpSentenceNote]]
-
- val sens =
- (minDelSize to delCombs.size).
- flatMap(i ⇒
- delCombs.combinations(i).
- filter(delComb ⇒ !toksByIdx.exists(_.count(note ⇒ !delComb.contains(note)) > 1))
- ).
- sortBy(_.size).
- flatMap(delComb ⇒
- // Already processed with less subset of same deleted tokens.
- if (!deleted.exists(_.forall(delComb.contains))) {
- val nsClone = ns.clone()
-
- delComb.foreach(nsClone.removeNote)
-
- // Has overlapped notes for some tokens.
- require(!nsClone.exists(_.count(!_.isNlp) > 1))
-
- deleted += delComb
-
- val notNlpTypes = getNotNlpNotes(nsClone).map(_.noteType).distinct
-
- if (collapse(nsClone, notNlpTypes)) Some(nsClone) else None
- }
- else
- None
- )
-
- // It removes sentences which have only one difference - 'direct' flag of their user tokens.
- // `Direct` sentences have higher priority.
- case class Key(
- sysNotes: Seq[Map[String, java.io.Serializable]],
- userNotes: Seq[Map[String, java.io.Serializable]]
- )
- case class Value(sentence: NCNlpSentence, directCount: Int)
-
- val m = mutable.HashMap.empty[Key, Value]
-
- sens.map(sen ⇒ {
- val notes = sen.flatten
-
- val sysNotes = notes.filter(_.isSystem)
- val nlpNotes = notes.filter(_.isNlp)
- val userNotes = notes.filter(_.isUser)
-
- def get(seq: Seq[NCNlpSentenceNote], keys2Skip: String*): Seq[Map[String, java.io.Serializable]] =
- seq.map(p ⇒
- // We have to delete some keys to have possibility to compare sentences.
- p.clone().filter(_._1 != "direct")
- )
-
- (Key(get(sysNotes), get(userNotes)), sen, nlpNotes.map(p ⇒ if (p.isDirect) 0 else 1).sum)
- }).
- foreach { case (key, sen, directCnt) ⇒
- m.get(key) match {
- case Some(v) ⇒
- // Best sentence is sentence with `direct` synonyms.
- if (v.directCount > directCnt)
- m += key → Value(sen, directCnt)
- case None ⇒ m += key → Value(sen, directCnt)
- }
- }
-
- m.values.map(_.sentence).toSeq
- }
- else {
- if (collapse(ns, getNotNlpNotes(ns).map(_.noteType).distinct)) Seq(ns) else Seq.empty
- }.distinct
-
- sens.foreach(sen ⇒
- sen.foreach(tok ⇒
- tok.size match {
- case 1 ⇒ require(tok.head.isNlp, s"Unexpected non-'nlpcraft:nlp' token: $tok")
- case 2 ⇒ require(tok.head.isNlp ^ tok.last.isNlp, s"Unexpected token notes: $tok")
- case _ ⇒ require(requirement = false, s"Unexpected token notes count: $tok")
- }
- )
- )
-
- // Drops similar sentences (with same tokens structure).
- // Among similar sentences we prefer one with minimal free words count.
- sens.groupBy(_.flatten.filter(!_.isNlp).map(note ⇒ getKey(note, withIndexes = false))).
- map { case (_, seq) ⇒ seq.minBy(_.filter(p ⇒ p.isNlp && !p.isStopWord).map(_.wordIndexes.length).sum) }.
- toSeq
- }
-
- /**
- *
- * @param mdl
- * @param srvReqId
- * @param nlpToks
- * @return
- */
- def convert(mdl: NCModelDecorator, srvReqId: String, nlpToks: Seq[Seq[NCNlpSentenceToken]]): Seq[Seq[NCToken]] = {
- val seq = nlpToks.map(_.map(nlpTok ⇒ NCTokenImpl(mdl, srvReqId, nlpTok) → nlpTok))
- val toks = seq.map(_.map { case (tok, _) ⇒ tok })
-
- case class Key(id: String, from: Int, to: Int)
-
- val keys2Toks = toks.flatten.map(t ⇒ Key(t.getId, t.getStartCharIndex, t.getEndCharIndex) → t).toMap
- val partsKeys = mutable.HashSet.empty[Key]
-
- seq.flatten.foreach { case (tok, tokNlp) ⇒
- if (tokNlp.isUser) {
- val userNotes = tokNlp.filter(_.isUser)
-
- require(userNotes.size == 1)
-
- val optList: Option[util.List[util.HashMap[String, Serializable]]] = userNotes.head.dataOpt("parts")
-
- optList match {
- case Some(list) ⇒
- val keys =
- list.asScala.map(m ⇒
- Key(
- m.get("id").asInstanceOf[String],
- m.get("startcharindex").asInstanceOf[Integer],
- m.get("endcharindex").asInstanceOf[Integer]
- )
- )
- val parts = keys.map(keys2Toks)
-
- parts.zip(list.asScala).foreach { case (part, map) ⇒
- map.get(TOK_META_ALIASES_KEY) match {
- case null ⇒ // No-op.
- case aliases ⇒ part.getMetadata.put(TOK_META_ALIASES_KEY, aliases.asInstanceOf[Object])
- }
- }
-
- tok.setParts(parts)
- partsKeys ++= keys
-
- case None ⇒ // No-op.
- }
- }
- }
-
- // We can't collapse parts earlier, because we need them here (setParts method, few lines above.)
- toks.filter(sen ⇒
- !sen.exists(t ⇒
- t.getId != "nlpcraft:nlp" &&
- partsKeys.contains(Key(t.getId, t.getStartCharIndex, t.getEndCharIndex))
- )
- )
- }
-
- /**
- * Checks whether important tokens deleted as stopwords or not.
- *
- * @param ns Sentence.
- * @param toks Tokens in which some stopwords can be deleted.
- * @param isImportant Token important criteria.
- */
- def validImportant(
- ns: NCNlpSentence,
- toks: Seq[NCNlpSentenceToken],
- isImportant: NCNlpSentenceToken ⇒ Boolean
- ): Boolean = {
- val idxs = toks.map(_.index)
-
- require(idxs == idxs.sorted)
-
- val toks2 = ns.slice(idxs.head, idxs.last + 1)
-
- toks.length == toks2.length || toks.count(isImportant) == toks2.count(isImportant)
- }
-
- /**
- *
- * @param n1
- * @param n2
- * @param sen
- * @return
- */
- def equalOrSimilar(n1: NCNlpSentenceNote, n2: NCNlpSentenceNote, sen: NCNlpSentence): Boolean = {
- require(n1.noteType == n2.noteType)
-
- val stopIdxs = sen.filter(_.isStopWord).map(_.index)
-
- // One possible difference - stopwords indexes.
- def wordsEqualOrSimilar0(n1: NCNlpSentenceNote, n2: NCNlpSentenceNote): Boolean = {
- val set1 = n1.wordIndexes.toSet
- val set2 = n2.wordIndexes.toSet
-
- set1 == set2 || set1.subsetOf(set2) && set2.diff(set1).forall(stopIdxs.contains)
- }
-
- def wordsEqualOrSimilar(n1: NCNlpSentenceNote, n2: NCNlpSentenceNote): Boolean =
- wordsEqualOrSimilar0(n1, n2) || wordsEqualOrSimilar0(n2, n1)
-
- def tokensEqualOrSimilar0(set1: Set[NCNlpSentenceToken], set2: Set[NCNlpSentenceToken]): Boolean =
- set1 == set2 || set1.subsetOf(set2) && set2.diff(set1).forall(_.isStopWord)
-
- def tokensEqualOrSimilar(set1: Set[NCNlpSentenceToken], set2: Set[NCNlpSentenceToken]): Boolean =
- tokensEqualOrSimilar0(set1, set2) || tokensEqualOrSimilar0(set2, set1)
-
- def getList(n: NCNlpSentenceNote, refIdxName: String): Set[NCNlpSentenceToken] =
- n.getOrElse(refIdxName, Collections.emptyList).asInstanceOf[java.util.List[Int]].asScala.
- map(sen(_)).toSet
-
- def getListList(n: NCNlpSentenceNote, refIdxName: String): Set[NCNlpSentenceToken] =
- n.getOrElse(refIdxName, Collections.emptyList).asInstanceOf[java.util.List[java.util.List[Int]]].asScala.
- flatMap(_.asScala.map(sen(_))).toSet
-
- def referencesEqualOrSimilar0(n1: NCNlpSentenceNote, n2: NCNlpSentenceNote): Boolean = {
- require(n1.noteType == n2.noteType)
-
- n1.noteType match {
- case "nlpcraft:sort" ⇒
- tokensEqualOrSimilar(getListList(n1, "subjindexes"), getListList(n2, "subjindexes")) &&
- tokensEqualOrSimilar(getListList(n1, "byindexes"), getListList(n2, "byindexes"))
- case "nlpcraft:limit" ⇒
- tokensEqualOrSimilar(getList(n1, "indexes"), getList(n2, "indexes"))
- case "nlpcraft:reference" ⇒
- tokensEqualOrSimilar(getList(n1, "indexes"), getList(n2, "indexes"))
-
- case _ ⇒ true
- }
- }
-
- def referencesEqualOrSimilar(n1: NCNlpSentenceNote, n2: NCNlpSentenceNote): Boolean =
- referencesEqualOrSimilar0(n1, n2) || referencesEqualOrSimilar0(n2, n1)
-
- def getUniqueKey0(n: NCNlpSentenceNote): Seq[Any] = getKey(n, withIndexes = false, withReferences = false)
-
- getUniqueKey0(n1) == getUniqueKey0(n2) && wordsEqualOrSimilar(n1, n1) && referencesEqualOrSimilar(n2, n1)
- }
-}
diff --git a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/limit/NCLimitEnricher.scala b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/limit/NCLimitEnricher.scala
index a533c68..fc826ea 100644
--- a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/limit/NCLimitEnricher.scala
+++ b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/limit/NCLimitEnricher.scala
@@ -27,7 +27,7 @@ import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote, NCNlpSe
import org.apache.nlpcraft.common.{NCE, NCService}
import org.apache.nlpcraft.probe.mgrs.NCModelDecorator
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher
-import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.NCEnricherUtils
+import org.apache.nlpcraft.probe.mgrs.nlp.impl.NCVariantsCreator
import scala.collection.JavaConverters._
import scala.collection.{Map, Seq, mutable}
@@ -201,6 +201,24 @@ object NCLimitEnricher extends NCProbeEnricher {
super.stop()
}
+ /**
+ * Checks whether important tokens deleted as stopwords or not.
+ *
+ * @param ns Sentence.
+ * @param toks Tokens in which some stopwords can be deleted.
+ */
+ private def validImportant(ns: NCNlpSentence, toks: Seq[NCNlpSentenceToken]): Boolean = {
+ def isImportant(t: NCNlpSentenceToken): Boolean = isUserNotValue(t) || TECH_WORDS.contains(t.stem)
+
+ val idxs = toks.map(_.index)
+
+ require(idxs == idxs.sorted)
+
+ val toks2 = ns.slice(idxs.head, idxs.last + 1)
+
+ toks.length == toks2.length || toks.count(isImportant) == toks2.count(isImportant)
+ }
+
@throws[NCE]
override def enrich(mdl: NCModelDecorator, ns: NCNlpSentence, senMeta: Map[String, Serializable], parent: Span = null): Unit =
startScopedSpan("enrich", parent,
@@ -211,13 +229,9 @@ object NCLimitEnricher extends NCProbeEnricher {
val numsMap = NCNumericManager.find(ns).filter(_.unit.isEmpty).map(p ⇒ p.tokens → p).toMap
val groupsMap = groupNums(ns, numsMap.values)
- def isImportant(t: NCNlpSentenceToken): Boolean = isUserNotValue(t) || TECH_WORDS.contains(t.stem)
-
// Tries to grab tokens reverse way.
// Example: A, B, C ⇒ ABC, BC, AB .. (BC will be processed first)
- for (toks ← ns.tokenMixWithStopWords().sortBy(p ⇒ (-p.size, -p.head.index))
- if NCEnricherUtils.validImportant(ns, toks, isImportant)
- )
+ for (toks ← ns.tokenMixWithStopWords().sortBy(p ⇒ (-p.size, -p.head.index)) if validImportant(ns, toks))
tryToMatch(numsMap, groupsMap, toks) match {
case Some(m) ⇒
for (refNote ← m.refNotes) {
@@ -232,7 +246,7 @@ object NCLimitEnricher extends NCProbeEnricher {
val note = NCNlpSentenceNote(m.matched.map(_.index), TOK_ID, params: _*)
- if (!notes.exists(n ⇒ NCEnricherUtils.equalOrSimilar(note, n, ns))) {
+ if (!notes.exists(n ⇒ ns.notesEqualOrSimilar(n, note))) {
notes += note
m.matched.foreach(_.add(note))
diff --git a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
index 0cc3978..255e575 100644
--- a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
+++ b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala
@@ -25,8 +25,7 @@ import org.apache.nlpcraft.common._
import org.apache.nlpcraft.common.nlp.{NCNlpSentenceToken, _}
import org.apache.nlpcraft.model._
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher
-import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.NCEnricherUtils
-import org.apache.nlpcraft.probe.mgrs.nlp.impl.NCRequestImpl
+import org.apache.nlpcraft.probe.mgrs.nlp.impl.{NCVariantsCreator, NCRequestImpl}
import org.apache.nlpcraft.probe.mgrs.{NCModelDecorator, NCSynonym}
import scala.collection.JavaConverters._
@@ -376,12 +375,8 @@ object NCModelEnricher extends NCProbeEnricher with DecorateAsScala {
found = false
if (collapsedSens == null)
- collapsedSens =
- NCEnricherUtils.convert(
- mdl,
- ns.srvReqId,
- NCEnricherUtils.collapse(mdl, ns.clone(), span).map(_.tokens)
- )
+ collapsedSens = NCVariantsCreator.
+ makeVariants(mdl, ns.srvReqId, ns.clone().collapse()).map(_.asScala)
if (seq == null)
seq = convert(ns, collapsedSens, toks)
diff --git a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/relation/NCRelationEnricher.scala b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/relation/NCRelationEnricher.scala
index 558fcee..799e687 100644
--- a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/relation/NCRelationEnricher.scala
+++ b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/relation/NCRelationEnricher.scala
@@ -26,7 +26,7 @@ import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote, NCNlpSe
import org.apache.nlpcraft.common.{NCE, NCService}
import org.apache.nlpcraft.probe.mgrs.NCModelDecorator
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher
-import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.NCEnricherUtils
+import org.apache.nlpcraft.probe.mgrs.nlp.impl.NCVariantsCreator
import scala.collection.JavaConverters._
import scala.collection.{Map, Seq, mutable}
@@ -119,6 +119,25 @@ object NCRelationEnricher extends NCProbeEnricher {
super.stop()
}
+ /**
+ * Checks whether important tokens deleted as stopwords or not.
+ *
+ * @param ns Sentence.
+ * @param toks Tokens in which some stopwords can be deleted.
+ */
+ private def validImportant(ns: NCNlpSentence, toks: Seq[NCNlpSentenceToken]): Boolean = {
+ def isImportant(t: NCNlpSentenceToken): Boolean =
+ t.exists(n ⇒ n.isUser || REL_TYPES.contains(n.noteType)) || ALL_FUNC_STEMS.contains(t.stem)
+
+ val idxs = toks.map(_.index)
+
+ require(idxs == idxs.sorted)
+
+ val toks2 = ns.slice(idxs.head, idxs.last + 1)
+
+ toks.length == toks2.length || toks.count(isImportant) == toks2.count(isImportant)
+ }
+
@throws[NCE]
override def enrich(mdl: NCModelDecorator, ns: NCNlpSentence, senMeta: Map[String, Serializable], parent: Span = null): Unit =
startScopedSpan("enrich", parent,
@@ -129,10 +148,7 @@ object NCRelationEnricher extends NCProbeEnricher {
// Example: A, B, C ⇒ ABC, AB, BC .. (AB will be processed first)
val notes = mutable.HashSet.empty[NCNlpSentenceNote]
- def isImportant(t: NCNlpSentenceToken): Boolean =
- t.exists(n ⇒ n.isUser || REL_TYPES.contains(n.noteType)) || ALL_FUNC_STEMS.contains(t.stem)
-
- for (toks ← ns.tokenMixWithStopWords() if NCEnricherUtils.validImportant(ns, toks, isImportant))
+ for (toks ← ns.tokenMixWithStopWords() if validImportant(ns, toks))
tryToMatch(toks) match {
case Some(m) ⇒
for (refNote ← m.refNotes) {
@@ -144,7 +160,7 @@ object NCRelationEnricher extends NCProbeEnricher {
"note" → refNote
)
- if (!notes.exists(n ⇒ NCEnricherUtils.equalOrSimilar(note, n, ns))) {
+ if (!notes.exists(n ⇒ ns.notesEqualOrSimilar(n, note))) {
notes += note
m.matched.filter(_ != m.matchedHead).foreach(_.addStopReason(note))
diff --git a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
index fc9d4a8..c8ebb35 100644
--- a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
+++ b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
@@ -26,7 +26,7 @@ import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.nlp.{NCNlpSentence, NCNlpSentenceNote, NCNlpSentenceToken}
import org.apache.nlpcraft.probe.mgrs.NCModelDecorator
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher
-import org.apache.nlpcraft.probe.mgrs.nlp.enrichers.NCEnricherUtils
+import org.apache.nlpcraft.probe.mgrs.nlp.impl.NCVariantsCreator
import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
@@ -423,15 +423,32 @@ object NCSortEnricher extends NCProbeEnricher {
res
}
+ /**
+ * Checks whether important tokens deleted as stopwords or not.
+ *
+ * @param ns Sentence.
+ * @param toks Tokens in which some stopwords can be deleted.
+ */
+ private def validImportant(ns: NCNlpSentence, toks: Seq[NCNlpSentenceToken]): Boolean = {
+ def isImportant(t: NCNlpSentenceToken): Boolean = isUserNotValue(t) || MASK_WORDS.contains(t.stem)
+
+ val idxs = toks.map(_.index)
+
+ require(idxs == idxs.sorted)
+
+ val toks2 = ns.slice(idxs.head, idxs.last + 1)
+
+ toks.length == toks2.length || toks.count(isImportant) == toks2.count(isImportant)
+ }
+
override def enrich(mdl: NCModelDecorator, ns: NCNlpSentence, meta: Map[String, Serializable], parent: Span): Unit =
startScopedSpan("enrich", parent,
"srvReqId" → ns.srvReqId,
"modelId" → mdl.model.getId,
"txt" → ns.text) { _ ⇒
val notes = mutable.HashSet.empty[NCNlpSentenceNote]
- def isImportant(t: NCNlpSentenceToken): Boolean = isUserNotValue(t) || MASK_WORDS.contains(t.stem)
- for (toks ← ns.tokenMixWithStopWords() if NCEnricherUtils.validImportant(ns, toks, isImportant)) {
+ for (toks ← ns.tokenMixWithStopWords() if validImportant(ns, toks)) {
tryToMatch(toks) match {
case Some(m) ⇒
def addNotes(
@@ -449,7 +466,7 @@ object NCSortEnricher extends NCProbeEnricher {
def mkNote(params: ArrayBuffer[(String, Any)]): Unit = {
val note = NCNlpSentenceNote(m.main.map(_.index), TOK_ID, params: _*)
- if (!notes.exists(n ⇒ NCEnricherUtils.equalOrSimilar(note, n, ns))) {
+ if (!notes.exists(n ⇒ ns.notesEqualOrSimilar(n, note))) {
notes += note
m.main.foreach(_.add(note))
diff --git a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/impl/NCVariantsCreator.scala b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/impl/NCVariantsCreator.scala
new file mode 100644
index 0000000..9cc3789
--- /dev/null
+++ b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/impl/NCVariantsCreator.scala
@@ -0,0 +1,79 @@
+package org.apache.nlpcraft.probe.mgrs.nlp.impl
+
+import java.io.Serializable
+import java.util
+
+import org.apache.nlpcraft.common.TOK_META_ALIASES_KEY
+import org.apache.nlpcraft.common.nlp.NCNlpSentence
+import org.apache.nlpcraft.model.NCVariant
+import org.apache.nlpcraft.model.impl.{NCTokenImpl, NCVariantImpl}
+import org.apache.nlpcraft.probe.mgrs.NCModelDecorator
+
+import scala.collection.JavaConverters._
+import scala.collection.{Seq, mutable}
+import scala.language.implicitConversions
+
+/**
+ *
+ */
+private[nlp] object NCVariantsCreator {
+ /**
+ *
+ * @param mdl
+ * @param srvReqId
+ * @param sens
+ * @return
+ */
+ def makeVariants(mdl: NCModelDecorator, srvReqId: String, sens: Seq[NCNlpSentence]): Seq[NCVariant] = {
+ val seq = sens.map(_.toSeq.map(nlpTok ⇒ NCTokenImpl(mdl, srvReqId, nlpTok) → nlpTok))
+ val toks = seq.map(_.map { case (tok, _) ⇒ tok })
+
+ case class Key(id: String, from: Int, to: Int)
+
+ val keys2Toks = toks.flatten.map(t ⇒ Key(t.getId, t.getStartCharIndex, t.getEndCharIndex) → t).toMap
+ val partsKeys = mutable.HashSet.empty[Key]
+
+ seq.flatten.foreach { case (tok, tokNlp) ⇒
+ if (tokNlp.isUser) {
+ val userNotes = tokNlp.filter(_.isUser)
+
+ require(userNotes.size == 1)
+
+ val optList: Option[util.List[util.HashMap[String, Serializable]]] = userNotes.head.dataOpt("parts")
+
+ optList match {
+ case Some(list) ⇒
+ val keys =
+ list.asScala.map(m ⇒
+ Key(
+ m.get("id").asInstanceOf[String],
+ m.get("startcharindex").asInstanceOf[Integer],
+ m.get("endcharindex").asInstanceOf[Integer]
+ )
+ )
+ val parts = keys.map(keys2Toks)
+
+ parts.zip(list.asScala).foreach { case (part, map) ⇒
+ map.get(TOK_META_ALIASES_KEY) match {
+ case null ⇒ // No-op.
+ case aliases ⇒ part.getMetadata.put(TOK_META_ALIASES_KEY, aliases.asInstanceOf[Object])
+ }
+ }
+
+ tok.setParts(parts)
+ partsKeys ++= keys
+
+ case None ⇒ // No-op.
+ }
+ }
+ }
+
+ // We can't collapse parts earlier, because we need them here (setParts method, few lines above.)
+ toks.filter(sen ⇒
+ !sen.exists(t ⇒
+ t.getId != "nlpcraft:nlp" &&
+ partsKeys.contains(Key(t.getId, t.getStartCharIndex, t.getEndCharIndex))
+ )
+ ).map(p ⇒ new NCVariantImpl(p.asJava))
+ }
+}