You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by kr...@apache.org on 2026/08/16 01:12:58 UTC

(opennlp) 05/06: OPENNLP-1897: Document the normalized-term workflow with a mirror-tested example

This is an automated email from the ASF dual-hosted git repository.

krickert pushed a commit to branch OPENNLP-1897-term-vectors
in repository https://gitbox.apache.org/repos/asf/opennlp.git

commit b306b6a89e9f70ff35ad6a95ca63b81629d7215f
Author: Kristian Rickert <kr...@gmail.com>
AuthorDate: Mon Aug 10 01:21:47 2026 -0400

    OPENNLP-1897: Document the normalized-term workflow with a mirror-tested example
    
    Extend the term vector section of the manual with the plain-normalizer path:
    a whitespace tokenizer plus a shipped case folder built by
    TextNormalizer.builder().caseFold().build(), folding "Word word WORD" into
    one term with three occurrence spans that stay the tokens' original spans.
    The example lives in opennlp-runtime because the shipped folds do, and
    TermVectorNormalizedExampleTest#testCaseFoldedTermsKeepOriginalSpans asserts
    the behavior shown in the listing.
---
 .../TermVectorNormalizedExampleTest.java           | 69 ++++++++++++++++++++++
 opennlp-docs/src/docbkx/document.xml               | 30 ++++++++--
 2 files changed, 95 insertions(+), 4 deletions(-)

diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java
new file mode 100644
index 000000000..aa5b7e5db
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.termvector;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.document.Annotation;
+import opennlp.tools.document.Document;
+import opennlp.tools.document.DocumentAnalyzer;
+import opennlp.tools.document.TokenizerAnnotator;
+import opennlp.tools.tokenize.WhitespaceTokenizer;
+import opennlp.tools.util.Span;
+import opennlp.tools.util.normalizer.CharSequenceNormalizer;
+import opennlp.tools.util.normalizer.TextNormalizer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Mirrors the normalized-term workflow shown in the term vector section of the manual:
+ * a whitespace tokenizer feeding a {@link TermVectorAnnotator} built with a shipped,
+ * plain {@link CharSequenceNormalizer} case folder. The folder defines term identity per
+ * token, so case variants group under one term, while every occurrence span stays the
+ * token's own span in the original text.
+ */
+public class TermVectorNormalizedExampleTest {
+
+  /**
+   * The documented example: {@code "Word word WORD"} yields one term, {@code "word"},
+   * with three occurrence spans, each the token's exact original span.
+   */
+  @Test
+  void testCaseFoldedTermsKeepOriginalSpans() {
+    final CharSequenceNormalizer folder = TextNormalizer.builder().caseFold().build();
+    final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
+        .add(new TokenizerAnnotator(WhitespaceTokenizer.INSTANCE))
+        .add(new TermVectorAnnotator(folder))
+        .build();
+
+    final Document document = analyzer.analyze("Word word WORD");
+
+    final List<Annotation<TermVector>> vectors =
+        document.get(TermVectorAnnotator.TERM_VECTORS);
+    assertEquals(1, vectors.size());
+    assertEquals(new TermVector("word", 3,
+        List.of(new Span(0, 4), new Span(5, 9), new Span(10, 14))), vectors.get(0).value());
+
+    // The spans point at the original surface forms, not the folded term.
+    final List<String> surfaceForms = vectors.get(0).value().spans().stream()
+        .map(s -> s.getCoveredText(document.text()).toString()).toList();
+    assertEquals(List.of("Word", "word", "WORD"), surfaceForms);
+  }
+}
diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml
index 0abe7aa0c..d157eac2f 100644
--- a/opennlp-docs/src/docbkx/document.xml
+++ b/opennlp-docs/src/docbkx/document.xml
@@ -214,11 +214,13 @@ int firstTokenLength = lengths.get(0).value(); // 3, for "The" at [0..3)]]>
 			term vector layer for index consumers: one <code>TermVector</code> per distinct
 			term, carrying the term string, its occurrence count, and the occurrence spans in
 			original text coordinates. Term identity comes from the annotator's inputs: without
-			a normalizer the term is the token's covered text as-is; with an
+			a normalizer the term is the token's covered text as-is; with a plain
+			<code>CharSequenceNormalizer</code>, the general path, each token's covered text is
+			normalized on its own to become the term; with an
 			<code>OffsetAwareNormalizer</code> the document text is normalized once and each
-			token's covered normalized text becomes the term, so tokens differing only by a
-			normalization fold group together while their spans keep pointing into the original
-			text. The layer preserves first-occurrence order.
+			token's covered normalized text becomes the term. On every path tokens differing
+			only by a normalization fold group together while their spans keep pointing into
+			the original text. The layer preserves first-occurrence order.
 			<code>TermVectorPipelineTest#testTokenizerAndTermVectorPipeline</code> asserts the
 			behavior shown here.
 			<programlisting language="java"><![CDATA[
@@ -232,6 +234,26 @@ List<Annotation<TermVector>> vectors = document.get(TermVectorAnnotator.TERM_VEC
 // vectors.get(1).value() is ("dog", 2, [[4..7), [19..22)])]]>
 			</programlisting>
 		</para>
+		<para>
+			The normalized-term workflow passes any shipped
+			<code>CharSequenceNormalizer</code> to the plain-normalizer constructor: the
+			normalizer is applied to each token's covered text to produce the term, so the
+			case, NFC, and accent folds that <code>buildAligned()</code> rejects all work
+			here, and every occurrence span stays the token's own span in the original text.
+			<code>TermVectorNormalizedExampleTest#testCaseFoldedTermsKeepOriginalSpans</code>
+			asserts the behavior shown here.
+			<programlisting language="java"><![CDATA[
+CharSequenceNormalizer folder = TextNormalizer.builder().caseFold().build();
+DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
+    .add(new TokenizerAnnotator(WhitespaceTokenizer.INSTANCE))
+    .add(new TermVectorAnnotator(folder))
+    .build();
+
+Document document = analyzer.analyze("Word word WORD");
+List<Annotation<TermVector>> vectors = document.get(TermVectorAnnotator.TERM_VECTORS);
+// vectors.get(0).value() is ("word", 3, [[0..4), [5..9), [10..14)])]]>
+			</programlisting>
+		</para>
 		<para>
 			Scoring-only consumers that never read offsets can skip storing them:
 			<code>new TermVectorAnnotator(TermVectorAnnotator.Mode.SCORING_ONLY)</code> emits