You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ba...@apache.org on 2026/09/21 08:17:36 UTC
(hop) branch main updated: Fixes #8415 : Add a Text chunker transform to split documents for emb… (#8419)
This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 23fd61820a Fixes #8415 : Add a Text chunker transform to split documents for emb… (#8419)
23fd61820a is described below
commit 23fd61820a40738ff090b2d9aed51fbaceea29b7
Author: Bart Maertens <ba...@know.bi>
AuthorDate: Mon Sep 21 10:17:29 2026 +0200
Fixes #8415 : Add a Text chunker transform to split documents for emb… (#8419)
* Fixes #8415 : Add a Text chunker transform to split documents for embedding and retrieval
* Issue #8415 : Address review feedback on the Text chunker transform
* Issue #8415 : Fix the dialog leftovers from the GuiCompositeWidgets rewrite
The generated dialogs fill an enum combo with toString() and read it back
with Enum.valueOf, and build a plain SWT Combo when an element has no
variable support. The rewrite missed both, so:
- ChunkingStrategyType no longer overrides toString(), otherwise a changed
strategy was dropped on OK without any error. fromString still accepts the
old display text, so pipelines saved earlier keep loading
- the stream field combos are filled through GuiCompositeWidgets.setComboValues
and their saved selection is restored afterwards, since setting items clears
the widget text
- content type enablement reads the strategy widget through Combo as well as
ComboVar, and parses it rather than comparing one spelling, so it follows a
live change
Adds a guard asserting name().equals(toString()) for every enum the dialog
shows, and one pinning that the old display text still parses.
---
assemblies/plugins/pom.xml | 6 +
.../assets/images/transforms/icons/chunker.svg | 26 ++
docs/hop-user-manual/modules/ROOT/nav.adoc | 1 +
.../pages/pipeline/transforms/textchunker.adoc | 108 ++++++
.../transforms/0115-text-chunker-character.hpl | 125 ++++++
.../transforms/0115-text-chunker-paragraph.hpl | 125 ++++++
.../0115-text-chunker-structure-markdown.hpl | 126 ++++++
.../transforms/0117-text-chunker-redaction.hpl | 153 ++++++++
.../datasets/golden-text-chunker-character.csv | 4 +
.../datasets/golden-text-chunker-paragraph.csv | 4 +
.../datasets/golden-text-chunker-redaction.csv | 24 ++
.../golden-text-chunker-structure-markdown.csv | 10 +
.../transforms/main-0115-text-chunker.hwf | 88 +++++
.../dataset/golden-text-chunker-character.json | 39 ++
.../dataset/golden-text-chunker-paragraph.json | 39 ++
.../dataset/golden-text-chunker-redaction.json | 15 +
.../golden-text-chunker-structure-markdown.json | 39 ++
.../0115-text-chunker-character UNIT.json | 43 +++
.../0115-text-chunker-paragraph UNIT.json | 43 +++
.../0115-text-chunker-structure-markdown UNIT.json | 43 +++
.../0117-text-chunker-redaction UNIT.json | 28 ++
plugins/transforms/pom.xml | 1 +
plugins/transforms/textchunker/pom.xml | 61 +++
.../textchunker/src/assembly/assembly.xml | 43 +++
.../hop/pipeline/transforms/chunker/Chunk.java | 64 +++
.../pipeline/transforms/chunker/TextChunker.java | 263 +++++++++++++
.../transforms/chunker/TextChunkerData.java | 64 +++
.../transforms/chunker/TextChunkerDialog.java | 202 ++++++++++
.../transforms/chunker/TextChunkerMeta.java | 427 +++++++++++++++++++++
.../chunking/CharacterChunkingStrategy.java | 88 +++++
.../chunker/chunking/ChunkingStrategy.java | 43 +++
.../chunker/chunking/ChunkingStrategyFactory.java | 67 ++++
.../chunker/chunking/ChunkingStrategyType.java | 77 ++++
.../chunking/ParagraphChunkingStrategy.java | 150 ++++++++
.../chunking/StructureChunkingStrategy.java | 115 ++++++
.../chunker/document/AsciiDocDocumentParser.java | 35 ++
.../transforms/chunker/document/ContentType.java | 63 +++
.../chunker/document/ContentTypeResolver.java | 42 ++
.../transforms/chunker/document/DocumentNode.java | 115 ++++++
.../chunker/document/DocumentParser.java | 25 ++
.../chunker/document/DocumentParserRegistry.java | 81 ++++
.../chunker/document/HeadingParserSupport.java | 112 ++++++
.../chunker/document/MarkdownDocumentParser.java | 35 ++
.../chunker/document/PlainTextDocumentParser.java | 31 ++
.../chunker/document/SecretRedaction.java | 71 ++++
.../document/hopxml/HopPipelineXmlParser.java | 119 ++++++
.../document/hopxml/HopWorkflowXmlParser.java | 117 ++++++
.../chunker/document/hopxml/HopXmlSupport.java | 307 +++++++++++++++
.../document/metadata/HopMetadataJsonParser.java | 232 +++++++++++
.../textchunker/src/main/resources/chunker.svg | 26 ++
.../chunker/messages/messages_en_US.properties | 85 ++++
.../textchunker/src/main/resources/version.xml | 20 +
.../samples/transforms/ai-text-chunker-basic.hpl | 166 ++++++++
.../transforms/chunker/TextChunkerMetaTest.java | 187 +++++++++
.../transforms/chunker/TextChunkerTest.java | 230 +++++++++++
.../chunking/CharacterChunkingStrategyTest.java | 213 ++++++++++
.../chunking/ParagraphChunkingStrategyTest.java | 267 +++++++++++++
.../chunking/StructureChunkingStrategyTest.java | 102 +++++
.../document/DocumentParserRegistryTest.java | 93 +++++
.../chunker/document/SecretRedactionTest.java | 65 ++++
.../document/hopxml/HopPipelineXmlParserTest.java | 120 ++++++
.../document/hopxml/HopWorkflowXmlParserTest.java | 62 +++
.../chunker/document/hopxml/HopXmlSupportTest.java | 83 ++++
.../metadata/HopMetadataJsonParserTest.java | 128 ++++++
64 files changed, 5986 insertions(+)
diff --git a/assemblies/plugins/pom.xml b/assemblies/plugins/pom.xml
index b0c9a2d16d..fa18ad39b4 100644
--- a/assemblies/plugins/pom.xml
+++ b/assemblies/plugins/pom.xml
@@ -1542,6 +1542,12 @@
<version>${project.version}</version>
<type>zip</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-transform-textchunker</artifactId>
+ <version>${project.version}</version>
+ <type>zip</type>
+ </dependency>
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-textfile</artifactId>
diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/chunker.svg b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/chunker.svg
new file mode 100644
index 0000000000..6bb4fe69af
--- /dev/null
+++ b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/chunker.svg
@@ -0,0 +1,26 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64" viewBox="0 0 64 64">
+ <defs>
+ <linearGradient id="gradient" x1="0%" y1="0%" x2="100%" y2="100%">
+ <stop offset="0%" style="stop-color:#4a90e2;stop-opacity:1" />
+ <stop offset="100%" style="stop-color:#2a6fc9;stop-opacity:1" />
+ </linearGradient>
+ </defs>
+ <rect width="64" height="64" rx="8" fill="url(#gradient)"/>
+
+ <!-- Document icon -->
+ <rect x="12" y="14" width="40" height="36" rx="2" fill="white"/>
+ <rect x="14" y="16" width="36" height="32" rx="1" fill="#f0f0f0"/>
+
+ <!-- Lines representing text -->
+ <line x1="16" y1="20" x2="48" y2="20" stroke="#666" stroke-width="1.5"/>
+ <line x1="16" y1="24" x2="44" y2="24" stroke="#666" stroke-width="1.5"/>
+ <line x1="16" y1="28" x2="48" y2="28" stroke="#666" stroke-width="1.5"/>
+
+ <!-- Plus sign for chunking -->
+ <line x1="30" y1="34" x2="34" y2="34" stroke="white" stroke-width="2"/>
+ <line x1="32" y1="32" x2="32" y2="36" stroke="white" stroke-width="2"/>
+
+ <!-- Additional lines -->
+ <line x1="16" y1="36" x2="30" y2="36" stroke="#666" stroke-width="1.5"/>
+ <line x1="38" y1="36" x2="48" y2="36" stroke="#666" stroke-width="1.5"/>
+</svg>
diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc
index c05479893a..133debb528 100644
--- a/docs/hop-user-manual/modules/ROOT/nav.adoc
+++ b/docs/hop-user-manual/modules/ROOT/nav.adoc
@@ -317,6 +317,7 @@ under the License.
*** xref:pipeline/transforms/tableinput.adoc[Table Input]
*** xref:pipeline/transforms/tableoutput.adoc[Table Output]
*** xref:pipeline/transforms/terafast.adoc[Teradata Bulk Loader]
+*** xref:pipeline/transforms/textchunker.adoc[Text chunker]
*** xref:pipeline/transforms/textfileinput.adoc[Text File Input]
*** xref:pipeline/transforms/textfileoutput.adoc[Text File Output]
*** xref:pipeline/transforms/tokenreplacement.adoc[Token replacement]
diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textchunker.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textchunker.adoc
new file mode 100644
index 0000000000..190025be31
--- /dev/null
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/textchunker.adoc
@@ -0,0 +1,108 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:documentationPath: /pipeline/transforms/
+:language: en_US
+:description: Split text into chunks for embedding and retrieval, without breaking words or sentences.
+
+= image:transforms/icons/chunker.svg[Text chunker transform Icon, role="image-doc-icon"] Text chunker
+
+[%noheader,cols="3a,1a", role="table-no-borders" ]
+|===
+|
+== Description
+
+The Text chunker transform splits a text field into smaller chunks and emits one output row per chunk, carrying the fields of the input row along with each.
+
+Chunking is the first step of a retrieval pipeline: an embedding model has a bounded context, and retrieval works better on passages than on whole documents. A typical pipeline is Text chunker, then an embedding transform, then a vector store.
+
+The transform has no dependency on an AI provider or a model. It is plain text processing and can be used on its own wherever a large document needs to be broken into passages.
+
+|
+== Supported Engines
+[%noheader,cols="2,1a",frame=none, role="table-supported-engines"]
+!===
+!Hop Engine! image:check_mark.svg[Supported, 24]
+!Single Threaded! image:check_mark.svg[Supported, 24]
+!Native Spark! image:check_mark.svg[Supported, 24]
+!Beam Spark! image:question_mark.svg[Maybe Supported, 24]
+!Beam Flink! image:question_mark.svg[Maybe Supported, 24]
+!Beam Dataflow! image:question_mark.svg[Maybe Supported, 24]
+!===
+|===
+
+== Chunking strategies
+
+[options="header"]
+|===
+|Strategy |Behaviour
+
+|Character
+|Split on a fixed character count, backing off to the nearest word boundary so words are not cut in half. Overlap carries the tail of each chunk into the next, which keeps a sentence that straddles a boundary retrievable from both sides.
+
+|Paragraph
+|Split on paragraph breaks, packing whole paragraphs up to the chunk size. A paragraph longer than the chunk size falls back to character splitting.
+
+|Structure
+|Parse the document into a heading tree and emit one chunk per section, prefixing each with a breadcrumb of its heading path so an isolated chunk still says where it came from. A section larger than the chunk size falls back to character splitting within that section.
+|===
+
+=== Structure parsers
+
+The Structure strategy selects a parser from the content type:
+
+[options="header"]
+|===
+|Content type |Parsed structure
+
+|Auto|Infer from the content type field, or from the text itself.
+|Plain|Blank-line separated blocks.
+|Markdown|`#` heading levels.
+|AsciiDoc|`=` heading levels.
+|Pipeline|A Hop pipeline (`.hpl`): one section per transform, plus the hops between them.
+|Workflow|A Hop workflow (`.hwf`): one section per action, plus the hops between them.
+|Metadata|A Hop metadata JSON document: an overview plus one section per entry.
+|===
+
+The Hop-native parsers make a pipeline, a workflow or a metadata file retrievable per transform, action or entry rather than as one opaque blob.
+
+== Options
+
+[options="header"]
+|===
+|Option |Description
+
+|Transform name|Name of the transform, unique within the pipeline.
+|Input field|Field holding the text to split.
+|Source document ID field|Optional field holding the identifier of the source document. When empty, a row counter is used.
+|Output chunk field|Output field that receives the chunk text.
+|Chunking strategy|Character, Paragraph or Structure, as described above.
+|Content type|Document format used by the Structure strategy.
+|Content type field|Optional field naming the content type per row, for a stream that mixes formats.
+|Chunk size|Maximum chunk size in characters. For Paragraph and Structure this is a ceiling that triggers the character fallback.
+|Chunk overlap|Characters carried from the end of one chunk into the start of the next. Character strategy only.
+|Include metadata|Add the chunk metadata fields below to the output.
+|Chunk index field|Output field for the position of the chunk within its document.
+|Chunk start position field|Output field for the character offset of the chunk in the source text.
+|Document ID field|Output field for the document identifier.
+|Total chunks field|Output field for the number of chunks the document produced.
+|===
+
+== Notes
+
+Chunk overlap applies to the Character strategy. The Paragraph and Structure strategies split on natural boundaries, where an overlap would duplicate whole paragraphs or sections between neighbouring chunks.
+
+Keep *Include metadata* enabled when the chunks are going into a vector store: the document ID and chunk index are what let a later search merge neighbouring chunks back into a passage, and re-index a single document without touching the rest.
diff --git a/integration-tests/transforms/0115-text-chunker-character.hpl b/integration-tests/transforms/0115-text-chunker-character.hpl
new file mode 100644
index 0000000000..0ebe471140
--- /dev/null
+++ b/integration-tests/transforms/0115-text-chunker-character.hpl
@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0115-text-chunker-character</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Character chunking backs off to a word boundary and carries an overlap into the next chunk.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/16 12:00:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop> <from>Document</from> <to>Chunk</to> <enabled>Y</enabled> </hop>
+ <hop> <from>Chunk</from> <to>OUTPUT</to> <enabled>Y</enabled> </hop>
+ </order>
+ <transform>
+ <name>Document</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>Apache Hop is a data orchestration platform for building pipelines and workflows.</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>content</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Chunk</name>
+ <type>TextChunker</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <inputField>content</inputField>
+ <outputChunkField>chunk_text</outputChunkField>
+ <chunkingStrategy>CHARACTER</chunkingStrategy>
+ <chunkSize>40</chunkSize>
+ <chunkOverlap>10</chunkOverlap>
+ <includeMetadata>Y</includeMetadata>
+ <chunkIndexField>chunk_index</chunkIndexField>
+ <chunkStartPosField>chunk_start_position</chunkStartPosField>
+ <documentIdField>chunk_doc_id</documentIdField>
+ <chunkCountField>total_chunks</chunkCountField>
+ <attributes/>
+ <GUI>
+ <xloc>320</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>OUTPUT</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>544</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0115-text-chunker-paragraph.hpl b/integration-tests/transforms/0115-text-chunker-paragraph.hpl
new file mode 100644
index 0000000000..8ba9525fe2
--- /dev/null
+++ b/integration-tests/transforms/0115-text-chunker-paragraph.hpl
@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0115-text-chunker-paragraph</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Paragraph chunking splits on paragraph breaks and reports the offset of each paragraph.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/16 12:00:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop> <from>Document</from> <to>Chunk</to> <enabled>Y</enabled> </hop>
+ <hop> <from>Chunk</from> <to>OUTPUT</to> <enabled>Y</enabled> </hop>
+ </order>
+ <transform>
+ <name>Document</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>First paragraph text. Second paragraph text. Third paragraph text.</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>content</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Chunk</name>
+ <type>TextChunker</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <inputField>content</inputField>
+ <outputChunkField>chunk_text</outputChunkField>
+ <chunkingStrategy>PARAGRAPH</chunkingStrategy>
+ <chunkSize>25</chunkSize>
+ <chunkOverlap>0</chunkOverlap>
+ <includeMetadata>Y</includeMetadata>
+ <chunkIndexField>chunk_index</chunkIndexField>
+ <chunkStartPosField>chunk_start_position</chunkStartPosField>
+ <documentIdField>chunk_doc_id</documentIdField>
+ <chunkCountField>total_chunks</chunkCountField>
+ <attributes/>
+ <GUI>
+ <xloc>320</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>OUTPUT</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>544</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0115-text-chunker-structure-markdown.hpl b/integration-tests/transforms/0115-text-chunker-structure-markdown.hpl
new file mode 100644
index 0000000000..b48f201a46
--- /dev/null
+++ b/integration-tests/transforms/0115-text-chunker-structure-markdown.hpl
@@ -0,0 +1,126 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0115-text-chunker-structure-markdown</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Structure chunking parses Markdown headings and prefixes each section with its heading path.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/16 12:00:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop> <from>Document</from> <to>Chunk</to> <enabled>Y</enabled> </hop>
+ <hop> <from>Chunk</from> <to>OUTPUT</to> <enabled>Y</enabled> </hop>
+ </order>
+ <transform>
+ <name>Document</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item># Title Intro line. ## Section A Body A. ## Section B Body B.</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>content</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Chunk</name>
+ <type>TextChunker</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <inputField>content</inputField>
+ <outputChunkField>chunk_text</outputChunkField>
+ <chunkingStrategy>STRUCTURE</chunkingStrategy>
+ <contentType>MARKDOWN</contentType>
+ <chunkSize>200</chunkSize>
+ <chunkOverlap>0</chunkOverlap>
+ <includeMetadata>Y</includeMetadata>
+ <chunkIndexField>chunk_index</chunkIndexField>
+ <chunkStartPosField>chunk_start_position</chunkStartPosField>
+ <documentIdField>chunk_doc_id</documentIdField>
+ <chunkCountField>total_chunks</chunkCountField>
+ <attributes/>
+ <GUI>
+ <xloc>320</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>OUTPUT</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>544</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0117-text-chunker-redaction.hpl b/integration-tests/transforms/0117-text-chunker-redaction.hpl
new file mode 100644
index 0000000000..b91f36e91c
--- /dev/null
+++ b/integration-tests/transforms/0117-text-chunker-redaction.hpl
@@ -0,0 +1,153 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0117-text-chunker-redaction</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Credentials in Hop XML and metadata JSON must not reach chunk text, which is
+ normally sent to an embedding model.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/17 09:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/17 09:00:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop> <from>Definitions</from> <to>Chunk</to> <enabled>Y</enabled> </hop>
+ <hop> <from>Chunk</from> <to>Sort chunks</to> <enabled>Y</enabled> </hop>
+ <hop> <from>Sort chunks</from> <to>OUTPUT</to> <enabled>Y</enabled> </hop>
+ </order>
+ <transform>
+ <name>Definitions</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>pipeline</item>
+ <item><pipeline><info><name>secret-demo</name></info> <transform><name>DB</name><type>TableInput</type> <connection><hostname>localhost</hostname><username>hop</username> <password>letmein</password><lookupKeys>customer_id</lookupKeys></connection> </transform></pipeline></item>
+ </line>
+ <line>
+ <item>metadata</item>
+ <item>{"name":"prod-rest","restConnection":{"baseUrl":"https://api.example.com", "bearerToken":"tok-abcdef123456","proxyPassword":"hunter2"}}</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length><precision>-1</precision><set_empty_string>N</set_empty_string>
+ <name>source_type</name><type>String</type>
+ </field>
+ <field>
+ <length>-1</length><precision>-1</precision><set_empty_string>N</set_empty_string>
+ <name>content</name><type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI><xloc>96</xloc><yloc>96</yloc></GUI>
+ </transform>
+ <transform>
+ <name>Chunk</name>
+ <type>TextChunker</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <inputField>content</inputField>
+ <outputChunkField>chunk_text</outputChunkField>
+ <chunkingStrategy>STRUCTURE</chunkingStrategy>
+ <contentType>AUTO</contentType>
+ <contentTypeField>source_type</contentTypeField>
+ <chunkSize>4000</chunkSize>
+ <chunkOverlap>0</chunkOverlap>
+ <includeMetadata>N</includeMetadata>
+ <attributes/>
+ <GUI><xloc>320</xloc><yloc>96</yloc></GUI>
+ </transform>
+ <transform>
+ <name>Sort chunks</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <directory>${java.io.tmpdir}</directory>
+ <prefix>out</prefix>
+ <sort_size>1000000</sort_size>
+ <free_memory/>
+ <compress>N</compress>
+ <compress_variable/>
+ <unique_rows>N</unique_rows>
+ <fields>
+ <field>
+ <name>chunk_text</name>
+ <ascending>Y</ascending>
+ <case_sensitive>Y</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI><xloc>432</xloc><yloc>96</yloc></GUI>
+ </transform>
+ <transform>
+ <name>OUTPUT</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI><xloc>544</xloc><yloc>96</yloc></GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/datasets/golden-text-chunker-character.csv b/integration-tests/transforms/datasets/golden-text-chunker-character.csv
new file mode 100644
index 0000000000..c979ed665d
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-text-chunker-character.csv
@@ -0,0 +1,4 @@
+chunk_text,chunk_index,chunk_start_position,total_chunks
+"Apache Hop is a data orchestration ",0,0,3
+"estration platform for building ",1,25,3
+" building pipelines and workflows.",2,47,3
diff --git a/integration-tests/transforms/datasets/golden-text-chunker-paragraph.csv b/integration-tests/transforms/datasets/golden-text-chunker-paragraph.csv
new file mode 100644
index 0000000000..38aabc712b
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-text-chunker-paragraph.csv
@@ -0,0 +1,4 @@
+chunk_text,chunk_index,chunk_start_position,total_chunks
+"First paragraph text.",0,0,3
+"Second paragraph text.",1,23,3
+"Third paragraph text.",2,47,3
diff --git a/integration-tests/transforms/datasets/golden-text-chunker-redaction.csv b/integration-tests/transforms/datasets/golden-text-chunker-redaction.csv
new file mode 100644
index 0000000000..f0df7ad9f2
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-text-chunker-redaction.csv
@@ -0,0 +1,24 @@
+chunk_text
+"Overview
+
+Hop metadata 'prod-rest'"
+"Rest Connection
+
+{
+ ""baseUrl"" : ""https://api.example.com"",
+ ""bearerToken"" : ""**redacted**"",
+ ""proxyPassword"" : ""**redacted**""
+}"
+"secret-demo > Overview
+
+Hop pipeline 'secret-demo'"
+"secret-demo > Transform: DB (TableInput)
+
+transform:
+ name: DB
+ type: TableInput
+ connection:
+ hostname: localhost
+ username: hop
+ password: **redacted**
+ lookupKeys: customer_id"
diff --git a/integration-tests/transforms/datasets/golden-text-chunker-structure-markdown.csv b/integration-tests/transforms/datasets/golden-text-chunker-structure-markdown.csv
new file mode 100644
index 0000000000..1ac9e8448e
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-text-chunker-structure-markdown.csv
@@ -0,0 +1,10 @@
+chunk_text,chunk_index,chunk_start_position,total_chunks
+"Title
+
+Intro line.",0,0,3
+"Title > Section A
+
+Body A.",1,22,3
+"Title > Section B
+
+Body B.",2,45,3
diff --git a/integration-tests/transforms/main-0115-text-chunker.hwf b/integration-tests/transforms/main-0115-text-chunker.hwf
new file mode 100644
index 0000000000..9febafd28b
--- /dev/null
+++ b/integration-tests/transforms/main-0115-text-chunker.hwf
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0115-text-chunker</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/09/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/16 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>START</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>96</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0115-text-chunker UNIT</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0115-text-chunker-character UNIT</name>
+ </test_name>
+ <test_name>
+ <name>0115-text-chunker-paragraph UNIT</name>
+ </test_name>
+ <test_name>
+ <name>0115-text-chunker-structure-markdown UNIT</name>
+ </test_name>
+ <test_name>
+ <name>0117-text-chunker-redaction UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>256</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>START</from>
+ <to>0115-text-chunker UNIT</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/transforms/metadata/dataset/golden-text-chunker-character.json b/integration-tests/transforms/metadata/dataset/golden-text-chunker-character.json
new file mode 100644
index 0000000000..97e03e8832
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-text-chunker-character.json
@@ -0,0 +1,39 @@
+{
+ "base_filename": "golden-text-chunker-character.csv",
+ "name": "golden-text-chunker-character",
+ "description": "Character chunking: word-boundary back-off, overlap, and chunk metadata",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "chunk_text",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "chunk_index",
+ "field_format": "0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "chunk_start_position",
+ "field_format": "0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "total_chunks",
+ "field_format": "0"
+ }
+ ]
+}
diff --git a/integration-tests/transforms/metadata/dataset/golden-text-chunker-paragraph.json b/integration-tests/transforms/metadata/dataset/golden-text-chunker-paragraph.json
new file mode 100644
index 0000000000..a3ace8652f
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-text-chunker-paragraph.json
@@ -0,0 +1,39 @@
+{
+ "base_filename": "golden-text-chunker-paragraph.csv",
+ "name": "golden-text-chunker-paragraph",
+ "description": "Paragraph chunking: one chunk per paragraph with its source offset",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "chunk_text",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "chunk_index",
+ "field_format": "0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "chunk_start_position",
+ "field_format": "0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "total_chunks",
+ "field_format": "0"
+ }
+ ]
+}
diff --git a/integration-tests/transforms/metadata/dataset/golden-text-chunker-redaction.json b/integration-tests/transforms/metadata/dataset/golden-text-chunker-redaction.json
new file mode 100644
index 0000000000..180a05ae4a
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-text-chunker-redaction.json
@@ -0,0 +1,15 @@
+{
+ "base_filename": "golden-text-chunker-redaction.csv",
+ "name": "golden-text-chunker-redaction",
+ "description": "Chunk text from a Hop pipeline definition and a metadata document, with credentials redacted.",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "chunk_text",
+ "field_format": ""
+ }
+ ]
+}
diff --git a/integration-tests/transforms/metadata/dataset/golden-text-chunker-structure-markdown.json b/integration-tests/transforms/metadata/dataset/golden-text-chunker-structure-markdown.json
new file mode 100644
index 0000000000..6f89051cd6
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-text-chunker-structure-markdown.json
@@ -0,0 +1,39 @@
+{
+ "base_filename": "golden-text-chunker-structure-markdown.csv",
+ "name": "golden-text-chunker-structure-markdown",
+ "description": "Structure chunking of Markdown: one chunk per section, prefixed with its heading path",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "chunk_text",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "chunk_index",
+ "field_format": "0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "chunk_start_position",
+ "field_format": "0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "total_chunks",
+ "field_format": "0"
+ }
+ ]
+}
diff --git a/integration-tests/transforms/metadata/unit-test/0115-text-chunker-character UNIT.json b/integration-tests/transforms/metadata/unit-test/0115-text-chunker-character UNIT.json
new file mode 100644
index 0000000000..93a5cbd6b1
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0115-text-chunker-character UNIT.json
@@ -0,0 +1,43 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": false,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "chunk_text",
+ "data_set_field": "chunk_text"
+ },
+ {
+ "transform_field": "chunk_index",
+ "data_set_field": "chunk_index"
+ },
+ {
+ "transform_field": "chunk_start_position",
+ "data_set_field": "chunk_start_position"
+ },
+ {
+ "transform_field": "total_chunks",
+ "data_set_field": "total_chunks"
+ }
+ ],
+ "field_order": [
+ "chunk_text",
+ "chunk_index",
+ "chunk_start_position",
+ "total_chunks"
+ ],
+ "transform_name": "OUTPUT",
+ "data_set_name": "golden-text-chunker-character"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0115-text-chunker-character UNIT",
+ "description": "Character chunking: word-boundary back-off, overlap, and chunk metadata",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0115-text-chunker-character.hpl",
+ "test_type": "UNIT_TEST"
+}
diff --git a/integration-tests/transforms/metadata/unit-test/0115-text-chunker-paragraph UNIT.json b/integration-tests/transforms/metadata/unit-test/0115-text-chunker-paragraph UNIT.json
new file mode 100644
index 0000000000..29dacf2b53
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0115-text-chunker-paragraph UNIT.json
@@ -0,0 +1,43 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": false,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "chunk_text",
+ "data_set_field": "chunk_text"
+ },
+ {
+ "transform_field": "chunk_index",
+ "data_set_field": "chunk_index"
+ },
+ {
+ "transform_field": "chunk_start_position",
+ "data_set_field": "chunk_start_position"
+ },
+ {
+ "transform_field": "total_chunks",
+ "data_set_field": "total_chunks"
+ }
+ ],
+ "field_order": [
+ "chunk_text",
+ "chunk_index",
+ "chunk_start_position",
+ "total_chunks"
+ ],
+ "transform_name": "OUTPUT",
+ "data_set_name": "golden-text-chunker-paragraph"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0115-text-chunker-paragraph UNIT",
+ "description": "Paragraph chunking: one chunk per paragraph with its source offset",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0115-text-chunker-paragraph.hpl",
+ "test_type": "UNIT_TEST"
+}
diff --git a/integration-tests/transforms/metadata/unit-test/0115-text-chunker-structure-markdown UNIT.json b/integration-tests/transforms/metadata/unit-test/0115-text-chunker-structure-markdown UNIT.json
new file mode 100644
index 0000000000..84616afe12
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0115-text-chunker-structure-markdown UNIT.json
@@ -0,0 +1,43 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": false,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "chunk_text",
+ "data_set_field": "chunk_text"
+ },
+ {
+ "transform_field": "chunk_index",
+ "data_set_field": "chunk_index"
+ },
+ {
+ "transform_field": "chunk_start_position",
+ "data_set_field": "chunk_start_position"
+ },
+ {
+ "transform_field": "total_chunks",
+ "data_set_field": "total_chunks"
+ }
+ ],
+ "field_order": [
+ "chunk_text",
+ "chunk_index",
+ "chunk_start_position",
+ "total_chunks"
+ ],
+ "transform_name": "OUTPUT",
+ "data_set_name": "golden-text-chunker-structure-markdown"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0115-text-chunker-structure-markdown UNIT",
+ "description": "Structure chunking of Markdown: one chunk per section, prefixed with its heading path",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0115-text-chunker-structure-markdown.hpl",
+ "test_type": "UNIT_TEST"
+}
diff --git a/integration-tests/transforms/metadata/unit-test/0117-text-chunker-redaction UNIT.json b/integration-tests/transforms/metadata/unit-test/0117-text-chunker-redaction UNIT.json
new file mode 100644
index 0000000000..3c79382567
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0117-text-chunker-redaction UNIT.json
@@ -0,0 +1,28 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": false,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "chunk_text",
+ "data_set_field": "chunk_text"
+ }
+ ],
+ "field_order": [
+ "chunk_text"
+ ],
+ "transform_name": "OUTPUT",
+ "data_set_name": "golden-text-chunker-redaction"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0117-text-chunker-redaction UNIT",
+ "description": "Credentials in Hop XML and metadata JSON do not reach chunk text",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0117-text-chunker-redaction.hpl",
+ "test_type": "UNIT_TEST"
+}
diff --git a/plugins/transforms/pom.xml b/plugins/transforms/pom.xml
index 82f4740e66..37abb86be8 100644
--- a/plugins/transforms/pom.xml
+++ b/plugins/transforms/pom.xml
@@ -161,6 +161,7 @@
<module>tableexists</module>
<module>tableinput</module>
<module>tableoutput</module>
+ <module>textchunker</module>
<module>textfile</module>
<module>tika</module>
<module>tokenreplacement</module>
diff --git a/plugins/transforms/textchunker/pom.xml b/plugins/transforms/textchunker/pom.xml
new file mode 100644
index 0000000000..5c305626b9
--- /dev/null
+++ b/plugins/transforms/textchunker/pom.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-plugins-transforms</artifactId>
+ <version>2.20.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>hop-transform-textchunker</artifactId>
+ <packaging>jar</packaging>
+ <name>Hop Plugins Transforms Text Chunker</name>
+
+ <properties>
+ <!-- Jackson is compiled against but not shipped: the Hop distribution already carries
+ it in lib/core. Without these excludes the provided-scope dependency is packaged
+ into lib/core a second time, which is both 2.4 MB of waste and a second definition
+ of ObjectMapper for HopURLClassLoader to trip over. -->
+ <hop.libcore.exclude1>com.fasterxml.jackson.core:jackson-databind</hop.libcore.exclude1>
+ <hop.libcore.exclude2>com.fasterxml.jackson.core:jackson-core</hop.libcore.exclude2>
+ <hop.libcore.exclude3>com.fasterxml.jackson.core:jackson-annotations</hop.libcore.exclude3>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-libs</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/plugins/transforms/textchunker/src/assembly/assembly.xml b/plugins/transforms/textchunker/src/assembly/assembly.xml
new file mode 100644
index 0000000000..f987db82cd
--- /dev/null
+++ b/plugins/transforms/textchunker/src/assembly/assembly.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.2.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.2.0 http://maven.apache.org/xsd/assembly-2.2.0.xsd">
+ <id>hop-transform-textchunker</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <baseDirectory>.</baseDirectory>
+ <files>
+ <file>
+ <source>${project.basedir}/src/main/resources/version.xml</source>
+ <outputDirectory>${hop.plugin.libdir}</outputDirectory>
+ <filtered>true</filtered>
+ </file>
+ </files>
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}/src/main/samples</directory>
+ <outputDirectory>config/projects/samples/</outputDirectory>
+ </fileSet>
+ </fileSets>
+ <componentDescriptors>
+ <componentDescriptor>${maven.multiModuleProjectDirectory}/assemblies/shared/hop-plugin-libs.xml</componentDescriptor>
+ </componentDescriptors>
+</assembly>
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/Chunk.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/Chunk.java
new file mode 100644
index 0000000000..52f9202acf
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/Chunk.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+/** Represents a text chunk with metadata for tracking. */
+@Getter
+@AllArgsConstructor
+public class Chunk {
+
+ /** The text content of this chunk. */
+ private final String content;
+
+ /** The index of this chunk in the sequence (0-based). */
+ private final int index;
+
+ /** The starting position of this chunk in the original text. */
+ private final int startPosition;
+
+ /** The ending position (exclusive) of this chunk in the original text. */
+ private final int endPosition;
+
+ /**
+ * Gets the length of the chunk content in characters.
+ *
+ * @return The character count
+ */
+ public int getLength() {
+ return content != null ? content.length() : 0;
+ }
+
+ @Override
+ public String toString() {
+ return "Chunk{"
+ + "index="
+ + index
+ + ", start="
+ + startPosition
+ + ", end="
+ + endPosition
+ + ", length="
+ + getLength()
+ + ", content='"
+ + (content.length() > 50 ? content.substring(0, 50) + "..." : content)
+ + '\''
+ + '}';
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunker.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunker.java
new file mode 100644
index 0000000000..3a5463d470
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunker.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker;
+
+import java.util.List;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.RowDataUtil;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.BaseTransform;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategy;
+import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyFactory;
+import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyType;
+import org.apache.hop.pipeline.transforms.chunker.chunking.StructureChunkingStrategy;
+import org.apache.hop.pipeline.transforms.chunker.document.ContentType;
+import org.apache.hop.pipeline.transforms.chunker.document.ContentTypeResolver;
+
+/**
+ * Text Chunker transform - splits text into chunks for AI/ML processing. Supports multiple chunking
+ * strategies (character-based with word boundary respect, paragraph-based, structure-aware) with
+ * configurable overlap.
+ */
+public class TextChunker extends BaseTransform<TextChunkerMeta, TextChunkerData> {
+
+ private static final Class<?> PKG = TextChunkerMeta.class;
+
+ /** The chunking strategy instance. */
+ private ChunkingStrategy strategy;
+
+ /**
+ * Counter used to synthesise a document ID when no source field is configured. Prefixed with the
+ * transform copy number so parallel copies cannot produce colliding identifiers.
+ */
+ private long documentIdCounter = 0;
+
+ public TextChunker(
+ TransformMeta transformMeta,
+ TextChunkerMeta meta,
+ TextChunkerData data,
+ int copyNr,
+ PipelineMeta pipelineMeta,
+ Pipeline pipeline) {
+ super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline);
+ }
+
+ @Override
+ public boolean init() {
+ if (!super.init()) {
+ return false;
+ }
+ data.chunkSize = Const.toInt(resolve(meta.getChunkSize()), -1);
+ data.chunkOverlap = Const.toInt(resolve(meta.getChunkOverlap()), -1);
+ if (data.chunkSize <= 0) {
+ logError(
+ BaseMessages.getString(
+ PKG, "TextChunker.Validation.ChunkSizePositive", meta.getChunkSize()));
+ return false;
+ }
+ if (data.chunkOverlap < 0) {
+ data.chunkOverlap = 0;
+ }
+ strategy = ChunkingStrategyFactory.createStrategy(meta.getChunkingStrategy());
+ logBasic(
+ BaseMessages.getString(
+ PKG, "TextChunker.Log.Initialized", String.valueOf(meta.getChunkingStrategy())));
+ return true;
+ }
+
+ @Override
+ public boolean processRow() throws HopException {
+ Object[] row = getRow();
+
+ if (row == null) {
+ setOutputDone();
+ return false;
+ }
+
+ if (first) {
+ first = false;
+ data.inputRowMeta = getInputRowMeta();
+ data.outputRowMeta = data.inputRowMeta.clone();
+ meta.getFields(
+ data.outputRowMeta, getTransformName(), null, null, this, getMetadataProvider());
+ resolveFieldIndices();
+ }
+
+ try {
+ chunkRow(row);
+ } catch (HopException e) {
+ if (getTransformMeta().isDoingErrorHandling()) {
+ putError(data.inputRowMeta, row, 1, e.getMessage(), meta.getInputField(), "TEXTCHUNKER001");
+ } else {
+ throw e;
+ }
+ }
+
+ return true;
+ }
+
+ private void chunkRow(Object[] row) throws HopException {
+ String text = data.inputRowMeta.getString(row, data.inputFieldIndex);
+ String documentIdValue = resolveDocumentId(row);
+
+ if (Utils.isEmpty(text)) {
+ if (isRowLevel()) {
+ logRowlevel(BaseMessages.getString(PKG, "TextChunker.Log.EmptyText"));
+ }
+ putRow(data.outputRowMeta, createOutputRow(row, "", 0, 0, documentIdValue, 0));
+ return;
+ }
+
+ List<Chunk> chunks = chunk(row, text);
+ int totalChunks = chunks.size();
+
+ // Output each chunk as a new row (must copy row data per chunk; the buffer is reused).
+ for (Chunk chunk : chunks) {
+ putRow(
+ data.outputRowMeta,
+ createOutputRow(
+ row,
+ chunk.getContent(),
+ chunk.getIndex(),
+ chunk.getStartPosition(),
+ documentIdValue,
+ totalChunks));
+ }
+ }
+
+ private List<Chunk> chunk(Object[] row, String text) throws HopException {
+ if (meta.getChunkingStrategy() == ChunkingStrategyType.STRUCTURE
+ && strategy instanceof StructureChunkingStrategy structureStrategy) {
+ ContentType contentType = resolveContentType(row);
+ structureStrategy.setContentType(contentType);
+ return structureStrategy.chunk(text, data.chunkSize, data.chunkOverlap, contentType);
+ }
+ return strategy.chunk(text, data.chunkSize, data.chunkOverlap);
+ }
+
+ /**
+ * Resolves every input and output field index once, so the row loop never scans the row metadata.
+ */
+ private void resolveFieldIndices() throws HopException {
+ data.inputFieldIndex = data.inputRowMeta.indexOfValue(meta.getInputField());
+ if (data.inputFieldIndex < 0) {
+ throw new HopException(
+ BaseMessages.getString(
+ PKG, "TextChunker.Validation.InputFieldNotFound", meta.getInputField()));
+ }
+
+ data.sourceDocumentIdFieldIndex =
+ requireOptionalInputField(
+ meta.getSourceDocumentIdField(),
+ "TextChunker.Validation.SourceDocumentIdFieldNotFound");
+ data.contentTypeFieldIndex =
+ requireOptionalInputField(
+ meta.getContentTypeField(), "TextChunker.Validation.ContentTypeFieldNotFound");
+
+ data.outputChunkFieldIndex = data.outputRowMeta.indexOfValue(meta.getOutputChunkField());
+ if (meta.isIncludeMetadata()) {
+ data.chunkIndexFieldIndex = data.outputRowMeta.indexOfValue(meta.getChunkIndexField());
+ data.chunkStartPosFieldIndex = data.outputRowMeta.indexOfValue(meta.getChunkStartPosField());
+ data.documentIdFieldIndex = data.outputRowMeta.indexOfValue(meta.getDocumentIdField());
+ data.chunkCountFieldIndex = data.outputRowMeta.indexOfValue(meta.getChunkCountField());
+ }
+ }
+
+ private int requireOptionalInputField(String fieldName, String messageKey) throws HopException {
+ if (Utils.isEmpty(fieldName)) {
+ return -1;
+ }
+ int index = data.inputRowMeta.indexOfValue(fieldName);
+ if (index < 0) {
+ throw new HopException(BaseMessages.getString(PKG, messageKey, fieldName));
+ }
+ return index;
+ }
+
+ private String resolveDocumentId(Object[] row) throws HopException {
+ if (data.sourceDocumentIdFieldIndex >= 0) {
+ return data.inputRowMeta.getString(row, data.sourceDocumentIdFieldIndex);
+ }
+ return getCopy() + "_" + documentIdCounter++;
+ }
+
+ private ContentType resolveContentType(Object[] row) throws HopException {
+ if (data.contentTypeFieldIndex >= 0) {
+ String fieldValue = data.inputRowMeta.getString(row, data.contentTypeFieldIndex);
+ ContentType fromSource = ContentTypeResolver.fromSourceType(fieldValue);
+ if (fromSource != ContentType.AUTO) {
+ return fromSource;
+ }
+ ContentType parsed = ContentType.fromString(fieldValue);
+ if (parsed != ContentType.AUTO) {
+ return parsed;
+ }
+ }
+ ContentType configured = meta.getContentType();
+ return configured != null ? configured : ContentType.AUTO;
+ }
+
+ /**
+ * Creates an output row with the chunk data and metadata.
+ *
+ * @param inputRow The original input row
+ * @param chunkContent The chunk text content
+ * @param chunkIndex The 0-based index of this chunk
+ * @param chunkStartPos The start position of this chunk in the original text
+ * @param documentId The document ID for this row
+ * @param totalChunks The total number of chunks for this document
+ * @return The output row with chunk data
+ */
+ private Object[] createOutputRow(
+ Object[] inputRow,
+ String chunkContent,
+ int chunkIndex,
+ int chunkStartPos,
+ String documentId,
+ int totalChunks) {
+
+ Object[] outputRow = RowDataUtil.createResizedCopy(inputRow, data.outputRowMeta.size());
+
+ setField(outputRow, data.outputChunkFieldIndex, chunkContent);
+
+ if (meta.isIncludeMetadata()) {
+ setField(outputRow, data.chunkIndexFieldIndex, (long) chunkIndex);
+ setField(outputRow, data.chunkStartPosFieldIndex, (long) chunkStartPos);
+ setField(outputRow, data.documentIdFieldIndex, documentId);
+ setField(outputRow, data.chunkCountFieldIndex, (long) totalChunks);
+ }
+
+ return outputRow;
+ }
+
+ private static void setField(Object[] row, int index, Object value) {
+ if (index >= 0 && index < row.length) {
+ row[index] = value;
+ }
+ }
+
+ @Override
+ public void dispose() {
+ strategy = null;
+ super.dispose();
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerData.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerData.java
new file mode 100644
index 0000000000..0818d0828d
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerData.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker;
+
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.pipeline.transform.BaseTransformData;
+import org.apache.hop.pipeline.transform.ITransformData;
+
+/** Data class for the TextChunker transform. Holds runtime data that persists between rows. */
+public class TextChunkerData extends BaseTransformData implements ITransformData {
+
+ /** The input row metadata. */
+ public IRowMeta inputRowMeta;
+
+ /** The output row metadata. */
+ public IRowMeta outputRowMeta;
+
+ /** Chunk size and overlap resolved once in init, so variables are not re-resolved per row. */
+ public int chunkSize;
+
+ public int chunkOverlap;
+
+ /** The index of the input field containing text to chunk. */
+ public int inputFieldIndex = -1;
+
+ /** The index of the optional input field carrying the business document identifier. */
+ public int sourceDocumentIdFieldIndex = -1;
+
+ /** The index of the optional input field carrying the content type hint. */
+ public int contentTypeFieldIndex = -1;
+
+ /** The index of the output chunk field, resolved once against {@link #outputRowMeta}. */
+ public int outputChunkFieldIndex = -1;
+
+ /** The index of the chunk index field (if metadata is enabled). */
+ public int chunkIndexFieldIndex = -1;
+
+ /** The index of the chunk start position field (if metadata is enabled). */
+ public int chunkStartPosFieldIndex = -1;
+
+ /** The index of the original document ID field (if metadata is enabled). */
+ public int documentIdFieldIndex = -1;
+
+ /** The index of the total chunk count field (if metadata is enabled). */
+ public int chunkCountFieldIndex = -1;
+
+ public TextChunkerData() {
+ super();
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerDialog.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerDialog.java
new file mode 100644
index 0000000000..cacfc080d9
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerDialog.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker;
+
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyType;
+import org.apache.hop.ui.core.dialog.BaseDialog;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgetsAdapter;
+import org.apache.hop.ui.core.widget.ComboVar;
+import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
+import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Combo;
+import org.eclipse.swt.widgets.Control;
+import org.eclipse.swt.widgets.Shell;
+
+public class TextChunkerDialog extends BaseTransformDialog {
+
+ private static final Class<?> PKG = TextChunkerMeta.class;
+
+ private final TextChunkerMeta input;
+ private GuiCompositeWidgets widgets;
+ private boolean loading;
+
+ public TextChunkerDialog(
+ Shell parent,
+ IVariables variables,
+ TextChunkerMeta transformMeta,
+ PipelineMeta pipelineMeta) {
+ super(parent, variables, transformMeta, pipelineMeta);
+ input = transformMeta;
+ }
+
+ @Override
+ public String open() {
+ createShell(BaseMessages.getString(PKG, "TextChunkerDialog.Shell.Title"));
+ buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build();
+
+ changed = input.hasChanged();
+ loading = true;
+
+ widgets =
+ GuiCompositeWidgets.addScrolledComposite(
+ shell,
+ variables,
+ wTransformName,
+ wOk,
+ TextChunkerMeta.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ input);
+ widgets.setWidgetsListener(
+ new GuiCompositeWidgetsAdapter() {
+ @Override
+ public void widgetModified(
+ GuiCompositeWidgets compositeWidgets, Control changedWidget, String widgetId) {
+ if (!loading) {
+ input.setChanged();
+ }
+ if (TextChunkerMeta.WIDGET_CHUNKING_STRATEGY.equals(widgetId)
+ || TextChunkerMeta.WIDGET_INCLUDE_METADATA.equals(widgetId)) {
+ enableFields();
+ }
+ }
+ });
+
+ setFieldComboValues();
+ enableFields();
+ loading = false;
+ input.setChanged(changed);
+
+ focusTransformName();
+ BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel());
+ return transformName;
+ }
+
+ /**
+ * Stream field names cannot come from {@code comboValuesMethod}, which is handed only a log
+ * channel and a metadata provider, so the three field combos are filled once the widgets exist.
+ */
+ private void setFieldComboValues() {
+ try {
+ IRowMeta fields = pipelineMeta.getPrevTransformFields(variables, transformName);
+ String[] names = fields == null ? new String[0] : fields.getFieldNames();
+ setComboItems(TextChunkerMeta.WIDGET_INPUT_FIELD, names);
+ setComboItems(TextChunkerMeta.WIDGET_SOURCE_DOCUMENT_ID_FIELD, names);
+ setComboItems(TextChunkerMeta.WIDGET_CONTENT_TYPE_FIELD, names);
+ } catch (Exception e) {
+ LogChannel.UI.logError("Error getting source fields", e);
+ }
+ }
+
+ /** Fills a combo without losing the selection the transform was saved with. */
+ private void setComboItems(String widgetId, String[] names) {
+ // Setting items clears the widget's text, so put the saved selection back afterwards. This is
+ // the same dance GitInputDialog does around setComboValues.
+ String selected = comboText(widgetId, "");
+ widgets.setComboValues(widgetId, names);
+ if (!Utils.isEmpty(selected)) {
+ setComboText(widgetId, selected);
+ }
+ }
+
+ private void setComboText(String widgetId, String text) {
+ Control control = widgets.getWidgetsMap().get(widgetId);
+ if (control == null || control.isDisposed()) {
+ return;
+ }
+ if (control instanceof ComboVar comboVar) {
+ comboVar.setText(text);
+ } else if (control instanceof Combo combo) {
+ combo.setText(text);
+ }
+ }
+
+ private void enableFields() {
+ // fromString accepts the constant name and the old display text, so this follows the widget
+ // whatever it holds rather than assuming one spelling.
+ boolean structure =
+ ChunkingStrategyType.fromString(
+ comboText(
+ TextChunkerMeta.WIDGET_CHUNKING_STRATEGY, input.getChunkingStrategy().name()))
+ == ChunkingStrategyType.STRUCTURE;
+ setEnabled(TextChunkerMeta.WIDGET_CONTENT_TYPE, structure);
+ setEnabled(TextChunkerMeta.WIDGET_CONTENT_TYPE_FIELD, structure);
+
+ boolean metadata =
+ isChecked(TextChunkerMeta.WIDGET_INCLUDE_METADATA, input.isIncludeMetadata());
+ setEnabled(TextChunkerMeta.WIDGET_CHUNK_INDEX_FIELD, metadata);
+ setEnabled(TextChunkerMeta.WIDGET_CHUNK_START_POS_FIELD, metadata);
+ setEnabled(TextChunkerMeta.WIDGET_DOCUMENT_ID_FIELD, metadata);
+ setEnabled(TextChunkerMeta.WIDGET_CHUNK_COUNT_FIELD, metadata);
+ }
+
+ /**
+ * Reads a generated combo. {@code GuiCompositeWidgets} builds a plain SWT {@link Combo} when the
+ * element has no variable support and a {@link ComboVar} when it does, so both have to be handled
+ * or this silently returns the fallback.
+ */
+ private String comboText(String widgetId, String fallback) {
+ Control control = widgets.getWidgetsMap().get(widgetId);
+ if (control instanceof ComboVar comboVar && !comboVar.isDisposed()) {
+ return comboVar.getText();
+ }
+ if (control instanceof Combo combo && !combo.isDisposed()) {
+ return combo.getText();
+ }
+ return fallback;
+ }
+
+ private boolean isChecked(String widgetId, boolean fallback) {
+ Control control = widgets.getWidgetsMap().get(widgetId);
+ if (control instanceof Button button && !button.isDisposed()) {
+ return button.getSelection();
+ }
+ return fallback;
+ }
+
+ private void setEnabled(String widgetId, boolean enabled) {
+ Control label = widgets.getLabelsMap().get(widgetId);
+ if (label != null && !label.isDisposed()) {
+ label.setEnabled(enabled);
+ }
+ Control widget = widgets.getWidgetsMap().get(widgetId);
+ if (widget != null && !widget.isDisposed()) {
+ widget.setEnabled(enabled);
+ }
+ }
+
+ private void cancel() {
+ transformName = null;
+ input.setChanged(changed);
+ dispose();
+ }
+
+ private void ok() {
+ if (Utils.isEmpty(wTransformName.getText())) {
+ return;
+ }
+ widgets.getWidgetsContents(input, TextChunkerMeta.GUI_PLUGIN_ELEMENT_PARENT_ID);
+ transformName = wTransformName.getText();
+ input.setChanged();
+ dispose();
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMeta.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMeta.java
new file mode 100644
index 0000000000..1ae005da8d
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMeta.java
@@ -0,0 +1,427 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker;
+
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.CheckResult;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.ICheckResult;
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.exception.HopPluginException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.BaseTransformMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyType;
+import org.apache.hop.pipeline.transforms.chunker.document.ContentType;
+
+@Getter
+@Setter
+@Transform(
+ id = "TextChunker",
+ image = "chunker.svg",
+ name = "i18n::TextChunker.Name",
+ description = "i18n::TextChunker.Description",
+ categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Transform",
+ documentationUrl = "/pipeline/transforms/textchunker.html",
+ keywords = "i18n::TextChunker.Keywords")
+@GuiPlugin
+public class TextChunkerMeta extends BaseTransformMeta<TextChunker, TextChunkerData> {
+
+ private static final Class<?> PKG = TextChunkerMeta.class;
+
+ public static final String GUI_PLUGIN_ELEMENT_PARENT_ID = "TEXT_CHUNKER_DIALOG_OPTIONS";
+ public static final String WIDGET_INPUT_FIELD = "TEXT_CHUNKER_INPUT_FIELD";
+ public static final String WIDGET_SOURCE_DOCUMENT_ID_FIELD = "TEXT_CHUNKER_SOURCE_DOC_ID_FIELD";
+ public static final String WIDGET_CONTENT_TYPE = "TEXT_CHUNKER_CONTENT_TYPE";
+ public static final String WIDGET_CONTENT_TYPE_FIELD = "TEXT_CHUNKER_CONTENT_TYPE_FIELD";
+ public static final String WIDGET_CHUNKING_STRATEGY = "TEXT_CHUNKER_CHUNKING_STRATEGY";
+ public static final String WIDGET_INCLUDE_METADATA = "TEXT_CHUNKER_INCLUDE_METADATA";
+ public static final String WIDGET_CHUNK_INDEX_FIELD = "TEXT_CHUNKER_CHUNK_INDEX_FIELD";
+ public static final String WIDGET_CHUNK_START_POS_FIELD = "TEXT_CHUNKER_CHUNK_START_POS_FIELD";
+ public static final String WIDGET_DOCUMENT_ID_FIELD = "TEXT_CHUNKER_DOCUMENT_ID_FIELD";
+ public static final String WIDGET_CHUNK_COUNT_FIELD = "TEXT_CHUNKER_CHUNK_COUNT_FIELD";
+
+ private static final String GROUP_INPUT = "Input";
+ private static final String GROUP_CHUNKING = "Chunking";
+ private static final String GROUP_METADATA = "Chunk metadata";
+
+ /** The field containing text to chunk. */
+ @GuiWidgetElement(
+ id = WIDGET_INPUT_FIELD,
+ order = "0100",
+ type = GuiElementType.COMBO,
+ label = "i18n::TextChunker.inputField.Label",
+ toolTip = "i18n::TextChunker.inputField.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_INPUT)
+ @HopMetadataProperty(key = "inputField", injectionKey = "INPUT_FIELD")
+ private String inputField;
+
+ /** The name of the field to output chunks to. */
+ @GuiWidgetElement(
+ order = "0300",
+ type = GuiElementType.TEXT,
+ label = "i18n::TextChunker.outputChunkField.Label",
+ toolTip = "i18n::TextChunker.outputChunkField.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_INPUT)
+ @HopMetadataProperty(key = "outputChunkField", injectionKey = "OUTPUT_CHUNK_FIELD")
+ private String outputChunkField = "chunk_text";
+
+ /** The chunking strategy to use. */
+ @GuiWidgetElement(
+ id = WIDGET_CHUNKING_STRATEGY,
+ order = "0400",
+ type = GuiElementType.COMBO,
+ label = "i18n::TextChunker.chunkingStrategy.Label",
+ toolTip = "i18n::TextChunker.chunkingStrategy.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_CHUNKING)
+ @HopMetadataProperty(key = "chunkingStrategy", injectionKey = "CHUNKING_STRATEGY")
+ private ChunkingStrategyType chunkingStrategy = ChunkingStrategyType.CHARACTER;
+
+ /**
+ * The maximum size for each chunk (characters for CHARACTER strategy, approximate for PARAGRAPH).
+ */
+ @GuiWidgetElement(
+ order = "0700",
+ type = GuiElementType.TEXT,
+ label = "i18n::TextChunker.chunkSize.Label",
+ toolTip = "i18n::TextChunker.chunkSize.Tooltip",
+ variables = true,
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_CHUNKING)
+ @HopMetadataProperty(key = "chunkSize", injectionKey = "CHUNK_SIZE")
+ private String chunkSize = "1000";
+
+ /** The number of characters to overlap between chunks (for CHARACTER strategy). */
+ @GuiWidgetElement(
+ order = "0800",
+ type = GuiElementType.TEXT,
+ label = "i18n::TextChunker.chunkOverlap.Label",
+ toolTip = "i18n::TextChunker.chunkOverlap.Tooltip",
+ variables = true,
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_CHUNKING)
+ @HopMetadataProperty(key = "chunkOverlap", injectionKey = "CHUNK_OVERLAP")
+ private String chunkOverlap = "200";
+
+ /** Whether to include metadata fields in the output. */
+ @GuiWidgetElement(
+ id = WIDGET_INCLUDE_METADATA,
+ order = "0900",
+ type = GuiElementType.CHECKBOX,
+ label = "i18n::TextChunker.includeMetadata.Label",
+ toolTip = "i18n::TextChunker.includeMetadata.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_METADATA)
+ @HopMetadataProperty(key = "includeMetadata", injectionKey = "INCLUDE_METADATA")
+ private boolean includeMetadata = true;
+
+ /** Field name for the chunk index (if metadata is enabled). */
+ @GuiWidgetElement(
+ id = WIDGET_CHUNK_INDEX_FIELD,
+ order = "1000",
+ type = GuiElementType.TEXT,
+ label = "i18n::TextChunker.chunkIndexField.Label",
+ toolTip = "i18n::TextChunker.chunkIndexField.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_METADATA)
+ @HopMetadataProperty(key = "chunkIndexField", injectionKey = "CHUNK_INDEX_FIELD")
+ private String chunkIndexField = "chunk_index";
+
+ /** Field name for the chunk start position (if metadata is enabled). */
+ @GuiWidgetElement(
+ id = WIDGET_CHUNK_START_POS_FIELD,
+ order = "1100",
+ type = GuiElementType.TEXT,
+ label = "i18n::TextChunker.chunkStartPosField.Label",
+ toolTip = "i18n::TextChunker.chunkStartPosField.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_METADATA)
+ @HopMetadataProperty(key = "chunkStartPosField", injectionKey = "CHUNK_START_POS_FIELD")
+ private String chunkStartPosField = "chunk_start_position";
+
+ /**
+ * Optional input field containing the business document identifier. When empty, a row counter is
+ * used.
+ */
+ @GuiWidgetElement(
+ id = WIDGET_SOURCE_DOCUMENT_ID_FIELD,
+ order = "0200",
+ type = GuiElementType.COMBO,
+ label = "i18n::TextChunker.sourceDocumentIdField.Label",
+ toolTip = "i18n::TextChunker.sourceDocumentIdField.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_INPUT)
+ @HopMetadataProperty(key = "sourceDocumentIdField", injectionKey = "SOURCE_DOCUMENT_ID_FIELD")
+ private String sourceDocumentIdField;
+
+ /** Field name for the original document ID (if metadata is enabled). */
+ @GuiWidgetElement(
+ id = WIDGET_DOCUMENT_ID_FIELD,
+ order = "1200",
+ type = GuiElementType.TEXT,
+ label = "i18n::TextChunker.documentIdField.Label",
+ toolTip = "i18n::TextChunker.documentIdField.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_METADATA)
+ @HopMetadataProperty(key = "documentIdField", injectionKey = "DOCUMENT_ID_FIELD")
+ private String documentIdField = "chunk_doc_id";
+
+ /** Field name for the total chunk count (if metadata is enabled). */
+ @GuiWidgetElement(
+ id = WIDGET_CHUNK_COUNT_FIELD,
+ order = "1300",
+ type = GuiElementType.TEXT,
+ label = "i18n::TextChunker.chunkCountField.Label",
+ toolTip = "i18n::TextChunker.chunkCountField.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_METADATA)
+ @HopMetadataProperty(key = "chunkCountField", injectionKey = "CHUNK_COUNT_FIELD")
+ private String chunkCountField = "total_chunks";
+
+ /** Document format for STRUCTURE strategy (Auto infers from source_type field or text). */
+ @GuiWidgetElement(
+ id = WIDGET_CONTENT_TYPE,
+ order = "0500",
+ type = GuiElementType.COMBO,
+ label = "i18n::TextChunker.contentType.Label",
+ toolTip = "i18n::TextChunker.contentType.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_CHUNKING)
+ @HopMetadataProperty(key = "contentType", injectionKey = "CONTENT_TYPE")
+ private ContentType contentType = ContentType.AUTO;
+
+ /**
+ * Optional input field (e.g. {@code source_type}) used to pick a document parser for STRUCTURE
+ * strategy.
+ */
+ @GuiWidgetElement(
+ id = WIDGET_CONTENT_TYPE_FIELD,
+ order = "0600",
+ type = GuiElementType.COMBO,
+ label = "i18n::TextChunker.contentTypeField.Label",
+ toolTip = "i18n::TextChunker.contentTypeField.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_CHUNKING)
+ @HopMetadataProperty(key = "contentTypeField", injectionKey = "CONTENT_TYPE_FIELD")
+ private String contentTypeField;
+
+ /** Default constructor. */
+ public TextChunkerMeta() {
+ super();
+ }
+
+ @Override
+ public Object clone() {
+ TextChunkerMeta retval = (TextChunkerMeta) super.clone();
+ return retval;
+ }
+
+ @Override
+ public void setDefault() {
+ inputField = "";
+ outputChunkField = "chunk_text";
+ chunkingStrategy = ChunkingStrategyType.CHARACTER;
+ chunkSize = "1000";
+ chunkOverlap = "200";
+ includeMetadata = true;
+ chunkIndexField = "chunk_index";
+ chunkStartPosField = "chunk_start_position";
+ documentIdField = "chunk_doc_id";
+ chunkCountField = "total_chunks";
+ contentType = ContentType.AUTO;
+ contentTypeField = "";
+ }
+
+ @Override
+ public boolean supportsErrorHandling() {
+ return true;
+ }
+
+ @Override
+ public void getFields(
+ IRowMeta row,
+ String origin,
+ IRowMeta[] info,
+ TransformMeta nextTransform,
+ IVariables variables,
+ IHopMetadataProvider metadataProvider)
+ throws HopTransformException {
+ try {
+ addField(row, new ValueMetaString(outputChunkField), origin);
+
+ if (includeMetadata) {
+ addField(row, new ValueMetaInteger(chunkIndexField), origin);
+ addField(row, new ValueMetaInteger(chunkStartPosField), origin);
+ addField(row, new ValueMetaString(documentIdField), origin);
+ addField(row, new ValueMetaInteger(chunkCountField), origin);
+ }
+ } catch (Exception e) {
+ throw new HopTransformException("Error creating output fields", e);
+ }
+ }
+
+ private static void addField(IRowMeta row, IValueMeta field, String origin)
+ throws HopPluginException {
+ String fieldName = field.getName();
+ if (fieldName == null || fieldName.isEmpty() || row.indexOfValue(fieldName) >= 0) {
+ return;
+ }
+ field.setOrigin(origin);
+ row.addValueMeta(field);
+ }
+
+ /**
+ * Gets the output field names that will be added by this transform.
+ *
+ * @return List of output field names
+ */
+ public List<String> getOutputFieldNames() {
+ List<String> fields = new ArrayList<>();
+ fields.add(outputChunkField);
+ if (includeMetadata) {
+ fields.add(chunkIndexField);
+ fields.add(chunkStartPosField);
+ fields.add(documentIdField);
+ fields.add(chunkCountField);
+ }
+ return fields;
+ }
+
+ /**
+ * Validates the configuration.
+ *
+ * @param remarks List to add validation messages to
+ * @param pipelineMeta The pipeline metadata
+ * @param transformMeta The transform metadata
+ * @param prev The previous transform's row metadata
+ * @param input The input field names
+ * @param output The output field names
+ * @param info The info row metadata
+ * @param variables The variables
+ * @param metadataProvider The metadata provider
+ */
+ @Override
+ public void check(
+ List<ICheckResult> remarks,
+ PipelineMeta pipelineMeta,
+ TransformMeta transformMeta,
+ IRowMeta prev,
+ String[] input,
+ String[] output,
+ IRowMeta info,
+ IVariables variables,
+ IHopMetadataProvider metadataProvider) {
+
+ if (Utils.isEmpty(inputField)) {
+ error(remarks, transformMeta, "TextChunker.Validation.InputFieldRequired");
+ } else if (prev != null && prev.indexOfValue(inputField) < 0) {
+ error(remarks, transformMeta, "TextChunker.Validation.InputFieldNotFound", inputField);
+ }
+
+ if (!Utils.isEmpty(sourceDocumentIdField)
+ && prev != null
+ && prev.indexOfValue(sourceDocumentIdField) < 0) {
+ error(
+ remarks,
+ transformMeta,
+ "TextChunker.Validation.SourceDocumentIdFieldNotFound",
+ sourceDocumentIdField);
+ }
+
+ if (!Utils.isEmpty(contentTypeField)
+ && prev != null
+ && prev.indexOfValue(contentTypeField) < 0) {
+ error(
+ remarks,
+ transformMeta,
+ "TextChunker.Validation.ContentTypeFieldNotFound",
+ contentTypeField);
+ }
+
+ if (Utils.isEmpty(outputChunkField)) {
+ error(remarks, transformMeta, "TextChunker.Validation.OutputChunkFieldRequired");
+ }
+
+ int resolvedSize = Const.toInt(variables.resolve(chunkSize), -1);
+ int resolvedOverlap = Const.toInt(variables.resolve(chunkOverlap), -1);
+
+ if (resolvedSize <= 0) {
+ error(remarks, transformMeta, "TextChunker.Validation.ChunkSizePositive");
+ }
+
+ if (resolvedOverlap < 0) {
+ error(remarks, transformMeta, "TextChunker.Validation.OverlapNonNegative");
+ } else if (resolvedSize > 0 && resolvedOverlap >= resolvedSize) {
+ warning(remarks, transformMeta, "TextChunker.Validation.OverlapWarning");
+ }
+
+ if (chunkingStrategy == ChunkingStrategyType.PARAGRAPH && resolvedOverlap > 0) {
+ warning(remarks, transformMeta, "TextChunker.Validation.ParagraphOverlapIgnored");
+ }
+ }
+
+ private static void error(
+ List<ICheckResult> remarks, TransformMeta transformMeta, String key, String... parameters) {
+ remarks.add(
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_ERROR,
+ BaseMessages.getString(PKG, key, parameters),
+ transformMeta));
+ }
+
+ private static void warning(
+ List<ICheckResult> remarks, TransformMeta transformMeta, String key, String... parameters) {
+ remarks.add(
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_WARNING,
+ BaseMessages.getString(PKG, key, parameters),
+ transformMeta));
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/CharacterChunkingStrategy.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/CharacterChunkingStrategy.java
new file mode 100644
index 0000000000..7351c93fd5
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/CharacterChunkingStrategy.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.chunking;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.pipeline.transforms.chunker.Chunk;
+
+/**
+ * Chunking strategy that splits text on fixed character count while respecting word boundaries.
+ * This ensures that chunks don't split words in the middle.
+ */
+public class CharacterChunkingStrategy implements ChunkingStrategy {
+
+ @Override
+ public List<Chunk> chunk(String text, int maxSize, int overlap) {
+ List<Chunk> chunks = new ArrayList<>();
+
+ if (text == null || text.isEmpty() || maxSize <= 0) {
+ return chunks;
+ }
+
+ if (overlap < 0) {
+ chunks.add(new Chunk(text, 0, 0, text.length()));
+ return chunks;
+ }
+
+ overlap = Math.min(overlap, maxSize - 1);
+
+ int start = 0;
+ int chunkIndex = 0;
+ int textLength = text.length();
+
+ while (start < textLength) {
+ int end = Math.min(start + maxSize, textLength);
+
+ if (end == textLength) {
+ String chunkContent = text.substring(start, end);
+ chunks.add(new Chunk(chunkContent, chunkIndex++, start, end));
+ break;
+ }
+
+ int separatorPos = -1;
+ for (int i = end - 1; i >= start; i--) {
+ if (Character.isWhitespace(text.charAt(i))) {
+ separatorPos = i;
+ break;
+ }
+ }
+
+ int chunkEnd = separatorPos >= start ? separatorPos + 1 : end;
+
+ if (chunkEnd <= start) {
+ chunkEnd = Math.min(start + maxSize, textLength);
+ }
+
+ String chunkContent = text.substring(start, chunkEnd);
+ chunks.add(new Chunk(chunkContent, chunkIndex++, start, chunkEnd));
+
+ if (chunkEnd >= textLength) {
+ break;
+ }
+
+ start = overlap > 0 ? Math.max(start + 1, chunkEnd - overlap) : chunkEnd;
+ }
+
+ return chunks;
+ }
+
+ @Override
+ public ChunkingStrategyType getType() {
+ return ChunkingStrategyType.CHARACTER;
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ChunkingStrategy.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ChunkingStrategy.java
new file mode 100644
index 0000000000..5bc8ff1c05
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ChunkingStrategy.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.chunking;
+
+import java.util.List;
+import org.apache.hop.pipeline.transforms.chunker.Chunk;
+
+/**
+ * Interface for text chunking strategies. Implementations define how text is divided into chunks.
+ */
+public interface ChunkingStrategy {
+
+ /**
+ * Chunks the given text according to the strategy's rules.
+ *
+ * @param text The text to chunk
+ * @param maxSize The maximum size for each chunk (interpretation depends on strategy)
+ * @param overlap The number of characters/units to overlap between chunks
+ * @return A list of chunks
+ */
+ List<Chunk> chunk(String text, int maxSize, int overlap);
+
+ /**
+ * Gets the type of this chunking strategy.
+ *
+ * @return The strategy type
+ */
+ ChunkingStrategyType getType();
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ChunkingStrategyFactory.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ChunkingStrategyFactory.java
new file mode 100644
index 0000000000..d6d022907e
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ChunkingStrategyFactory.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.chunking;
+
+/** Factory for creating chunking strategy instances based on strategy type. */
+public final class ChunkingStrategyFactory {
+
+ private ChunkingStrategyFactory() {
+ // Utility class - prevent instantiation
+ }
+
+ /**
+ * Creates a chunking strategy based on the given type.
+ *
+ * @param type The strategy type
+ * @return The corresponding chunking strategy
+ */
+ public static ChunkingStrategy createStrategy(ChunkingStrategyType type) {
+ if (type == null) {
+ return createDefaultStrategy();
+ }
+
+ switch (type) {
+ case CHARACTER:
+ return new CharacterChunkingStrategy();
+ case PARAGRAPH:
+ return new ParagraphChunkingStrategy();
+ case STRUCTURE:
+ return new StructureChunkingStrategy();
+ default:
+ return createDefaultStrategy();
+ }
+ }
+
+ /**
+ * Creates a chunking strategy based on the given type name.
+ *
+ * @param typeName The strategy type name
+ * @return The corresponding chunking strategy
+ */
+ public static ChunkingStrategy createStrategy(String typeName) {
+ return createStrategy(ChunkingStrategyType.fromString(typeName));
+ }
+
+ /**
+ * Creates the default chunking strategy (Character).
+ *
+ * @return The default strategy
+ */
+ public static ChunkingStrategy createDefaultStrategy() {
+ return new CharacterChunkingStrategy();
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ChunkingStrategyType.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ChunkingStrategyType.java
new file mode 100644
index 0000000000..ff5eb287cd
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ChunkingStrategyType.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.chunking;
+
+/** Enumeration of available chunking strategy types. */
+public enum ChunkingStrategyType {
+ /** Split text on fixed character count, respecting word boundaries. */
+ CHARACTER("Character"),
+
+ /** Split text on paragraph boundaries (double newlines). */
+ PARAGRAPH("Paragraph"),
+
+ /**
+ * Split on document structure (headings) with breadcrumb prefixes; character fallback per
+ * section.
+ */
+ STRUCTURE("Structure");
+
+ /**
+ * Display text kept only so {@link #fromString(String)} still accepts values written before the
+ * constant names were used. It is deliberately NOT exposed through {@code toString()}: the
+ * generated dialogs fill an enum combo with {@code toString()} and read it back with {@code
+ * Enum.valueOf}, so anything other than the constant name makes the combo unreadable and a
+ * changed selection is silently dropped.
+ */
+ private final String description;
+
+ ChunkingStrategyType(String description) {
+ this.description = description;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ /**
+ * Gets the strategy type from its string representation.
+ *
+ * @param type The string type
+ * @return The corresponding strategy type
+ */
+ public static ChunkingStrategyType fromString(String type) {
+ if (type == null || type.isEmpty()) {
+ return CHARACTER;
+ }
+ for (ChunkingStrategyType strategyType : values()) {
+ if (strategyType.name().equalsIgnoreCase(type)
+ || strategyType.getDescription().equalsIgnoreCase(type)) {
+ return strategyType;
+ }
+ }
+ return CHARACTER;
+ }
+
+ /**
+ * Gets all available strategy types as an array.
+ *
+ * @return Array of all strategy types
+ */
+ public static ChunkingStrategyType[] getAll() {
+ return values();
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ParagraphChunkingStrategy.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ParagraphChunkingStrategy.java
new file mode 100644
index 0000000000..e32d191659
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/ParagraphChunkingStrategy.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.chunking;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.hop.pipeline.transforms.chunker.Chunk;
+
+/**
+ * Chunking strategy that splits text on paragraph boundaries.
+ *
+ * <p>Paragraphs are identified by blank lines. A paragraph that on its own exceeds {@code maxSize}
+ * is split further with {@link CharacterChunkingStrategy}, because a chunk larger than the limit
+ * would be rejected downstream by the embedding model rather than simply being large.
+ *
+ * <p>Consecutive paragraphs are packed into one chunk while they fit within {@code maxSize}, which
+ * is the usual contract for an embedding chunker: it keeps related text together and avoids
+ * emitting many small chunks that each carry little context.
+ */
+public class ParagraphChunkingStrategy implements ChunkingStrategy {
+
+ /**
+ * A blank line. {@code \\R} matches any line ending, so positions stay valid in the original text
+ * and no CRLF normalisation pass is needed.
+ */
+ private static final Pattern PARAGRAPH_SEPARATOR = Pattern.compile("\\R\\s*\\R");
+
+ private final CharacterChunkingStrategy fallback = new CharacterChunkingStrategy();
+
+ @Override
+ public List<Chunk> chunk(String text, int maxSize, int overlap) {
+ List<Chunk> chunks = new ArrayList<>();
+
+ if (text == null || text.isEmpty()) {
+ return chunks;
+ }
+ if (maxSize <= 0) {
+ chunks.add(new Chunk(text, 0, 0, text.length()));
+ return chunks;
+ }
+
+ List<Paragraph> paragraphs = splitParagraphs(text);
+ if (paragraphs.isEmpty()) {
+ chunks.add(new Chunk(text, 0, 0, text.length()));
+ return chunks;
+ }
+
+ Paragraph first = null;
+ Paragraph last = null;
+ int chunkIndex = 0;
+
+ for (Paragraph paragraph : paragraphs) {
+ if (paragraph.length() > maxSize) {
+ // Emit whatever is buffered, then split the oversized paragraph on characters.
+ if (first != null) {
+ chunks.add(slice(text, first, last, chunkIndex++));
+ first = null;
+ last = null;
+ }
+ for (Chunk part : fallback.chunk(paragraph.text, maxSize, overlap)) {
+ chunks.add(
+ new Chunk(
+ part.getContent(),
+ chunkIndex++,
+ paragraph.start + part.getStartPosition(),
+ paragraph.start + part.getEndPosition()));
+ }
+ continue;
+ }
+
+ if (first == null) {
+ first = paragraph;
+ last = paragraph;
+ } else if (paragraph.end() - first.start() <= maxSize) {
+ last = paragraph;
+ } else {
+ chunks.add(slice(text, first, last, chunkIndex++));
+ first = paragraph;
+ last = paragraph;
+ }
+ }
+
+ if (first != null) {
+ chunks.add(slice(text, first, last, chunkIndex));
+ }
+
+ return chunks;
+ }
+
+ /**
+ * A packed chunk is the source text between the first and last paragraph it covers. Taking the
+ * slice rather than rejoining with a fixed separator keeps the reported start and end positions
+ * addressing the original text, whatever whitespace separated the paragraphs there.
+ */
+ private static Chunk slice(String text, Paragraph first, Paragraph last, int index) {
+ return new Chunk(text.substring(first.start(), last.end()), index, first.start(), last.end());
+ }
+
+ /**
+ * Splits on blank lines in a single pass, keeping each paragraph's offset in the original text.
+ */
+ private static List<Paragraph> splitParagraphs(String text) {
+ List<Paragraph> paragraphs = new ArrayList<>();
+ Matcher matcher = PARAGRAPH_SEPARATOR.matcher(text);
+ int start = 0;
+ while (matcher.find()) {
+ addParagraph(paragraphs, text, start, matcher.start());
+ start = matcher.end();
+ }
+ addParagraph(paragraphs, text, start, text.length());
+ return paragraphs;
+ }
+
+ private static void addParagraph(List<Paragraph> paragraphs, String text, int start, int end) {
+ if (end > start) {
+ paragraphs.add(new Paragraph(text.substring(start, end), start));
+ }
+ }
+
+ private record Paragraph(String text, int start) {
+ int end() {
+ return start + text.length();
+ }
+
+ int length() {
+ return text.length();
+ }
+ }
+
+ @Override
+ public ChunkingStrategyType getType() {
+ return ChunkingStrategyType.PARAGRAPH;
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/StructureChunkingStrategy.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/StructureChunkingStrategy.java
new file mode 100644
index 0000000000..7d4901314a
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/chunking/StructureChunkingStrategy.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.chunking;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.pipeline.transforms.chunker.Chunk;
+import org.apache.hop.pipeline.transforms.chunker.document.ContentType;
+import org.apache.hop.pipeline.transforms.chunker.document.DocumentNode;
+import org.apache.hop.pipeline.transforms.chunker.document.DocumentParser;
+import org.apache.hop.pipeline.transforms.chunker.document.DocumentParserRegistry;
+
+/**
+ * Structure-aware chunking: parse document into sections, prefix each chunk with a breadcrumb path,
+ * and fall back to {@link CharacterChunkingStrategy} when a section exceeds the size limit.
+ */
+public class StructureChunkingStrategy implements ChunkingStrategy {
+
+ private final CharacterChunkingStrategy fallback = new CharacterChunkingStrategy();
+ private ContentType contentType = ContentType.AUTO;
+
+ public void setContentType(ContentType contentType) {
+ this.contentType = contentType != null ? contentType : ContentType.AUTO;
+ }
+
+ @Override
+ public List<Chunk> chunk(String text, int maxSize, int overlap) {
+ return chunk(text, maxSize, overlap, contentType);
+ }
+
+ public List<Chunk> chunk(String text, int maxSize, int overlap, ContentType type) {
+ if (text == null || text.isEmpty() || maxSize <= 0) {
+ return List.of();
+ }
+
+ DocumentParser parser = DocumentParserRegistry.parserFor(type, text);
+ DocumentNode root = parser.parse(text);
+ List<DocumentNode.DocumentSection> sections = root.flattenSections();
+
+ if (sections.isEmpty()) {
+ return fallback.chunk(text, maxSize, overlap);
+ }
+
+ List<Chunk> chunks = new ArrayList<>();
+ int chunkIndex = 0;
+
+ for (DocumentNode.DocumentSection section : sections) {
+ String sectionText = formatSection(section);
+ if (sectionText.isEmpty()) {
+ continue;
+ }
+
+ if (sectionText.length() <= maxSize) {
+ chunks.add(
+ new Chunk(
+ sectionText,
+ chunkIndex++,
+ section.getStartPosition(),
+ section.getStartPosition() + sectionText.length()));
+ } else {
+ String breadcrumb = section.breadcrumb();
+ String prefix = breadcrumb.isEmpty() ? "" : breadcrumb + "\n\n";
+ String body = section.getBody();
+ int bodyMax = prefix.isEmpty() ? maxSize : Math.max(1, maxSize - prefix.length());
+ List<Chunk> split = fallback.chunk(body, bodyMax, overlap);
+ for (Chunk part : split) {
+ String content = prefix.isEmpty() ? part.getContent() : prefix + part.getContent();
+ chunks.add(
+ new Chunk(
+ content,
+ chunkIndex++,
+ section.getStartPosition() + part.getStartPosition(),
+ section.getStartPosition() + part.getEndPosition()));
+ }
+ }
+ }
+
+ if (chunks.isEmpty()) {
+ return fallback.chunk(text, maxSize, overlap);
+ }
+
+ return chunks;
+ }
+
+ private static String formatSection(DocumentNode.DocumentSection section) {
+ String breadcrumb = section.breadcrumb();
+ String body = section.getBody();
+ if (breadcrumb.isEmpty()) {
+ return body;
+ }
+ if (body.isEmpty()) {
+ return breadcrumb;
+ }
+ return breadcrumb + "\n\n" + body;
+ }
+
+ @Override
+ public ChunkingStrategyType getType() {
+ return ChunkingStrategyType.STRUCTURE;
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/AsciiDocDocumentParser.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/AsciiDocDocumentParser.java
new file mode 100644
index 0000000000..72c097d832
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/AsciiDocDocumentParser.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+import java.util.regex.Pattern;
+
+/** Splits AsciiDoc on {@code =} headings and builds a section tree. */
+public final class AsciiDocDocumentParser implements DocumentParser {
+
+ private static final Pattern HEADING = Pattern.compile("(?m)^(=+)\\s+(.+)$");
+
+ @Override
+ public ContentType getContentType() {
+ return ContentType.ASCIIDOC;
+ }
+
+ @Override
+ public DocumentNode parse(String text) {
+ return HeadingParserSupport.parse(text, HEADING, String::length);
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/ContentType.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/ContentType.java
new file mode 100644
index 0000000000..bea743783a
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/ContentType.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+/** Document format hint for structure-aware chunking. */
+public enum ContentType {
+ /** Detect from text heuristics or {@link ContentTypeResolver#fromSourceType(String)}. */
+ AUTO("Auto"),
+
+ /** Unstructured plain text (paragraph fallback only). */
+ PLAIN("Plain"),
+
+ /** Markdown with {@code #} headings. */
+ MARKDOWN("Markdown"),
+
+ /** AsciiDoc with {@code =} headings. */
+ ASCIIDOC("AsciiDoc"),
+
+ /** Hop pipeline (.hpl) XML. */
+ PIPELINE("Pipeline"),
+
+ /** Hop workflow (.hwf) XML. */
+ WORKFLOW("Workflow"),
+
+ /** Hop project metadata JSON (connections, run configs, …). */
+ METADATA("Metadata");
+
+ private final String description;
+
+ ContentType(String description) {
+ this.description = description;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public static ContentType fromString(String value) {
+ if (value == null || value.isBlank()) {
+ return AUTO;
+ }
+ for (ContentType type : values()) {
+ if (type.name().equalsIgnoreCase(value) || type.description.equalsIgnoreCase(value)) {
+ return type;
+ }
+ }
+ return AUTO;
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/ContentTypeResolver.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/ContentTypeResolver.java
new file mode 100644
index 0000000000..c742e6baf1
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/ContentTypeResolver.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+/** Maps RAG corpus {@code source_type} values to document parsers. */
+public final class ContentTypeResolver {
+
+ private ContentTypeResolver() {}
+
+ /**
+ * Map hop-rag-corpus {@code source_type} to a content type. Returns {@link ContentType#AUTO} when
+ * unknown.
+ */
+ public static ContentType fromSourceType(String sourceType) {
+ if (sourceType == null || sourceType.isBlank()) {
+ return ContentType.AUTO;
+ }
+ return switch (sourceType.trim().toLowerCase()) {
+ case "doc" -> ContentType.ASCIIDOC;
+ case "pipeline" -> ContentType.PIPELINE;
+ case "workflow" -> ContentType.WORKFLOW;
+ case "metadata" -> ContentType.METADATA;
+ case "article", "blog" -> ContentType.MARKDOWN;
+ case "plugin" -> ContentType.PLAIN;
+ default -> ContentType.AUTO;
+ };
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentNode.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentNode.java
new file mode 100644
index 0000000000..d1d826115d
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentNode.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Tree node produced by {@link DocumentParser} implementations. Each node may carry a section
+ * title, body text, and nested child sections.
+ */
+public final class DocumentNode {
+
+ private final String title;
+ private final String body;
+ private final int startPosition;
+ private final List<DocumentNode> children;
+
+ public DocumentNode(String title, String body, int startPosition, List<DocumentNode> children) {
+ this.title = title != null ? title : "";
+ this.body = body != null ? body : "";
+ this.startPosition = Math.max(0, startPosition);
+ this.children = children != null ? List.copyOf(children) : List.of();
+ }
+
+ public static DocumentNode leaf(String title, String body, int startPosition) {
+ return new DocumentNode(title, body, startPosition, List.of());
+ }
+
+ public static DocumentNode root(String body) {
+ return new DocumentNode("", body, 0, List.of());
+ }
+
+ public String getTitle() {
+ return title;
+ }
+
+ public String getBody() {
+ return body;
+ }
+
+ public int getStartPosition() {
+ return startPosition;
+ }
+
+ public List<DocumentNode> getChildren() {
+ return children;
+ }
+
+ /** Depth-first flattening into sections with breadcrumb paths (root title omitted when empty). */
+ public List<DocumentSection> flattenSections() {
+ List<DocumentSection> sections = new ArrayList<>();
+ flattenSections(new ArrayList<>(), sections);
+ return sections;
+ }
+
+ private void flattenSections(List<String> path, List<DocumentSection> sections) {
+ List<String> currentPath = path;
+ if (!title.isEmpty()) {
+ currentPath = new ArrayList<>(path);
+ currentPath.add(title);
+ }
+
+ if (!body.isBlank()) {
+ sections.add(new DocumentSection(List.copyOf(currentPath), body.strip(), startPosition));
+ }
+
+ for (DocumentNode child : children) {
+ child.flattenSections(currentPath, sections);
+ }
+ }
+
+ /** A document section with breadcrumb path and body text. */
+ public static final class DocumentSection {
+ private final List<String> path;
+ private final String body;
+ private final int startPosition;
+
+ public DocumentSection(List<String> path, String body, int startPosition) {
+ this.path = path != null ? List.copyOf(path) : List.of();
+ this.body = body != null ? body : "";
+ this.startPosition = startPosition;
+ }
+
+ public List<String> getPath() {
+ return path;
+ }
+
+ public String getBody() {
+ return body;
+ }
+
+ public int getStartPosition() {
+ return startPosition;
+ }
+
+ public String breadcrumb() {
+ return String.join(" > ", path);
+ }
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParser.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParser.java
new file mode 100644
index 0000000000..d211ec00ac
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParser.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+/** Parses a document string into a {@link DocumentNode} tree for structure-aware chunking. */
+public interface DocumentParser {
+
+ ContentType getContentType();
+
+ DocumentNode parse(String text);
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistry.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistry.java
new file mode 100644
index 0000000000..318ac8e427
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistry.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.regex.Pattern;
+import org.apache.hop.pipeline.transforms.chunker.document.hopxml.HopPipelineXmlParser;
+import org.apache.hop.pipeline.transforms.chunker.document.hopxml.HopWorkflowXmlParser;
+import org.apache.hop.pipeline.transforms.chunker.document.hopxml.HopXmlSupport;
+import org.apache.hop.pipeline.transforms.chunker.document.metadata.HopMetadataJsonParser;
+
+/** Resolves {@link DocumentParser} instances and auto-detects content type from text. */
+public final class DocumentParserRegistry {
+
+ private static final Map<ContentType, DocumentParser> PARSERS = new EnumMap<>(ContentType.class);
+
+ /** Must stay in sync with {@code AsciiDocDocumentParser.HEADING}. */
+ private static final Pattern ASCIIDOC_HEADING = Pattern.compile("(?m)^=+[ \\t]+\\S");
+
+ /** Must stay in sync with {@code MarkdownDocumentParser.HEADING}. */
+ private static final Pattern MARKDOWN_HEADING = Pattern.compile("(?m)^#{1,6}[ \\t]+\\S");
+
+ static {
+ PARSERS.put(ContentType.PLAIN, new PlainTextDocumentParser());
+ PARSERS.put(ContentType.MARKDOWN, new MarkdownDocumentParser());
+ PARSERS.put(ContentType.ASCIIDOC, new AsciiDocDocumentParser());
+ PARSERS.put(ContentType.PIPELINE, new HopPipelineXmlParser());
+ PARSERS.put(ContentType.WORKFLOW, new HopWorkflowXmlParser());
+ PARSERS.put(ContentType.METADATA, new HopMetadataJsonParser());
+ }
+
+ private DocumentParserRegistry() {}
+
+ public static DocumentParser parserFor(ContentType contentType, String text) {
+ ContentType resolved =
+ contentType == null || contentType == ContentType.AUTO ? detect(text) : contentType;
+ return PARSERS.getOrDefault(resolved, PARSERS.get(ContentType.PLAIN));
+ }
+
+ public static ContentType detect(String text) {
+ if (text == null || text.isBlank()) {
+ return ContentType.PLAIN;
+ }
+ String trimmed = text.stripLeading();
+ if (trimmed.startsWith("{") || text.contains("Configuration:\n{")) {
+ return ContentType.METADATA;
+ }
+ if (HopXmlSupport.looksLikeHopXml(text)) {
+ if (trimmed.contains("<workflow")) {
+ return ContentType.WORKFLOW;
+ }
+ if (trimmed.contains("<pipeline")) {
+ return ContentType.PIPELINE;
+ }
+ }
+ // find() rather than matches(): a heading anywhere in the document counts, including one on
+ // the very first line, which is the common case for a single-page document.
+ if (ASCIIDOC_HEADING.matcher(text).find()) {
+ return ContentType.ASCIIDOC;
+ }
+ if (MARKDOWN_HEADING.matcher(text).find()) {
+ return ContentType.MARKDOWN;
+ }
+ return ContentType.PLAIN;
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/HeadingParserSupport.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/HeadingParserSupport.java
new file mode 100644
index 0000000000..226af2d584
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/HeadingParserSupport.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.ToIntFunction;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/** Builds a {@link DocumentNode} tree from heading markers in Markdown or AsciiDoc. */
+final class HeadingParserSupport {
+
+ private HeadingParserSupport() {}
+
+ static DocumentNode parse(String text, Pattern headingPattern, ToIntFunction<String> levelFn) {
+ if (text == null || text.isEmpty()) {
+ return DocumentNode.root("");
+ }
+
+ Matcher matcher = headingPattern.matcher(text);
+ List<HeadingMatch> headings = new ArrayList<>();
+ while (matcher.find()) {
+ headings.add(
+ new HeadingMatch(
+ levelFn.applyAsInt(matcher.group(1)),
+ matcher.group(2).trim(),
+ matcher.start(),
+ matcher.end()));
+ }
+
+ if (headings.isEmpty()) {
+ return DocumentNode.root(text);
+ }
+
+ String preamble = text.substring(0, headings.get(0).start).strip();
+ MutableNode root = new MutableNode("", preamble, 0);
+ MutableNode[] stack = new MutableNode[7];
+ stack[0] = root;
+
+ for (int i = 0; i < headings.size(); i++) {
+ HeadingMatch heading = headings.get(i);
+ int bodyStart = heading.end;
+ int bodyEnd = (i + 1 < headings.size()) ? headings.get(i + 1).start : text.length();
+ String body = text.substring(bodyStart, bodyEnd).strip();
+
+ int level = Math.min(Math.max(heading.level, 1), stack.length - 1);
+ MutableNode node = new MutableNode(heading.title, body, heading.start);
+ stack[level] = node;
+ for (int j = level + 1; j < stack.length; j++) {
+ stack[j] = null;
+ }
+ MutableNode parent = stack[level - 1];
+ if (parent == null) {
+ parent = stack[0];
+ }
+ parent.children.add(node);
+ stack[level] = node;
+ }
+
+ return root.toImmutable();
+ }
+
+ private static final class HeadingMatch {
+ private final int level;
+ private final String title;
+ private final int start;
+ private final int end;
+
+ private HeadingMatch(int level, String title, int start, int end) {
+ this.level = level;
+ this.title = title;
+ this.start = start;
+ this.end = end;
+ }
+ }
+
+ private static final class MutableNode {
+ private final String title;
+ private final String body;
+ private final int startPosition;
+ private final List<MutableNode> children = new ArrayList<>();
+
+ private MutableNode(String title, String body, int startPosition) {
+ this.title = title;
+ this.body = body;
+ this.startPosition = startPosition;
+ }
+
+ private DocumentNode toImmutable() {
+ List<DocumentNode> childNodes = new ArrayList<>();
+ for (MutableNode child : children) {
+ childNodes.add(child.toImmutable());
+ }
+ return new DocumentNode(title, body, startPosition, childNodes);
+ }
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/MarkdownDocumentParser.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/MarkdownDocumentParser.java
new file mode 100644
index 0000000000..e3adfea30e
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/MarkdownDocumentParser.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+import java.util.regex.Pattern;
+
+/** Splits Markdown on {@code #} headings and builds a section tree. */
+public final class MarkdownDocumentParser implements DocumentParser {
+
+ private static final Pattern HEADING = Pattern.compile("(?m)^(#{1,6})\\s+(.+)$");
+
+ @Override
+ public ContentType getContentType() {
+ return ContentType.MARKDOWN;
+ }
+
+ @Override
+ public DocumentNode parse(String text) {
+ return HeadingParserSupport.parse(text, HEADING, marker -> marker.length());
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/PlainTextDocumentParser.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/PlainTextDocumentParser.java
new file mode 100644
index 0000000000..85a9d9aadd
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/PlainTextDocumentParser.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+/** Plain text — single root section with the full document body. */
+public final class PlainTextDocumentParser implements DocumentParser {
+
+ @Override
+ public ContentType getContentType() {
+ return ContentType.PLAIN;
+ }
+
+ @Override
+ public DocumentNode parse(String text) {
+ return DocumentNode.root(text != null ? text : "");
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/SecretRedaction.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/SecretRedaction.java
new file mode 100644
index 0000000000..d86e656c36
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/SecretRedaction.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * Keeps credentials out of chunk text.
+ *
+ * <p>Chunks produced from Hop definitions are normally sent to an embedding model, so a secret
+ * copied out of a {@code .hpl}, {@code .hwf} or metadata file would leave the installation.
+ */
+public final class SecretRedaction {
+
+ public static final String REDACTED = "**redacted**";
+
+ private static final List<String> SECRET_MARKERS =
+ List.of(
+ "password",
+ "passwd",
+ "pwd",
+ "secret",
+ "token",
+ "credential",
+ "passphrase",
+ "privatekey",
+ "apikey");
+
+ private SecretRedaction() {}
+
+ /**
+ * True when a field or tag name reads like a credential. Names are normalised, so {@code
+ * client_secret}, {@code API-KEY} and {@code privateKey} all match. A bare {@code key} does not:
+ * it is an ordinary field name in lookups and joins, and redacting it would gut the chunk.
+ */
+ public static boolean isSecretName(String name) {
+ if (name == null) {
+ return false;
+ }
+ String normalised = name.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]", "");
+ return SECRET_MARKERS.stream().anyMatch(normalised::contains);
+ }
+
+ /** True for Hop's own encrypted form, whatever the field is called. */
+ public static boolean isEncryptedValue(String value) {
+ return value != null && value.stripLeading().startsWith("Encrypted ");
+ }
+
+ /** The value to emit for a field, redacted when the name or the value gives it away. */
+ public static String redact(String name, String value) {
+ if (value == null || value.isEmpty()) {
+ return value;
+ }
+ return isSecretName(name) || isEncryptedValue(value) ? REDACTED : value;
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopPipelineXmlParser.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopPipelineXmlParser.java
new file mode 100644
index 0000000000..a1f1db0bae
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopPipelineXmlParser.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document.hopxml;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.pipeline.transforms.chunker.document.ContentType;
+import org.apache.hop.pipeline.transforms.chunker.document.DocumentNode;
+import org.apache.hop.pipeline.transforms.chunker.document.DocumentParser;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+/**
+ * Parses Hop pipeline (.hpl) XML into one section per overview, notepad block, transform, and hop
+ * graph.
+ */
+public final class HopPipelineXmlParser implements DocumentParser {
+
+ @Override
+ public ContentType getContentType() {
+ return ContentType.PIPELINE;
+ }
+
+ @Override
+ public DocumentNode parse(String text) {
+ if (!HopXmlSupport.looksLikeHopXml(text)) {
+ return DocumentNode.root(text != null ? text : "");
+ }
+
+ Document document = HopXmlSupport.parseDocument(text);
+ if (document == null) {
+ return DocumentNode.root(text);
+ }
+ Element root = document.getDocumentElement();
+ if (root == null || !"pipeline".equals(root.getTagName())) {
+ return DocumentNode.root(text);
+ }
+
+ Element info = HopXmlSupport.firstChild(root, "info");
+ String pipelineName = info != null ? HopXmlSupport.directChildText(info, "name") : "";
+ List<DocumentNode> children = new ArrayList<>();
+
+ StringBuilder overview = new StringBuilder();
+ if (!pipelineName.isEmpty()) {
+ overview.append("Hop pipeline '").append(pipelineName).append("'\n");
+ }
+ if (info != null) {
+ appendIfPresent(overview, "Description", HopXmlSupport.directChildText(info, "description"));
+ appendIfPresent(
+ overview,
+ "Extended description",
+ HopXmlSupport.directChildText(info, "extended_description"));
+ String params = HopXmlSupport.formatParameters(HopXmlSupport.firstChild(info, "parameters"));
+ appendIfPresent(overview, "Parameters", params);
+ }
+ if (overview.length() > 0) {
+ children.add(DocumentNode.leaf("Overview", overview.toString().strip(), 0));
+ }
+
+ String notes = HopXmlSupport.formatNotepads(root, "notepads");
+ if (!notes.isEmpty()) {
+ children.add(DocumentNode.leaf("Notes", notes, 0));
+ }
+
+ for (Element transform : HopXmlSupport.children(root, "transform")) {
+ String name = HopXmlSupport.directChildText(transform, "name");
+ String type = HopXmlSupport.directChildText(transform, "type");
+ String title = "Transform: " + name + " (" + type + ")";
+ String body = HopXmlSupport.serializeConfig(transform);
+ if (!body.isEmpty()) {
+ children.add(DocumentNode.leaf(title, body, 0));
+ }
+ }
+
+ Element order = HopXmlSupport.firstChild(root, "order");
+ if (order != null) {
+ StringBuilder hops = new StringBuilder();
+ for (Element hop : HopXmlSupport.children(order, "hop")) {
+ String from = HopXmlSupport.directChildText(hop, "from");
+ String to = HopXmlSupport.directChildText(hop, "to");
+ String enabled = HopXmlSupport.directChildText(hop, "enabled");
+ if (!from.isEmpty() && !to.isEmpty() && !"N".equalsIgnoreCase(enabled)) {
+ hops.append("- ").append(from).append(" → ").append(to).append('\n');
+ }
+ }
+ if (hops.length() > 0) {
+ children.add(DocumentNode.leaf("Data flow", hops.toString().strip(), 0));
+ }
+ }
+
+ if (children.isEmpty()) {
+ return DocumentNode.root(text);
+ }
+ return new DocumentNode(pipelineName, "", 0, children);
+ }
+
+ private static void appendIfPresent(StringBuilder sb, String label, String value) {
+ if (value != null && !value.isBlank()) {
+ if (sb.length() > 0) {
+ sb.append('\n');
+ }
+ sb.append(label).append(":\n").append(value.strip());
+ }
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopWorkflowXmlParser.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopWorkflowXmlParser.java
new file mode 100644
index 0000000000..66df27ddca
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopWorkflowXmlParser.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document.hopxml;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.pipeline.transforms.chunker.document.ContentType;
+import org.apache.hop.pipeline.transforms.chunker.document.DocumentNode;
+import org.apache.hop.pipeline.transforms.chunker.document.DocumentParser;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+/** Parses Hop workflow (.hwf) XML into one section per overview, action, and hop graph. */
+public final class HopWorkflowXmlParser implements DocumentParser {
+
+ @Override
+ public ContentType getContentType() {
+ return ContentType.WORKFLOW;
+ }
+
+ @Override
+ public DocumentNode parse(String text) {
+ if (!HopXmlSupport.looksLikeHopXml(text)) {
+ return DocumentNode.root(text != null ? text : "");
+ }
+
+ Document document = HopXmlSupport.parseDocument(text);
+ if (document == null) {
+ return DocumentNode.root(text);
+ }
+ Element root = document.getDocumentElement();
+ if (root == null || !"workflow".equals(root.getTagName())) {
+ return DocumentNode.root(text);
+ }
+
+ String workflowName = HopXmlSupport.directChildText(root, "name");
+ List<DocumentNode> children = new ArrayList<>();
+
+ StringBuilder overview = new StringBuilder();
+ if (!workflowName.isEmpty()) {
+ overview.append("Hop workflow '").append(workflowName).append("'\n");
+ }
+ appendIfPresent(overview, "Description", HopXmlSupport.directChildText(root, "description"));
+ appendIfPresent(
+ overview,
+ "Extended description",
+ HopXmlSupport.directChildText(root, "extended_description"));
+ String params = HopXmlSupport.formatParameters(HopXmlSupport.firstChild(root, "parameters"));
+ appendIfPresent(overview, "Parameters", params);
+
+ if (overview.length() > 0) {
+ children.add(DocumentNode.leaf("Overview", overview.toString().strip(), 0));
+ }
+
+ String notes = HopXmlSupport.formatNotepads(root, "notepads");
+ if (!notes.isEmpty()) {
+ children.add(DocumentNode.leaf("Notes", notes, 0));
+ }
+
+ Element actions = HopXmlSupport.firstChild(root, "actions");
+ if (actions != null) {
+ for (Element action : HopXmlSupport.children(actions, "action")) {
+ String name = HopXmlSupport.directChildText(action, "name");
+ String type = HopXmlSupport.directChildText(action, "type");
+ String title = "Action: " + name + " (" + type + ")";
+ String body = HopXmlSupport.serializeConfig(action);
+ if (!body.isEmpty()) {
+ children.add(DocumentNode.leaf(title, body, 0));
+ }
+ }
+ }
+
+ Element hopsParent = HopXmlSupport.firstChild(root, "hops");
+ if (hopsParent != null) {
+ StringBuilder hops = new StringBuilder();
+ for (Element hop : HopXmlSupport.children(hopsParent, "hop")) {
+ String from = HopXmlSupport.directChildText(hop, "from");
+ String to = HopXmlSupport.directChildText(hop, "to");
+ String enabled = HopXmlSupport.directChildText(hop, "enabled");
+ if (!from.isEmpty() && !to.isEmpty() && !"N".equalsIgnoreCase(enabled)) {
+ hops.append("- ").append(from).append(" → ").append(to).append('\n');
+ }
+ }
+ if (hops.length() > 0) {
+ children.add(DocumentNode.leaf("Control flow", hops.toString().strip(), 0));
+ }
+ }
+
+ if (children.isEmpty()) {
+ return DocumentNode.root(text);
+ }
+ return new DocumentNode(workflowName, "", 0, children);
+ }
+
+ private static void appendIfPresent(StringBuilder sb, String label, String value) {
+ if (value != null && !value.isBlank()) {
+ if (sb.length() > 0) {
+ sb.append('\n');
+ }
+ sb.append(label).append(":\n").append(value.strip());
+ }
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopXmlSupport.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopXmlSupport.java
new file mode 100644
index 0000000000..fc05aa6905
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopXmlSupport.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document.hopxml;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import org.apache.hop.pipeline.transforms.chunker.document.SecretRedaction;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+/** DOM helpers for Hop .hpl / .hwf XML. */
+public final class HopXmlSupport {
+
+ static final Set<String> SKIP_TAGS =
+ Set.of(
+ "GUI",
+ "partitioning",
+ "attributes",
+ "attributes_hac",
+ "custom_distribution",
+ "transform_error_handling",
+ "key_for_session_key",
+ "is_key_private",
+ "created_user",
+ "created_date",
+ "modified_user",
+ "modified_date",
+ "pipeline_version",
+ "workflow_version",
+ "capture_transform_performance",
+ "transform_performance_capturing_delay",
+ "transform_performance_capturing_size_limit",
+ "name_sync_with_filename",
+ "pipeline_status",
+ "xloc",
+ "yloc",
+ "width",
+ "heigth",
+ "fontname",
+ "fontsize",
+ "fontbold",
+ "fontitalic",
+ "fontcolorred",
+ "fontcolorgreen",
+ "fontcolorblue",
+ "backgroundcolorred",
+ "backgroundcolorgreen",
+ "backgroundcolorblue",
+ "bordercolorred",
+ "bordercolorgreen",
+ "bordercolorblue");
+
+ private static final int DEFAULT_MAX_CONFIG_CHARS = 6000;
+
+ /** Matches XML attributes with double-quoted values (e.g. {@code value="a<b"}). */
+ private static final Pattern DOUBLE_QUOTED_ATTR =
+ Pattern.compile("(\\s(?:[\\w:-]+))=\"([^\"]*)\"");
+
+ private HopXmlSupport() {}
+
+ public static boolean looksLikeHopXml(String text) {
+ if (text == null || text.isBlank()) {
+ return false;
+ }
+ String trimmed = text.stripLeading();
+ return trimmed.contains("<pipeline") || trimmed.contains("<workflow");
+ }
+
+ /**
+ * Parse Hop XML, returning {@code null} when the document cannot be parsed even after sanitizing
+ * unescaped {@code <} / {@code >} in attribute values (a common issue in real .hpl/.hwf files).
+ */
+ public static Document parseDocument(String xml) {
+ if (xml == null || xml.isBlank()) {
+ return null;
+ }
+ Document document = parseStrict(xml);
+ if (document != null) {
+ return document;
+ }
+ return parseStrict(sanitizeDoubleQuotedAttributes(xml));
+ }
+
+ /**
+ * @deprecated Prefer {@link #parseDocument(String)} which does not throw.
+ */
+ public static Document parse(String xml) {
+ Document document = parseDocument(xml);
+ if (document == null) {
+ throw new IllegalArgumentException("Invalid Hop XML");
+ }
+ return document;
+ }
+
+ static String sanitizeDoubleQuotedAttributes(String xml) {
+ Matcher matcher = DOUBLE_QUOTED_ATTR.matcher(xml);
+ StringBuilder result = new StringBuilder(xml.length() + 32);
+ int last = 0;
+ while (matcher.find()) {
+ result.append(xml, last, matcher.start());
+ String name = matcher.group(1);
+ String value = matcher.group(2);
+ if (value.indexOf('<') >= 0 || value.indexOf('>') >= 0) {
+ value = escapeRawLtGtInAttributeValue(value);
+ }
+ result.append(name).append("=\"").append(value).append('"');
+ last = matcher.end();
+ }
+ result.append(xml, last, xml.length());
+ return result.toString();
+ }
+
+ private static String escapeRawLtGtInAttributeValue(String value) {
+ return value.replace("<", "<").replace(">", ">");
+ }
+
+ private static Document parseStrict(String xml) {
+ try {
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+ factory.setExpandEntityReferences(false);
+ factory.setNamespaceAware(false);
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ return builder.parse(new InputSource(new StringReader(xml)));
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ public static Element firstChild(Element parent, String tagName) {
+ if (parent == null) {
+ return null;
+ }
+ for (Node node = parent.getFirstChild(); node != null; node = node.getNextSibling()) {
+ if (node.getNodeType() == Node.ELEMENT_NODE && tagName.equals(node.getNodeName())) {
+ return (Element) node;
+ }
+ }
+ return null;
+ }
+
+ public static List<Element> children(Element parent, String tagName) {
+ List<Element> result = new ArrayList<>();
+ if (parent == null) {
+ return result;
+ }
+ for (Node node = parent.getFirstChild(); node != null; node = node.getNextSibling()) {
+ if (node.getNodeType() == Node.ELEMENT_NODE && tagName.equals(node.getNodeName())) {
+ result.add((Element) node);
+ }
+ }
+ return result;
+ }
+
+ public static String directChildText(Element parent, String tagName) {
+ Element child = firstChild(parent, tagName);
+ return child != null ? textContent(child).trim() : "";
+ }
+
+ public static String textContent(Element element) {
+ StringBuilder sb = new StringBuilder();
+ appendText(element, sb);
+ return sb.toString();
+ }
+
+ private static void appendText(Node node, StringBuilder sb) {
+ if (node.getNodeType() == Node.TEXT_NODE || node.getNodeType() == Node.CDATA_SECTION_NODE) {
+ sb.append(node.getNodeValue());
+ return;
+ }
+ for (Node child = node.getFirstChild(); child != null; child = child.getNextSibling()) {
+ appendText(child, sb);
+ }
+ }
+
+ public static String formatParameters(Element parametersParent) {
+ if (parametersParent == null) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ for (Element parameter : children(parametersParent, "parameter")) {
+ String name = directChildText(parameter, "name");
+ if (name.isEmpty()) {
+ continue;
+ }
+ String defaultValue = directChildText(parameter, "default_value");
+ String description = directChildText(parameter, "description");
+ sb.append("- ").append(name).append(": default=").append(defaultValue);
+ if (!description.isEmpty()) {
+ sb.append(" — ").append(description);
+ }
+ sb.append('\n');
+ }
+ return sb.toString().strip();
+ }
+
+ public static String formatNotepads(Element root, String notepadsTag) {
+ Element notepads = firstChild(root, notepadsTag);
+ if (notepads == null) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ for (Element notepad : children(notepads, "notepad")) {
+ String note = directChildText(notepad, "note");
+ if (!note.isEmpty()) {
+ if (sb.length() > 0) {
+ sb.append("\n\n");
+ }
+ sb.append(note.trim());
+ }
+ }
+ return sb.toString().strip();
+ }
+
+ public static String serializeConfig(Element element) {
+ return serializeConfig(element, DEFAULT_MAX_CONFIG_CHARS);
+ }
+
+ public static String serializeConfig(Element element, int maxChars) {
+ StringBuilder sb = new StringBuilder();
+ serializeElement(element, sb, 0, 4);
+ String text = sb.toString().strip();
+ if (text.length() > maxChars) {
+ return text.substring(0, maxChars) + "\n...";
+ }
+ return text;
+ }
+
+ private static void serializeElement(Element element, StringBuilder sb, int depth, int maxDepth) {
+ if (depth > maxDepth || SKIP_TAGS.contains(element.getTagName())) {
+ return;
+ }
+
+ String tag = element.getTagName();
+ List<Element> childElements = new ArrayList<>();
+ StringBuilder textOnly = new StringBuilder();
+ for (Node node = element.getFirstChild(); node != null; node = node.getNextSibling()) {
+ if (node.getNodeType() == Node.ELEMENT_NODE) {
+ childElements.add((Element) node);
+ } else if (node.getNodeType() == Node.TEXT_NODE
+ || node.getNodeType() == Node.CDATA_SECTION_NODE) {
+ textOnly.append(node.getNodeValue());
+ }
+ }
+
+ String text = SecretRedaction.redact(tag, textOnly.toString().trim());
+ if (childElements.isEmpty()) {
+ if (!text.isEmpty() && !SKIP_TAGS.contains(tag)) {
+ indent(sb, depth).append(tag).append(": ").append(text).append('\n');
+ }
+ return;
+ }
+
+ if (childElements.size() == 1 && text.isEmpty()) {
+ indent(sb, depth).append(tag).append(":\n");
+ serializeElement(childElements.get(0), sb, depth + 1, maxDepth);
+ return;
+ }
+
+ indent(sb, depth).append(tag).append(":\n");
+ if (!text.isEmpty()) {
+ indent(sb, depth + 1).append(text).append('\n');
+ }
+ for (Element child : childElements) {
+ serializeElement(child, sb, depth + 1, maxDepth);
+ }
+ }
+
+ private static StringBuilder indent(StringBuilder sb, int depth) {
+ sb.append(" ".repeat(Math.max(0, depth)));
+ return sb;
+ }
+
+ public static List<Element> elementsByTag(NodeList nodeList) {
+ List<Element> elements = new ArrayList<>();
+ for (int i = 0; i < nodeList.getLength(); i++) {
+ Node node = nodeList.item(i);
+ if (node.getNodeType() == Node.ELEMENT_NODE) {
+ elements.add((Element) node);
+ }
+ }
+ return elements;
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/metadata/HopMetadataJsonParser.java b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/metadata/HopMetadataJsonParser.java
new file mode 100644
index 0000000000..f03acd4711
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/java/org/apache/hop/pipeline/transforms/chunker/document/metadata/HopMetadataJsonParser.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document.metadata;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.hop.core.json.HopJson;
+import org.apache.hop.core.util.JsonUtil;
+import org.apache.hop.pipeline.transforms.chunker.document.ContentType;
+import org.apache.hop.pipeline.transforms.chunker.document.DocumentNode;
+import org.apache.hop.pipeline.transforms.chunker.document.DocumentParser;
+import org.apache.hop.pipeline.transforms.chunker.document.SecretRedaction;
+
+/**
+ * Parses Hop project metadata JSON into one section per connection, engine, or top-level config
+ * block.
+ */
+public final class HopMetadataJsonParser implements DocumentParser {
+
+ @Override
+ public ContentType getContentType() {
+ return ContentType.METADATA;
+ }
+
+ @Override
+ public DocumentNode parse(String text) {
+ if (text == null || text.isBlank()) {
+ return DocumentNode.root("");
+ }
+
+ JsonNode root = parseRoot(text);
+ if (root == null || !root.isObject()) {
+ return DocumentNode.root(text);
+ }
+
+ List<DocumentNode> children = new ArrayList<>();
+ String name = textValue(root.get("name"));
+ String description = textValue(root.get("description"));
+
+ StringBuilder overview = new StringBuilder();
+ if (!name.isEmpty()) {
+ overview.append("Hop metadata '").append(name).append("'\n");
+ }
+ if (!description.isEmpty()) {
+ overview.append("Description: ").append(description).append('\n');
+ }
+ if (overview.length() > 0) {
+ children.add(DocumentNode.leaf("Overview", overview.toString().strip(), 0));
+ }
+
+ Iterator<Map.Entry<String, JsonNode>> fields = root.fields();
+ while (fields.hasNext()) {
+ Map.Entry<String, JsonNode> entry = fields.next();
+ String key = entry.getKey();
+ if ("name".equals(key) || "description".equals(key)) {
+ continue;
+ }
+ addSectionsForKey(children, key, entry.getValue());
+ }
+
+ if (children.isEmpty()) {
+ return DocumentNode.root(text);
+ }
+ return new DocumentNode("", "", 0, children);
+ }
+
+ private static void addSectionsForKey(List<DocumentNode> children, String key, JsonNode value) {
+ if (value == null || value.isNull()) {
+ return;
+ }
+ if ("rdbms".equals(key) && value.isObject()) {
+ value
+ .fields()
+ .forEachRemaining(
+ conn ->
+ children.add(
+ DocumentNode.leaf(
+ "Connection: " + conn.getKey(), formatConnection(conn.getValue()), 0)));
+ return;
+ }
+ if ("engineRunConfiguration".equals(key) && value.isObject()) {
+ value
+ .fields()
+ .forEachRemaining(
+ engine ->
+ children.add(
+ DocumentNode.leaf(
+ "Engine: " + engine.getKey(), prettyJson(engine.getValue()), 0)));
+ return;
+ }
+ if (value.isObject() && value.size() > 1 && allValuesAreObjects(value)) {
+ value
+ .fields()
+ .forEachRemaining(
+ child ->
+ children.add(
+ DocumentNode.leaf(
+ humanizeKey(key) + ": " + child.getKey(),
+ prettyJson(child.getValue()),
+ 0)));
+ return;
+ }
+ children.add(DocumentNode.leaf(humanizeKey(key), prettyJson(value), 0));
+ }
+
+ private static boolean allValuesAreObjects(JsonNode object) {
+ Iterator<JsonNode> values = object.elements();
+ while (values.hasNext()) {
+ if (!values.next().isObject()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static String formatConnection(JsonNode conn) {
+ if (conn == null || !conn.isObject()) {
+ return prettyJson(conn);
+ }
+ StringBuilder body = new StringBuilder();
+ appendLine(body, "Plugin", textValue(conn.get("pluginName")));
+ appendLine(body, "Host", textValue(conn.get("hostname")));
+ appendLine(body, "Port", textValue(conn.get("port")));
+ appendLine(body, "Database", textValue(conn.get("databaseName")));
+ appendLine(body, "Username", textValue(conn.get("username")));
+ appendLine(body, "Manual URL", textValue(conn.get("manualUrl")));
+ return body.toString().strip();
+ }
+
+ private static void appendLine(StringBuilder body, String label, String value) {
+ if (!value.isEmpty()) {
+ body.append(label).append(": ").append(value).append('\n');
+ }
+ }
+
+ private static String humanizeKey(String key) {
+ if (key == null || key.isEmpty()) {
+ return "Configuration";
+ }
+ String spaced = key.replaceAll("([a-z])([A-Z])", "$1 $2").replace('_', ' ');
+ return spaced.substring(0, 1).toUpperCase() + spaced.substring(1);
+ }
+
+ private static String textValue(JsonNode node) {
+ if (node == null || node.isNull()) {
+ return "";
+ }
+ if (node.isTextual()) {
+ return node.asText().strip();
+ }
+ return node.asText("").strip();
+ }
+
+ private static String prettyJson(JsonNode node) {
+ if (node == null || node.isNull()) {
+ return "";
+ }
+ try {
+ return HopJson.newMapper()
+ .writerWithDefaultPrettyPrinter()
+ .writeValueAsString(redactSecrets(node.deepCopy()));
+ } catch (Exception e) {
+ return node.toString();
+ }
+ }
+
+ /** Replace credential-bearing values before the node is turned into chunk text. */
+ private static JsonNode redactSecrets(JsonNode node) {
+ if (node instanceof ObjectNode object) {
+ Iterator<String> names = object.fieldNames();
+ List<String> fields = new ArrayList<>();
+ while (names.hasNext()) {
+ fields.add(names.next());
+ }
+ for (String field : fields) {
+ JsonNode value = object.get(field);
+ if (value != null && value.isValueNode()) {
+ String redacted = SecretRedaction.redact(field, value.asText());
+ if (SecretRedaction.REDACTED.equals(redacted)) {
+ object.put(field, SecretRedaction.REDACTED);
+ }
+ } else {
+ redactSecrets(value);
+ }
+ }
+ } else if (node != null && node.isArray()) {
+ node.forEach(HopMetadataJsonParser::redactSecrets);
+ }
+ return node;
+ }
+
+ static JsonNode parseRoot(String text) {
+ String trimmed = text.stripLeading();
+ if (trimmed.startsWith("{")) {
+ try {
+ return JsonUtil.parse(text);
+ } catch (Exception ignored) {
+ return null;
+ }
+ }
+ int marker = text.indexOf("Configuration:");
+ if (marker >= 0) {
+ String jsonPart = text.substring(marker + "Configuration:".length()).strip();
+ if (jsonPart.startsWith("{")) {
+ try {
+ return JsonUtil.parse(jsonPart);
+ } catch (Exception ignored) {
+ return null;
+ }
+ }
+ }
+ return null;
+ }
+}
diff --git a/plugins/transforms/textchunker/src/main/resources/chunker.svg b/plugins/transforms/textchunker/src/main/resources/chunker.svg
new file mode 100644
index 0000000000..6bb4fe69af
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/resources/chunker.svg
@@ -0,0 +1,26 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64" viewBox="0 0 64 64">
+ <defs>
+ <linearGradient id="gradient" x1="0%" y1="0%" x2="100%" y2="100%">
+ <stop offset="0%" style="stop-color:#4a90e2;stop-opacity:1" />
+ <stop offset="100%" style="stop-color:#2a6fc9;stop-opacity:1" />
+ </linearGradient>
+ </defs>
+ <rect width="64" height="64" rx="8" fill="url(#gradient)"/>
+
+ <!-- Document icon -->
+ <rect x="12" y="14" width="40" height="36" rx="2" fill="white"/>
+ <rect x="14" y="16" width="36" height="32" rx="1" fill="#f0f0f0"/>
+
+ <!-- Lines representing text -->
+ <line x1="16" y1="20" x2="48" y2="20" stroke="#666" stroke-width="1.5"/>
+ <line x1="16" y1="24" x2="44" y2="24" stroke="#666" stroke-width="1.5"/>
+ <line x1="16" y1="28" x2="48" y2="28" stroke="#666" stroke-width="1.5"/>
+
+ <!-- Plus sign for chunking -->
+ <line x1="30" y1="34" x2="34" y2="34" stroke="white" stroke-width="2"/>
+ <line x1="32" y1="32" x2="32" y2="36" stroke="white" stroke-width="2"/>
+
+ <!-- Additional lines -->
+ <line x1="16" y1="36" x2="30" y2="36" stroke="#666" stroke-width="1.5"/>
+ <line x1="38" y1="36" x2="48" y2="36" stroke="#666" stroke-width="1.5"/>
+</svg>
diff --git a/plugins/transforms/textchunker/src/main/resources/org/apache/hop/pipeline/transforms/chunker/messages/messages_en_US.properties b/plugins/transforms/textchunker/src/main/resources/org/apache/hop/pipeline/transforms/chunker/messages/messages_en_US.properties
new file mode 100644
index 0000000000..642b6d0430
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/resources/org/apache/hop/pipeline/transforms/chunker/messages/messages_en_US.properties
@@ -0,0 +1,85 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+TextChunkerDialog.Shell.Title=Text chunker
+TextChunkerDialog.GetFields.Error.Title=Error getting fields
+TextChunkerDialog.GetFields.Error.Message=Unable to read fields from the previous transform
+
+TextChunker.Name=Text chunker
+TextChunker.Description=Split text into chunks for AI/ML processing without breaking words or sentences
+TextChunker.Keywords=AI,LLM,text,chunk,split,paragraph,character,word,embedding
+
+TextChunker.inputField.Label=Input field
+TextChunker.inputField.Tooltip=Select the field containing text to chunk
+
+TextChunker.sourceDocumentIdField.Label=Source document ID field
+TextChunker.sourceDocumentIdField.Tooltip=Optional input field to use as the document identifier in metadata. When empty, a row counter is used.
+
+TextChunker.outputChunkField.Label=Output chunk field
+TextChunker.outputChunkField.Tooltip=Name of the field to output chunks to
+
+TextChunker.chunkingStrategy.Label=Chunking strategy
+TextChunker.chunkingStrategy.Tooltip=Choose how to split text: Character, Paragraph, or Structure (heading-aware with breadcrumb prefixes)
+
+TextChunker.contentType.Label=Content type
+TextChunker.contentType.Tooltip=Document format for Structure strategy. Auto detects from source_type field or text heuristics.
+
+TextChunker.contentTypeField.Label=Content type field
+TextChunker.contentTypeField.Tooltip=Optional input field (e.g. source_type) to select parser: doc=AsciiDoc, pipeline/workflow=Hop XML, article/blog=Markdown
+
+TextChunker.chunkSize.Label=Chunk size
+TextChunker.chunkSize.Tooltip=Maximum size for each chunk in characters (approximate for paragraphs)
+
+TextChunker.chunkOverlap.Label=Chunk overlap
+TextChunker.chunkOverlap.Tooltip=Number of characters to overlap between chunks (for continuity)
+
+TextChunker.includeMetadata.Label=Include metadata
+TextChunker.includeMetadata.Tooltip=Add metadata fields (chunk index, position, document ID) to output
+
+TextChunker.chunkIndexField.Label=Chunk index field
+TextChunker.chunkIndexField.Tooltip=Name of the field for chunk index (0-based)
+
+TextChunker.chunkStartPosField.Label=Chunk start position field
+TextChunker.chunkStartPosField.Tooltip=Name of the field for chunk start position in original text
+
+TextChunker.documentIdField.Label=Document ID field
+TextChunker.documentIdField.Tooltip=Name of the field for document ID (unique per input row)
+
+TextChunker.chunkCountField.Label=Total chunks field
+TextChunker.chunkCountField.Tooltip=Name of the field for total number of chunks per document
+
+ChunkingStrategy.CHARACTER=Character
+ChunkingStrategy.PARAGRAPH=Paragraph
+ChunkingStrategy.STRUCTURE=Structure
+
+TextChunker.Validation.InputFieldRequired=Input field must be specified
+TextChunker.Validation.ChunkSizePositive=Chunk size must be greater than 0
+TextChunker.Validation.OverlapNonNegative=Chunk overlap cannot be negative
+TextChunker.Validation.OverlapWarning=Chunk overlap should be less than chunk size for best results
+TextChunker.Validation.InputFieldNotFound=Input field ''{0}'' not found in the input stream
+
+TextChunker.Error.ReadingInputField=Error reading input field
+TextChunker.Error.CreatingOutputRow=Error creating output row
+TextChunker.Error.UnknownStrategy=Unknown chunking strategy: {0}
+
+TextChunker.Log.Initialized=Initialized Text Chunker with strategy: {0}
+TextChunker.Log.ProcessedDocument=Processed document with {0} chunks
+TextChunker.Log.EmptyText=Empty text received, emitting a single empty chunk row
+
+TextChunker.Validation.OutputChunkFieldRequired=Output chunk field name must be specified
+TextChunker.Validation.SourceDocumentIdFieldNotFound=Source document ID field ''{0}'' not found in the input stream
+TextChunker.Validation.ContentTypeFieldNotFound=Content type field ''{0}'' not found in the input stream
+TextChunker.Validation.ParagraphOverlapIgnored=The Paragraph strategy does not apply chunk overlap; the configured overlap is ignored
diff --git a/plugins/transforms/textchunker/src/main/resources/version.xml b/plugins/transforms/textchunker/src/main/resources/version.xml
new file mode 100644
index 0000000000..6be576acae
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/resources/version.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<version>${project.version}</version>
\ No newline at end of file
diff --git a/plugins/transforms/textchunker/src/main/samples/transforms/ai-text-chunker-basic.hpl b/plugins/transforms/textchunker/src/main/samples/transforms/ai-text-chunker-basic.hpl
new file mode 100644
index 0000000000..b7efcff88d
--- /dev/null
+++ b/plugins/transforms/textchunker/src/main/samples/transforms/ai-text-chunker-basic.hpl
@@ -0,0 +1,166 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>ai-text-chunker-basic</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Split long text into overlapping chunks for AI/RAG workflows using the Text Chunker transform.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/06/07 21:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/06/07 21:00:00.000</modified_date>
+ </info>
+ <notepads>
+ <notepad>
+ <note>AI Text Chunker sample: splits a document into fixed-size chunks while respecting word boundaries. Each input row produces multiple output rows (one per chunk) with optional metadata fields for chunk index, start position, document id, and total chunk count. Run the pipeline and inspect the preview on the Dummy transform.</note>
+ <xloc>80</xloc>
+ <yloc>48</yloc>
+ <width>720</width>
+ <heigth>72</heigth>
+ <fontname>Noto Sans</fontname>
+ <fontsize>10</fontsize>
+ <fontbold>N</fontbold>
+ <fontitalic>N</fontitalic>
+ <fontcolorred>14</fontcolorred>
+ <fontcolorgreen>58</fontcolorgreen>
+ <fontcolorblue>90</fontcolorblue>
+ <backgroundcolorred>201</backgroundcolorred>
+ <backgroundcolorgreen>232</backgroundcolorgreen>
+ <backgroundcolorblue>251</backgroundcolorblue>
+ <bordercolorred>14</bordercolorred>
+ <bordercolorgreen>58</bordercolorgreen>
+ <bordercolorblue>90</bordercolorblue>
+ </notepad>
+ </notepads>
+ <order>
+ <hop>
+ <from>Sample document</from>
+ <to>Text Chunker</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Text Chunker</from>
+ <to>Preview chunks</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Sample document</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <set_empty_string>N</set_empty_string>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>document_id</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <set_empty_string>N</set_empty_string>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>text</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ <item>Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>192</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Text Chunker</name>
+ <type>TextChunker</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <inputField>text</inputField>
+ <outputChunkField>chunk_text</outputChunkField>
+ <chunkingStrategy>CHARACTER</chunkingStrategy>
+ <chunkSize>100</chunkSize>
+ <chunkOverlap>20</chunkOverlap>
+ <includeMetadata>Y</includeMetadata>
+ <chunkIndexField>chunk_index</chunkIndexField>
+ <chunkStartPosField>chunk_start_position</chunkStartPosField>
+ <documentIdField>chunk_doc_id</documentIdField>
+ <chunkCountField>total_chunks</chunkCountField>
+ <attributes/>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>192</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Preview chunks</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>192</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMetaTest.java b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMetaTest.java
new file mode 100644
index 0000000000..bc51b4eab9
--- /dev/null
+++ b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerMetaTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.ICheckResult;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.xml.XmlHandler;
+import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyType;
+import org.apache.hop.pipeline.transforms.chunker.document.ContentType;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.w3c.dom.Document;
+
+class TextChunkerMetaTest {
+
+ @BeforeAll
+ static void setUpClass() throws Exception {
+ HopClientEnvironment.init();
+ }
+
+ /**
+ * Every {@code @HopMetadataProperty} has to survive a write/read cycle. Without this, a renamed
+ * or mistyped key only shows up as silently lost settings after a save and reopen.
+ */
+ @Test
+ void roundTripsEveryPropertyThroughXml() throws Exception {
+ TextChunkerMeta original = new TextChunkerMeta();
+ original.setInputField("body");
+ original.setOutputChunkField("piece");
+ original.setChunkingStrategy(ChunkingStrategyType.STRUCTURE);
+ original.setChunkSize("512");
+ original.setChunkOverlap("64");
+ original.setIncludeMetadata(true);
+ original.setChunkIndexField("idx");
+ original.setChunkStartPosField("pos");
+ original.setSourceDocumentIdField("doc");
+ original.setDocumentIdField("doc_out");
+ original.setChunkCountField("total");
+ original.setContentType(ContentType.ASCIIDOC);
+ original.setContentTypeField("source_type");
+
+ TextChunkerMeta copy = roundTrip(original);
+
+ assertEquals(original.getInputField(), copy.getInputField());
+ assertEquals(original.getOutputChunkField(), copy.getOutputChunkField());
+ assertEquals(original.getChunkingStrategy(), copy.getChunkingStrategy());
+ assertEquals(original.getChunkSize(), copy.getChunkSize());
+ assertEquals(original.getChunkOverlap(), copy.getChunkOverlap());
+ assertEquals(original.isIncludeMetadata(), copy.isIncludeMetadata());
+ assertEquals(original.getChunkIndexField(), copy.getChunkIndexField());
+ assertEquals(original.getChunkStartPosField(), copy.getChunkStartPosField());
+ assertEquals(original.getSourceDocumentIdField(), copy.getSourceDocumentIdField());
+ assertEquals(original.getDocumentIdField(), copy.getDocumentIdField());
+ assertEquals(original.getChunkCountField(), copy.getChunkCountField());
+ assertEquals(original.getContentType(), copy.getContentType());
+ assertEquals(original.getContentTypeField(), copy.getContentTypeField());
+ }
+
+ /** Chunk index, start position and total count are numbers, not strings. */
+ @Test
+ void exposesChunkMetadataAsIntegers() throws Exception {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+
+ IRowMeta row = new RowMeta();
+ row.addValueMeta(new ValueMetaString("text"));
+ meta.getFields(row, "chunker", null, null, new Variables(), null);
+
+ assertEquals(IValueMeta.TYPE_STRING, row.searchValueMeta("chunk_text").getType());
+ assertEquals(IValueMeta.TYPE_INTEGER, row.searchValueMeta("chunk_index").getType());
+ assertEquals(IValueMeta.TYPE_INTEGER, row.searchValueMeta("chunk_start_position").getType());
+ assertEquals(IValueMeta.TYPE_INTEGER, row.searchValueMeta("total_chunks").getType());
+ assertEquals(IValueMeta.TYPE_STRING, row.searchValueMeta("chunk_doc_id").getType());
+ }
+
+ @Test
+ void checkReportsMissingInputField() {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(remarks, null, new TransformMeta(), null, null, null, null, new Variables(), null);
+
+ assertTrue(
+ remarks.stream().anyMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_ERROR),
+ "an empty input field must be reported as an error");
+ }
+
+ @Test
+ void checkReportsInputFieldMissingFromTheStream() {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("body");
+
+ IRowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaString("something_else"));
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(remarks, null, new TransformMeta(), prev, null, null, null, new Variables(), null);
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r -> r.getType() == ICheckResult.TYPE_RESULT_ERROR && r.getText().contains("body")),
+ "an input field absent from the stream must be reported");
+ }
+
+ @Test
+ void checkAcceptsAValidConfiguration() {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("body");
+
+ IRowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaString("body"));
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(remarks, null, new TransformMeta(), prev, null, null, null, new Variables(), null);
+
+ assertTrue(
+ remarks.stream().noneMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_ERROR),
+ "a valid configuration must not produce errors");
+ }
+
+ /**
+ * The generated dialogs fill an enum combo with {@code toString()} and read it back with {@code
+ * Enum.valueOf}. An enum that overrides {@code toString()} therefore cannot be read back, and a
+ * changed selection is dropped without any error reaching the user.
+ */
+ @Test
+ void everyEnumOnTheDialogReadsBackFromItsDisplayedText() {
+ for (Enum<?>[] constants :
+ new Enum<?>[][] {ChunkingStrategyType.values(), ContentType.values()}) {
+ for (Enum<?> constant : constants) {
+ assertEquals(
+ constant.name(),
+ constant.toString(),
+ constant.getDeclaringClass().getSimpleName()
+ + " must not override toString(): the dialog reads the combo back with"
+ + " Enum.valueOf, which only accepts the constant name");
+ }
+ }
+ }
+
+ @Test
+ void theStrategyStillParsesTheDisplayTextItUsedToShow() {
+ // Pipelines saved while the combo showed "Character" have to keep working.
+ assertEquals(ChunkingStrategyType.CHARACTER, ChunkingStrategyType.fromString("Character"));
+ assertEquals(ChunkingStrategyType.PARAGRAPH, ChunkingStrategyType.fromString("Paragraph"));
+ assertEquals(ChunkingStrategyType.STRUCTURE, ChunkingStrategyType.fromString("Structure"));
+ assertEquals(ChunkingStrategyType.STRUCTURE, ChunkingStrategyType.fromString("STRUCTURE"));
+ }
+
+ private static TextChunkerMeta roundTrip(TextChunkerMeta original) throws Exception {
+ String xml = "<transform>" + XmlMetadataUtil.serializeObjectToXml(original) + "</transform>";
+ Document document = XmlHandler.loadXmlString(xml);
+ return XmlMetadataUtil.deSerializeFromXml(
+ XmlHandler.getSubNode(document, "transform"), TextChunkerMeta.class, null);
+ }
+}
diff --git a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerTest.java b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerTest.java
new file mode 100644
index 0000000000..63d9be9172
--- /dev/null
+++ b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/TextChunkerTest.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.transforms.chunker.chunking.ChunkingStrategyType;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class TextChunkerTest {
+
+ private TransformMockHelper<TextChunkerMeta, TextChunkerData> helper;
+ private List<Object[]> output;
+ private IRowMeta outputRowMeta;
+
+ @BeforeAll
+ static void setUpClass() throws Exception {
+ HopClientEnvironment.init();
+ }
+
+ @BeforeEach
+ void setUp() {
+ helper = new TransformMockHelper<>("TextChunker", TextChunkerMeta.class, TextChunkerData.class);
+ when(helper.logChannelFactory.create(any(), any())).thenReturn(helper.iLogChannel);
+ when(helper.pipeline.isRunning()).thenReturn(true);
+ output = new ArrayList<>();
+ }
+
+ @AfterEach
+ void tearDown() {
+ helper.cleanUp();
+ }
+
+ @Test
+ void emitsOneRowPerChunkWithNumericMetadata() throws Exception {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+ meta.setChunkingStrategy(ChunkingStrategyType.CHARACTER);
+ meta.setChunkSize("10");
+ meta.setChunkOverlap("0");
+
+ // 3 words of 9 characters each, so the character strategy produces several chunks.
+ run(meta, rowMeta(), List.<Object[]>of(new Object[] {"aaaaaaaa bbbbbbbb cccccccc"}));
+
+ assertTrue(output.size() > 1, "expected the text to be split into multiple chunks");
+
+ int indexPos = outputRowMeta.indexOfValue("chunk_index");
+ int startPos = outputRowMeta.indexOfValue("chunk_start_position");
+ int totalPos = outputRowMeta.indexOfValue("total_chunks");
+ int docPos = outputRowMeta.indexOfValue("chunk_doc_id");
+
+ for (int i = 0; i < output.size(); i++) {
+ Object[] row = output.get(i);
+ assertEquals(Long.valueOf(i), row[indexPos], "chunk_index must be a sequential Long");
+ assertTrue(row[startPos] instanceof Long, "chunk_start_position must be a Long");
+ assertEquals(
+ Long.valueOf(output.size()), row[totalPos], "total_chunks must be a Long and correct");
+ assertNotNull(row[docPos]);
+ }
+ }
+
+ @Test
+ void usesTheSourceDocumentIdFieldWhenConfigured() throws Exception {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+ meta.setSourceDocumentIdField("doc");
+ meta.setChunkSize("1000");
+
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("text"));
+ rowMeta.addValueMeta(new ValueMetaString("doc"));
+
+ run(meta, rowMeta, List.<Object[]>of(new Object[] {"short text", "DOC-42"}));
+
+ int docPos = outputRowMeta.indexOfValue("chunk_doc_id");
+ assertEquals(1, output.size());
+ assertEquals("DOC-42", output.get(0)[docPos]);
+ }
+
+ /** Document IDs are prefixed with the copy number so parallel copies cannot collide. */
+ @Test
+ void generatedDocumentIdsAreUniquePerCopy() throws Exception {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+ meta.setChunkSize("1000");
+
+ run(meta, rowMeta(), List.<Object[]>of(new Object[] {"one"}, new Object[] {"two"}));
+
+ int docPos = outputRowMeta.indexOfValue("chunk_doc_id");
+ assertEquals(2, output.size());
+ assertEquals("0_0", output.get(0)[docPos]);
+ assertEquals("0_1", output.get(1)[docPos]);
+ }
+
+ @Test
+ void emitsASingleRowForEmptyText() throws Exception {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+
+ run(meta, rowMeta(), List.<Object[]>of(new Object[] {""}));
+
+ int chunkPos = outputRowMeta.indexOfValue("chunk_text");
+ int totalPos = outputRowMeta.indexOfValue("total_chunks");
+ assertEquals(1, output.size());
+ assertEquals("", output.get(0)[chunkPos]);
+ assertEquals(0L, output.get(0)[totalPos]);
+ }
+
+ private static IRowMeta rowMeta() {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("text"));
+ return rowMeta;
+ }
+
+ private void run(TextChunkerMeta meta, IRowMeta inputRowMeta, List<Object[]> rows)
+ throws Exception {
+ TextChunkerData data = new TextChunkerData();
+ TextChunker transform =
+ spy(
+ new TextChunker(
+ helper.transformMeta, meta, data, 0, helper.pipelineMeta, helper.pipeline));
+ transform.init();
+ transform.setInputRowMeta(inputRowMeta);
+
+ Iterator<Object[]> iterator = rows.iterator();
+ doAnswer(invocation -> iterator.hasNext() ? iterator.next() : null).when(transform).getRow();
+ doAnswer(
+ invocation -> {
+ outputRowMeta = invocation.getArgument(0);
+ output.add(invocation.getArgument(1));
+ return null;
+ })
+ .when(transform)
+ .putRow(any(IRowMeta.class), any(Object[].class));
+
+ while (transform.processRow()) {
+ // drain
+ }
+ }
+
+ /**
+ * Cross-module contract: PgVectorUpsert reads chunk_index back out with getString() and parses it
+ * as an int. Now that the chunker emits it as an Integer rather than a String, the default
+ * conversion must still yield plain digits with no mask or grouping separator.
+ */
+ @Test
+ void integerChunkMetadataStringifiesToPlainDigits() throws Exception {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("chunk_index"));
+
+ for (long value : new long[] {0L, 5L, 42L, 1234L, 1234567L}) {
+ String rendered = rowMeta.getString(new Object[] {value}, 0);
+ assertEquals(
+ Long.toString(value), rendered, "chunk_index must render without mask or grouping");
+ assertEquals(value, Integer.parseInt(rendered.trim()));
+ }
+ }
+
+ @Test
+ void resolvesChunkSizeAndOverlapFromVariables() throws Exception {
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+ meta.setChunkingStrategy(ChunkingStrategyType.CHARACTER);
+ meta.setChunkSize("${CHUNK_SIZE}");
+ meta.setChunkOverlap("${CHUNK_OVERLAP}");
+
+ TextChunkerData data = new TextChunkerData();
+ TextChunker transform =
+ new TextChunker(helper.transformMeta, meta, data, 0, helper.pipelineMeta, helper.pipeline);
+ transform.setVariable("CHUNK_SIZE", "10");
+ transform.setVariable("CHUNK_OVERLAP", "3");
+
+ assertTrue(transform.init(), "init must succeed once the variables resolve");
+ assertEquals(10, data.chunkSize);
+ assertEquals(3, data.chunkOverlap);
+ }
+
+ @Test
+ void initFailsWhenTheResolvedChunkSizeIsNotPositive() {
+ // Metadata injection can set this to 0 even though check() reports an error.
+ TextChunkerMeta meta = new TextChunkerMeta();
+ meta.setDefault();
+ meta.setInputField("text");
+ meta.setChunkSize("0");
+
+ TextChunkerData data = new TextChunkerData();
+ TextChunker transform =
+ new TextChunker(helper.transformMeta, meta, data, 0, helper.pipelineMeta, helper.pipeline);
+
+ assertFalse(transform.init(), "a non-positive chunk size must fail init, not drop rows");
+ }
+}
diff --git a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/chunking/CharacterChunkingStrategyTest.java b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/chunking/CharacterChunkingStrategyTest.java
new file mode 100644
index 0000000000..1c252a674f
--- /dev/null
+++ b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/chunking/CharacterChunkingStrategyTest.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.chunking;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import org.apache.hop.pipeline.transforms.chunker.Chunk;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for CharacterChunkingStrategy. */
+public class CharacterChunkingStrategyTest {
+
+ private CharacterChunkingStrategy strategy;
+
+ @BeforeEach
+ public void setUp() {
+ strategy = new CharacterChunkingStrategy();
+ }
+
+ @Test
+ public void testNullText() {
+ List<Chunk> chunks = strategy.chunk(null, 100, 20);
+ assertTrue(chunks.isEmpty());
+ }
+
+ @Test
+ public void testEmptyText() {
+ List<Chunk> chunks = strategy.chunk("", 100, 20);
+ assertTrue(chunks.isEmpty());
+ }
+
+ @Test
+ public void testSingleChunk() {
+ String text = "This is a short text";
+ List<Chunk> chunks = strategy.chunk(text, 100, 20);
+ assertEquals(1, chunks.size());
+ assertEquals(text, chunks.get(0).getContent());
+ assertEquals(0, chunks.get(0).getIndex());
+ assertEquals(0, chunks.get(0).getStartPosition());
+ assertEquals(text.length(), chunks.get(0).getEndPosition());
+ }
+
+ @Test
+ public void testChunkingRespectsWordBoundaries() {
+ String text = "This is a test string for chunking.";
+ List<Chunk> chunks = strategy.chunk(text, 10, 2);
+
+ // Should not split words
+ for (Chunk chunk : chunks) {
+ String content = chunk.getContent();
+ // Check that the chunk doesn't end with a partial word (unless it's the last chunk)
+ if (chunk.getIndex() < chunks.size() - 1) {
+ // The last character should be a whitespace or the chunk should end at a word boundary
+ char lastChar = content.charAt(content.length() - 1);
+ assertTrue(
+ Character.isWhitespace(lastChar) || lastChar == '.' || lastChar == ',',
+ "Chunk should not end mid-word: " + content);
+ }
+ }
+ }
+
+ @Test
+ public void testChunkingWithOverlap() {
+ String text = "This is a test string for chunking with overlap.";
+ List<Chunk> chunks = strategy.chunk(text, 15, 5);
+
+ assertTrue(chunks.size() > 1, "Should have multiple chunks");
+
+ // Verify overlap exists between chunks
+ for (int i = 0; i < chunks.size() - 1; i++) {
+ Chunk current = chunks.get(i);
+ Chunk next = chunks.get(i + 1);
+
+ String currentContent = current.getContent();
+ String nextContent = next.getContent();
+
+ // The beginning of the next chunk should overlap with the end of the current chunk
+ String overlapPart = nextContent.substring(0, Math.min(5, nextContent.length()));
+ String endOfCurrent = currentContent.substring(Math.max(0, currentContent.length() - 5));
+
+ // At least some overlap should exist
+ assertTrue(
+ endOfCurrent.contains(overlapPart.substring(0, Math.min(2, overlapPart.length()))));
+ }
+ }
+
+ @Test
+ public void testVeryLongWord() {
+ String text = "ThisIsAVeryLongWordThatShouldBeSplitBecauseItExceedsTheChunkSizeLimit";
+ List<Chunk> chunks = strategy.chunk(text, 20, 5);
+
+ assertFalse(chunks.isEmpty());
+
+ // Overlap means chunks deliberately share characters, so summing their lengths would exceed
+ // the source. What has to hold is that they cover it and address it correctly.
+ assertEquals(0, chunks.get(0).getStartPosition());
+ assertEquals(text.length(), chunks.get(chunks.size() - 1).getEndPosition());
+ for (Chunk chunk : chunks) {
+ assertEquals(
+ text.substring(chunk.getStartPosition(), chunk.getEndPosition()), chunk.getContent());
+ }
+
+ // A word with no whitespace in it still gets the configured overlap.
+ assertTrue(
+ chunks.get(1).getStartPosition() < chunks.get(0).getEndPosition(),
+ "chunks after a non-whitespace split must still overlap");
+ }
+
+ @Test
+ public void testZeroChunkSize() {
+ List<Chunk> chunks = strategy.chunk("test", 0, 0);
+ assertTrue(chunks.isEmpty());
+ }
+
+ @Test
+ public void testNegativeOverlap() {
+ String text = "This is a test.";
+ List<Chunk> chunks = strategy.chunk(text, 10, -5);
+ assertEquals(1, chunks.size());
+ }
+
+ @Test
+ public void testOverlapLargerThanChunkSize() {
+ String text = "This is a test.";
+ List<Chunk> chunks = strategy.chunk(text, 10, 15);
+ // Should still work, but overlap will be capped
+ assertFalse(chunks.isEmpty());
+ }
+
+ @Test
+ public void testMultipleSpaces() {
+ String text = "This has multiple spaces between words.";
+ List<Chunk> chunks = strategy.chunk(text, 10, 2);
+
+ // Should handle multiple spaces correctly
+ assertFalse(chunks.isEmpty());
+
+ // Verify all text is preserved (overlapping chunks must be merged by span, not naive concat)
+ assertEquals(text, reconstructFromChunks(text, chunks));
+ }
+
+ @Test
+ public void testPunctuation() {
+ String text = "Hello, world! How are you? I'm fine, thank you.";
+ List<Chunk> chunks = strategy.chunk(text, 15, 3);
+
+ assertFalse(chunks.isEmpty());
+
+ // Verify all text is preserved including punctuation
+ assertEquals(text, reconstructFromChunks(text, chunks));
+ }
+
+ @Test
+ public void testNewlines() {
+ String text = "Line one\nLine two\nLine three";
+ List<Chunk> chunks = strategy.chunk(text, 10, 2);
+
+ assertFalse(chunks.isEmpty());
+
+ // Verify newlines are preserved
+ assertEquals(text, reconstructFromChunks(text, chunks));
+ }
+
+ private static String reconstructFromChunks(String text, List<Chunk> chunks) {
+ if (chunks.isEmpty()) {
+ return "";
+ }
+ StringBuilder reconstructed =
+ new StringBuilder(text.substring(0, chunks.get(0).getEndPosition()));
+ for (int i = 1; i < chunks.size(); i++) {
+ Chunk previous = chunks.get(i - 1);
+ Chunk current = chunks.get(i);
+ reconstructed.append(text, previous.getEndPosition(), current.getEndPosition());
+ }
+ return reconstructed.toString();
+ }
+
+ @Test
+ public void testStrategyType() {
+ assertEquals(ChunkingStrategyType.CHARACTER, strategy.getType());
+ }
+
+ @Test
+ public void testExactChunkSize() {
+ // Test that we can get exact chunk sizes when possible
+ String text = "12345 67890 12345 67890";
+ List<Chunk> chunks = strategy.chunk(text, 11, 0);
+
+ // Should split on spaces at approximately 11 characters
+ for (Chunk chunk : chunks) {
+ // Each chunk should be close to 11 characters
+ assertTrue(chunk.getLength() <= 11 || chunk.getIndex() == chunks.size() - 1);
+ }
+ }
+}
diff --git a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/chunking/ParagraphChunkingStrategyTest.java b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/chunking/ParagraphChunkingStrategyTest.java
new file mode 100644
index 0000000000..a0bea39405
--- /dev/null
+++ b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/chunking/ParagraphChunkingStrategyTest.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.chunking;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import org.apache.hop.pipeline.transforms.chunker.Chunk;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for ParagraphChunkingStrategy. */
+public class ParagraphChunkingStrategyTest {
+
+ private ParagraphChunkingStrategy strategy;
+
+ @BeforeEach
+ public void setUp() {
+ strategy = new ParagraphChunkingStrategy();
+ }
+
+ @Test
+ public void testNullText() {
+ List<Chunk> chunks = strategy.chunk(null, 100, 20);
+ assertTrue(chunks.isEmpty());
+ }
+
+ @Test
+ public void testEmptyText() {
+ List<Chunk> chunks = strategy.chunk("", 100, 20);
+ assertTrue(chunks.isEmpty());
+ }
+
+ @Test
+ public void testSingleParagraph() {
+ String text = "This is a single paragraph with no breaks.";
+ List<Chunk> chunks = strategy.chunk(text, 100, 20);
+ assertEquals(1, chunks.size());
+ assertEquals(text, chunks.get(0).getContent());
+ }
+
+ @Test
+ public void testMultipleParagraphs() {
+ String text = "First paragraph.\n\nSecond paragraph.\n\nThird paragraph.";
+ List<Chunk> chunks = strategy.chunk(text, 20, 0);
+
+ // Should split on double newlines
+ assertEquals(3, chunks.size());
+ assertEquals("First paragraph.", chunks.get(0).getContent());
+ assertEquals("Second paragraph.", chunks.get(1).getContent());
+ assertEquals("Third paragraph.", chunks.get(2).getContent());
+ }
+
+ @Test
+ public void testParagraphsAreKeptWholeWhenTheyFit() {
+ // Within maxSize, a paragraph is never split.
+ String text =
+ "This is paragraph one.\n\nThis is paragraph two with more text that is quite long.";
+ List<Chunk> chunks = strategy.chunk(text, 60, 0);
+
+ assertEquals(2, chunks.size());
+ assertEquals("This is paragraph one.", chunks.get(0).getContent());
+ assertEquals(
+ "This is paragraph two with more text that is quite long.", chunks.get(1).getContent());
+ }
+
+ @Test
+ public void testOversizedParagraphIsSplitOnCharacters() {
+ // A paragraph larger than maxSize has to be split: an embedding model would reject a chunk
+ // over its token limit, so emitting it whole is not an option.
+ String text =
+ "This is paragraph one.\n\nThis is paragraph two with more text that is quite long.";
+ List<Chunk> chunks = strategy.chunk(text, 10, 2);
+
+ assertTrue(chunks.size() > 2, "oversized paragraphs should be split further");
+ for (Chunk chunk : chunks) {
+ assertTrue(
+ chunk.getContent().length() <= 10, "chunk exceeds maxSize: '" + chunk.getContent() + "'");
+ }
+ }
+
+ @Test
+ public void testChunkPositionsPointIntoTheOriginalText() {
+ String text = "First paragraph.\r\n\r\nSecond paragraph.";
+ List<Chunk> chunks = strategy.chunk(text, 100, 0);
+
+ for (Chunk chunk : chunks) {
+ assertEquals(
+ chunk.getContent(),
+ text.substring(chunk.getStartPosition(), chunk.getEndPosition()),
+ "start/end positions must address the chunk in the source text");
+ }
+ }
+
+ @Test
+ public void testWindowsLineEndings() {
+ String text = "First paragraph.\r\n\r\nSecond paragraph.";
+ List<Chunk> chunks = strategy.chunk(text, 20, 0);
+
+ assertEquals(2, chunks.size());
+ assertEquals("First paragraph.", chunks.get(0).getContent().replace("\r", ""));
+ assertEquals("Second paragraph.", chunks.get(1).getContent().replace("\r", ""));
+ }
+
+ @Test
+ public void testMixedLineEndings() {
+ String text = "First paragraph.\n\r\nSecond paragraph.";
+ List<Chunk> chunks = strategy.chunk(text, 20, 0);
+
+ assertEquals(2, chunks.size());
+ }
+
+ @Test
+ public void testSingleNewline() {
+ // Single newline should not create a paragraph break
+ String text = "Line one\nLine two";
+ List<Chunk> chunks = strategy.chunk(text, 100, 20);
+
+ assertEquals(1, chunks.size());
+ assertEquals(text, chunks.get(0).getContent());
+ }
+
+ @Test
+ public void testEmptyParagraphs() {
+ String text = "First\n\n\n\nSecond";
+ List<Chunk> chunks = strategy.chunk(text, 100, 20);
+
+ // Should handle empty paragraphs gracefully
+ assertTrue(chunks.size() >= 1);
+ assertTrue(chunks.size() <= 2);
+ }
+
+ @Test
+ public void testTrailingNewlines() {
+ String text = "First paragraph.\n\nSecond paragraph.\n\n";
+ List<Chunk> chunks = strategy.chunk(text, 20, 0);
+
+ assertEquals(2, chunks.size());
+ }
+
+ @Test
+ public void testLeadingNewlines() {
+ String text = "\n\nFirst paragraph.\n\nSecond paragraph.";
+ List<Chunk> chunks = strategy.chunk(text, 20, 0);
+
+ assertEquals(2, chunks.size());
+ }
+
+ @Test
+ public void testGroupingBySize() {
+ // Test that paragraphs are grouped when they fit within maxSize
+ String text = "Short.\n\nShort.\n\nVery long paragraph that exceeds the chunk size limit.";
+ List<Chunk> chunks = strategy.chunk(text, 50, 10);
+
+ // First two paragraphs should be grouped together
+ assertTrue(chunks.size() >= 2);
+
+ // First chunk should contain both short paragraphs
+ String firstChunk = chunks.get(0).getContent();
+ assertTrue(firstChunk.contains("Short.") && firstChunk.contains("Short."));
+ }
+
+ @Test
+ public void testAllTextPreserved() {
+ String text = "Paragraph one.\n\nParagraph two.\n\nParagraph three.";
+ List<Chunk> chunks = strategy.chunk(text, 100, 20);
+
+ // Reconstruct the original text (without normalizing line endings)
+ StringBuilder reconstructed = new StringBuilder();
+ for (int i = 0; i < chunks.size(); i++) {
+ if (i > 0) {
+ reconstructed.append("\n\n");
+ }
+ reconstructed.append(chunks.get(i).getContent());
+ }
+
+ assertEquals(text, reconstructed.toString());
+ }
+
+ @Test
+ public void testStrategyType() {
+ assertEquals(ChunkingStrategyType.PARAGRAPH, strategy.getType());
+ }
+
+ @Test
+ public void testTextWithTabsAndSpaces() {
+ String text = "Paragraph one.\n\n\tParagraph two with tab.\n\n Paragraph three with spaces.";
+ List<Chunk> chunks = strategy.chunk(text, 35, 0);
+
+ assertEquals(3, chunks.size());
+ }
+
+ @Test
+ public void testVeryLongParagraphs() {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 10; i++) {
+ if (i > 0) {
+ sb.append("\n\n");
+ }
+ sb.append("This is paragraph number ").append(i).append(" with some text.");
+ }
+ String text = sb.toString();
+
+ List<Chunk> chunks = strategy.chunk(text, 20, 5);
+
+ // Should split on paragraph boundaries
+ assertTrue(chunks.size() > 1);
+
+ // Each chunk should contain complete paragraphs
+ for (Chunk chunk : chunks) {
+ assertFalse(chunk.getContent().contains("\n\n"));
+ }
+ }
+
+ @Test
+ public void testRealWorldExample() {
+ String text =
+ "Lorem ipsum dolor sit amet, consectetur adipiscing elit.\n\n"
+ + "Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n\n"
+ + "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris.";
+
+ List<Chunk> chunks = strategy.chunk(text, 100, 20);
+
+ assertEquals(3, chunks.size());
+ assertTrue(chunks.get(0).getContent().contains("Lorem ipsum"));
+ assertTrue(chunks.get(1).getContent().contains("Sed do eiusmod"));
+ assertTrue(chunks.get(2).getContent().contains("Ut enim ad minim"));
+ }
+
+ @Test
+ public void packsConsecutiveParagraphsUpToMaxSize() {
+ // The manual promises "packing whole paragraphs up to the chunk size", so paragraphs that
+ // comfortably fit together must not come back as separate chunks.
+ String text = "First paragraph.\n\nSecond paragraph.\n\nThird paragraph.";
+ List<Chunk> chunks = strategy.chunk(text, 1000, 0);
+
+ assertEquals(1, chunks.size());
+ assertEquals(text, chunks.get(0).getContent());
+ assertEquals(0, chunks.get(0).getStartPosition());
+ }
+
+ @Test
+ public void packedChunksStillAddressTheSourceText() {
+ // Separators in the source are preserved, so start/end stay usable as offsets.
+ String text = "One.\n \nTwo.\n\nThree.";
+ for (Chunk chunk : strategy.chunk(text, 1000, 0)) {
+ assertEquals(
+ text.substring(chunk.getStartPosition(), chunk.getEndPosition()), chunk.getContent());
+ }
+ }
+}
diff --git a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/chunking/StructureChunkingStrategyTest.java b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/chunking/StructureChunkingStrategyTest.java
new file mode 100644
index 0000000000..94152ace39
--- /dev/null
+++ b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/chunking/StructureChunkingStrategyTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.chunking;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import org.apache.hop.pipeline.transforms.chunker.Chunk;
+import org.apache.hop.pipeline.transforms.chunker.document.ContentType;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class StructureChunkingStrategyTest {
+
+ private StructureChunkingStrategy strategy;
+
+ @BeforeEach
+ void setUp() {
+ strategy = new StructureChunkingStrategy();
+ }
+
+ @Test
+ void splitsAsciiDocByHeadingsWithBreadcrumb() {
+ String text =
+ "= Working with git\n\n"
+ + "Version control is important.\n\n"
+ + "== File Explorer Toolbar\n\n"
+ + "If git was found, buttons are enabled.\n\n"
+ + "=== Git Configuration options\n\n"
+ + "Global configuration options are available.";
+
+ List<Chunk> chunks = strategy.chunk(text, 500, 50, ContentType.ASCIIDOC);
+
+ assertEquals(3, chunks.size());
+ assertTrue(chunks.get(0).getContent().contains("Working with git"));
+ assertTrue(chunks.get(1).getContent().contains("File Explorer Toolbar"));
+ assertTrue(chunks.get(1).getContent().contains("buttons are enabled"));
+ assertTrue(chunks.get(2).getContent().contains("Git Configuration options"));
+ }
+
+ @Test
+ void fallsBackToCharacterSplitForLargeSection() {
+ StringBuilder body = new StringBuilder();
+ for (int i = 0; i < 200; i++) {
+ body.append("word").append(i).append(' ');
+ }
+ String text = "= Section\n\n" + body;
+
+ List<Chunk> chunks = strategy.chunk(text.toString(), 120, 20, ContentType.ASCIIDOC);
+
+ assertTrue(chunks.size() > 1);
+ assertTrue(chunks.get(0).getContent().contains("Section"));
+ assertTrue(chunks.stream().allMatch(c -> c.getContent().contains("Section")));
+ }
+
+ @Test
+ void plainContentTypeKeepsSingleSection() {
+ String text = "Title line\n\nBody paragraph one.\n\nBody paragraph two.";
+ List<Chunk> chunks = strategy.chunk(text, 500, 0, ContentType.PLAIN);
+ assertEquals(1, chunks.size());
+ assertEquals(text, chunks.get(0).getContent());
+ }
+
+ @Test
+ void strategyTypeIsStructure() {
+ assertEquals(ChunkingStrategyType.STRUCTURE, strategy.getType());
+ }
+
+ @Test
+ void chunksHopPipelineXmlByTransform() {
+ String xml =
+ """
+ <pipeline>
+ <info><name>demo</name></info>
+ <order><hop><from>A</from><to>B</to><enabled>Y</enabled></hop></order>
+ <transform><name>A</name><type>RowGenerator</type></transform>
+ <transform><name>B</name><type>FilterRows</type><compare/></transform>
+ </pipeline>
+ """;
+
+ List<Chunk> chunks = strategy.chunk(xml, 800, 0, ContentType.PIPELINE);
+
+ assertTrue(chunks.size() >= 3);
+ assertTrue(chunks.stream().anyMatch(c -> c.getContent().contains("FilterRows")));
+ assertTrue(chunks.stream().anyMatch(c -> c.getContent().contains("Data flow")));
+ }
+}
diff --git a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistryTest.java b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistryTest.java
new file mode 100644
index 0000000000..1f55987594
--- /dev/null
+++ b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/DocumentParserRegistryTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+class DocumentParserRegistryTest {
+
+ @Test
+ void detectsAsciiDocFromEqualsHeadings() {
+ String text = "= Title\n\nIntro\n\n== Section\n\nBody";
+ assertEquals(ContentType.ASCIIDOC, DocumentParserRegistry.detect(text));
+ }
+
+ @Test
+ void detectsMarkdownFromHashHeadings() {
+ String text = "# Title\n\nIntro\n\n## Section\n\nBody";
+ assertEquals(ContentType.MARKDOWN, DocumentParserRegistry.detect(text));
+ }
+
+ @Test
+ void mapsDocSourceTypeToAsciiDoc() {
+ assertEquals(ContentType.ASCIIDOC, ContentTypeResolver.fromSourceType("doc"));
+ }
+
+ @Test
+ void mapsArticleAndBlogSourceTypeToMarkdown() {
+ assertEquals(ContentType.METADATA, ContentTypeResolver.fromSourceType("metadata"));
+ assertEquals(ContentType.MARKDOWN, ContentTypeResolver.fromSourceType("article"));
+ assertEquals(ContentType.MARKDOWN, ContentTypeResolver.fromSourceType("blog"));
+ assertEquals(ContentType.PLAIN, ContentTypeResolver.fromSourceType("plugin"));
+ }
+
+ @Test
+ void mapsPipelineSourceTypeToPipelineXml() {
+ assertEquals(ContentType.PIPELINE, ContentTypeResolver.fromSourceType("pipeline"));
+ }
+
+ @Test
+ void detectsPipelineXml() {
+ assertEquals(
+ ContentType.PIPELINE,
+ DocumentParserRegistry.detect("<pipeline><info><name>x</name></info></pipeline>"));
+ }
+
+ /**
+ * Regression test: detection used to require a newline before the heading, so a document whose
+ * only heading sat on the first line was classified as plain text and never chunked by section.
+ */
+ @Test
+ void detectsAHeadingOnTheVeryFirstLine() {
+ assertEquals(ContentType.MARKDOWN, DocumentParserRegistry.detect("# Title\n\nSome body text."));
+ assertEquals(ContentType.ASCIIDOC, DocumentParserRegistry.detect("= Title\n\nSome body text."));
+ }
+
+ @Test
+ void detectsAHeadingLaterInTheDocument() {
+ assertEquals(
+ ContentType.MARKDOWN, DocumentParserRegistry.detect("Intro\n\n## Section\n\nbody"));
+ assertEquals(
+ ContentType.ASCIIDOC, DocumentParserRegistry.detect("Intro\n\n== Section\n\nbody"));
+ }
+
+ @Test
+ void doesNotTreatAHashInProseAsAHeading() {
+ assertEquals(ContentType.PLAIN, DocumentParserRegistry.detect("issue #42 was fixed yesterday"));
+ }
+
+ @Test
+ void aHeadingOnTheFirstLineIsChunkedByStructure() {
+ String markdown = "# Title\n\nFirst body.\n\n## Sub\n\nSecond body.";
+ DocumentParser parser = DocumentParserRegistry.parserFor(ContentType.AUTO, markdown);
+
+ assertEquals(ContentType.MARKDOWN, parser.getContentType());
+ assertEquals(2, parser.parse(markdown).flattenSections().size());
+ }
+}
diff --git a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/SecretRedactionTest.java b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/SecretRedactionTest.java
new file mode 100644
index 0000000000..d5686edf62
--- /dev/null
+++ b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/SecretRedactionTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class SecretRedactionTest {
+
+ @ParameterizedTest
+ @ValueSource(
+ strings = {
+ "password",
+ "PASSWORD",
+ "proxyPassword",
+ "client_secret",
+ "API-KEY",
+ "apiKey",
+ "privateKey",
+ "private_key",
+ "accessToken",
+ "passphrase",
+ "credentials"
+ })
+ void recognisesCredentialNames(String name) {
+ assertTrue(SecretRedaction.isSecretName(name), name);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"key", "keyField", "name", "hostname", "databaseName", "lookupKeys"})
+ void leavesOrdinaryFieldNamesAlone(String name) {
+ // A bare "key" is a normal field name in lookups and joins. Redacting it would gut the chunk.
+ assertFalse(SecretRedaction.isSecretName(name), name);
+ }
+
+ @Test
+ void redactsHopEncryptedValuesWhateverTheFieldIsCalled() {
+ assertEquals(
+ SecretRedaction.REDACTED, SecretRedaction.redact("someField", "Encrypted 2be98afc86aa"));
+ }
+
+ @Test
+ void leavesOrdinaryValuesUntouched() {
+ assertEquals("localhost", SecretRedaction.redact("hostname", "localhost"));
+ }
+}
diff --git a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopPipelineXmlParserTest.java b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopPipelineXmlParserTest.java
new file mode 100644
index 0000000000..1b0e7f7074
--- /dev/null
+++ b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopPipelineXmlParserTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document.hopxml;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import org.apache.hop.pipeline.transforms.chunker.document.DocumentNode;
+import org.junit.jupiter.api.Test;
+
+class HopPipelineXmlParserTest {
+
+ private static final String MINIMAL_PIPELINE =
+ """
+ <?xml version="1.0" encoding="UTF-8"?>
+ <pipeline>
+ <info>
+ <name>filter-demo</name>
+ <description>Filters rows by score</description>
+ <parameters>
+ <parameter>
+ <name>THRESHOLD</name>
+ <default_value>30</default_value>
+ <description>Minimum score</description>
+ </parameter>
+ </parameters>
+ </info>
+ <notepads>
+ <notepad><note>Example pipeline for FilterRows</note></notepad>
+ </notepads>
+ <order>
+ <hop><from>Generate</from><to>Filter</to><enabled>Y</enabled></hop>
+ </order>
+ <transform>
+ <name>Filter</name>
+ <type>FilterRows</type>
+ <compare>
+ <condition>
+ <leftvalue>score</leftvalue>
+ <function>></function>
+ <value><text>30</text></value>
+ </condition>
+ </compare>
+ </transform>
+ </pipeline>
+ """;
+
+ @Test
+ void parsesPipelineIntoSections() {
+ DocumentNode root = new HopPipelineXmlParser().parse(MINIMAL_PIPELINE);
+ List<DocumentNode.DocumentSection> sections = root.flattenSections();
+
+ assertTrue(sections.size() >= 4);
+ assertTrue(
+ sections.stream()
+ .anyMatch(
+ s -> s.breadcrumb().contains("Overview") && s.getBody().contains("filter-demo")));
+ assertTrue(sections.stream().anyMatch(s -> s.breadcrumb().contains("Notes")));
+ assertTrue(
+ sections.stream()
+ .anyMatch(
+ s ->
+ s.breadcrumb().contains("Transform: Filter")
+ && s.getBody().contains("FilterRows")));
+ assertTrue(sections.stream().anyMatch(s -> s.breadcrumb().contains("Data flow")));
+ }
+
+ @Test
+ void parsesPipelineWithUnescapedLtInOptionValueAttribute() {
+ String xml =
+ """
+ <pipeline>
+ <info><name>bad-option</name></info>
+ <transform>
+ <name>Kafka Consumer</name>
+ <type>KafkaConsumer</type>
+ <advancedConfig>
+ <option property="filter" value="score<100"/>
+ </advancedConfig>
+ </transform>
+ </pipeline>
+ """;
+
+ DocumentNode root = new HopPipelineXmlParser().parse(xml);
+ List<DocumentNode.DocumentSection> sections = root.flattenSections();
+
+ assertFalse(sections.isEmpty());
+ assertTrue(
+ sections.stream()
+ .anyMatch(s -> s.getBody().contains("bad-option") || s.getBody().contains("score")),
+ () ->
+ sections.stream().map(s -> s.breadcrumb() + " -> " + s.getBody()).toList().toString());
+ }
+
+ @Test
+ void fallsBackToPlainRootWhenXmlIsNotRecoverable() {
+ String xml = "<pipeline><transform><broken";
+
+ DocumentNode root = new HopPipelineXmlParser().parse(xml);
+
+ assertEquals(xml, root.getBody());
+ assertTrue(root.getChildren().isEmpty());
+ }
+}
diff --git a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopWorkflowXmlParserTest.java b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopWorkflowXmlParserTest.java
new file mode 100644
index 0000000000..d84f149373
--- /dev/null
+++ b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopWorkflowXmlParserTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document.hopxml;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import org.apache.hop.pipeline.transforms.chunker.document.DocumentNode;
+import org.junit.jupiter.api.Test;
+
+class HopWorkflowXmlParserTest {
+
+ private static final String MINIMAL_WORKFLOW =
+ """
+ <?xml version="1.0" encoding="UTF-8"?>
+ <workflow>
+ <name>child-wf</name>
+ <description>Child workflow sample</description>
+ <parameters></parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <type>SPECIAL</type>
+ </action>
+ <action>
+ <name>Wait</name>
+ <type>DELAY</type>
+ <maximumTimeout>1</maximumTimeout>
+ </action>
+ </actions>
+ <hops>
+ <hop><from>Start</from><to>Wait</to><enabled>Y</enabled></hop>
+ </hops>
+ </workflow>
+ """;
+
+ @Test
+ void parsesWorkflowIntoSections() {
+ DocumentNode root = new HopWorkflowXmlParser().parse(MINIMAL_WORKFLOW);
+ List<DocumentNode.DocumentSection> sections = root.flattenSections();
+
+ assertTrue(sections.size() >= 4);
+ assertTrue(sections.stream().anyMatch(s -> s.getBody().contains("child-wf")));
+ assertTrue(sections.stream().anyMatch(s -> s.breadcrumb().contains("Action: Start")));
+ assertTrue(sections.stream().anyMatch(s -> s.breadcrumb().contains("Action: Wait")));
+ assertTrue(sections.stream().anyMatch(s -> s.breadcrumb().contains("Control flow")));
+ }
+}
diff --git a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopXmlSupportTest.java b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopXmlSupportTest.java
new file mode 100644
index 0000000000..ef252616f7
--- /dev/null
+++ b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/hopxml/HopXmlSupportTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document.hopxml;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+import org.w3c.dom.Document;
+
+class HopXmlSupportTest {
+
+ @Test
+ void sanitizeEscapesLtInDoubleQuotedAttributeValues() {
+ String raw = "<option property=\"filter\" value=\"score<100\"/>";
+ String sanitized = HopXmlSupport.sanitizeDoubleQuotedAttributes(raw);
+ assertEquals("<option property=\"filter\" value=\"score<100\"/>", sanitized);
+ }
+
+ @Test
+ void parseDocumentRecoversFromUnescapedLtInAttributes() {
+ String xml =
+ """
+ <pipeline>
+ <info><name>demo</name></info>
+ <transform>
+ <name>T</name>
+ <type>Dummy</type>
+ <advancedConfig>
+ <option property="x" value="a<b"/>
+ </advancedConfig>
+ </transform>
+ </pipeline>
+ """;
+
+ Document document = HopXmlSupport.parseDocument(xml);
+
+ assertNotNull(document);
+ assertEquals("pipeline", document.getDocumentElement().getTagName());
+ }
+
+ @Test
+ void serializeConfigRedactsSecretBearingTags() throws Exception {
+ String xml =
+ """
+ <transform>
+ <name>DB</name>
+ <type>TableInput</type>
+ <connection>
+ <hostname>localhost</hostname>
+ <username>hop</username>
+ <password>letmein</password>
+ <client_secret>abc123</client_secret>
+ <lookupKeys>customer_id</lookupKeys>
+ </connection>
+ </transform>
+ """;
+ Document doc = HopXmlSupport.parseDocument(xml);
+ String text = HopXmlSupport.serializeConfig(doc.getDocumentElement());
+
+ assertFalse(text.contains("letmein"), text);
+ assertFalse(text.contains("abc123"), text);
+ // Non-secret content, including a field literally named with "keys", survives.
+ assertTrue(text.contains("localhost"), text);
+ assertTrue(text.contains("customer_id"), text);
+ }
+}
diff --git a/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/metadata/HopMetadataJsonParserTest.java b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/metadata/HopMetadataJsonParserTest.java
new file mode 100644
index 0000000000..c932ce05f3
--- /dev/null
+++ b/plugins/transforms/textchunker/src/test/java/org/apache/hop/pipeline/transforms/chunker/document/metadata/HopMetadataJsonParserTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.chunker.document.metadata;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import org.apache.hop.pipeline.transforms.chunker.document.DocumentNode;
+import org.junit.jupiter.api.Test;
+
+class HopMetadataJsonParserTest {
+
+ private static final String RDBMS_JSON =
+ """
+ {
+ "rdbms": {
+ "H2": {
+ "databaseName": "hop-samples",
+ "pluginId": "H2",
+ "hostname": "localhost",
+ "port": "8082",
+ "username": "hop"
+ }
+ },
+ "name": "hop-samples"
+ }
+ """;
+
+ @Test
+ void parsesRdbmsMetadataIntoSections() {
+ DocumentNode root = new HopMetadataJsonParser().parse(RDBMS_JSON);
+ List<DocumentNode.DocumentSection> sections = root.flattenSections();
+
+ assertTrue(sections.stream().anyMatch(s -> s.breadcrumb().contains("Overview")));
+ assertTrue(
+ sections.stream()
+ .anyMatch(
+ s ->
+ s.breadcrumb().contains("Connection: H2")
+ && s.getBody().contains("hop-samples")));
+ }
+
+ @Test
+ void parsesTextCorpusFormatWithConfigurationBlock() {
+ String text =
+ """
+ Hop metadata item 'hop-samples'
+ Category: rdbms
+ Path: metadata/rdbms/hop-samples.json
+ Configuration:
+ """
+ + RDBMS_JSON.strip();
+
+ DocumentNode root = new HopMetadataJsonParser().parse(text);
+ List<DocumentNode.DocumentSection> sections = root.flattenSections();
+
+ assertTrue(sections.stream().anyMatch(s -> s.breadcrumb().contains("Connection: H2")));
+ }
+
+ @Test
+ void connectionChunksDoNotCarryCredentials() {
+ String json =
+ """
+ {
+ "rdbms": {
+ "PG": {
+ "databaseName": "hop",
+ "pluginId": "POSTGRESQL",
+ "hostname": "localhost",
+ "username": "hop",
+ "password": "Encrypted 2be98afc86aa7f2e4bb18bd63c99dbdde"
+ }
+ },
+ "name": "pg"
+ }
+ """;
+ String all =
+ new HopMetadataJsonParser()
+ .parse(json).flattenSections().stream()
+ .map(section -> String.join(" > ", section.getPath()) + " " + section.getBody())
+ .collect(java.util.stream.Collectors.joining("\n"));
+
+ assertFalse(all.contains("2be98afc86aa7f2e4bb18bd63c99dbdde"), all);
+ assertFalse(all.contains("Encrypted "), all);
+ assertTrue(all.contains("localhost"), all);
+ }
+
+ @Test
+ void genericMetadataChunksDoNotCarryCredentials() {
+ // Not an rdbms connection, so this goes through the generic pretty-printed leaf path.
+ String json =
+ """
+ {
+ "name": "prod-rest",
+ "restConnection": {
+ "baseUrl": "https://api.example.com",
+ "authType": "BEARER",
+ "bearerToken": "tok-abcdef123456",
+ "proxyPassword": "hunter2"
+ }
+ }
+ """;
+ String all =
+ new HopMetadataJsonParser()
+ .parse(json).flattenSections().stream()
+ .map(section -> String.join(" > ", section.getPath()) + " " + section.getBody())
+ .collect(java.util.stream.Collectors.joining("\n"));
+
+ assertFalse(all.contains("tok-abcdef123456"), all);
+ assertFalse(all.contains("hunter2"), all);
+ assertTrue(all.contains("https://api.example.com"), all);
+ }
+}