You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2026/09/25 06:03:21 UTC
(hop) branch main updated: Harden avro code, confirm working engines, fixes #3901 (#8577)
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 633ebf958a Harden avro code, confirm working engines, fixes #3901 (#8577)
633ebf958a is described below
commit 633ebf958a176c06c4354f0d28c6079650028c95
Author: Hans Van Akelyen <ha...@gmail.com>
AuthorDate: Fri Sep 25 08:03:15 2026 +0200
Harden avro code, confirm working engines, fixes #3901 (#8577)
* Harden avro code, confirm working engines, fixes #3901
* fix remark
---
.../pages/pipeline/transforms/avro-decode.adoc | 8 +-
.../pages/pipeline/transforms/avro-encode.adoc | 9 +-
.../pages/pipeline/transforms/avro-file-input.adoc | 10 +-
.../pipeline/transforms/avro-file-output.adoc | 2 +
.../0015-avro-schema-file-create-test-files.hpl | 285 ++++++++++++++++
.../0015-avro-schema-file-read.hpl | 223 +++++++++++++
.../0015-avro-schema-file-validate.hpl | 370 +++++++++++++++++++++
.../0015-avro-schema-file-write.hpl | 188 +++++++++++
.../beam_directrunner/files/avro-3901/test.avsc | 10 +
.../main-0015-avro-schema-file.hwf | 234 +++++++++++++
.../0009-avro-schema-file-create-test-files.hpl | 285 ++++++++++++++++
.../spark-native/0009-avro-schema-file-read.hpl | 223 +++++++++++++
.../0009-avro-schema-file-validate.hpl | 370 +++++++++++++++++++++
.../spark-native/0009-avro-schema-file-write.hpl | 188 +++++++++++
.../spark-native/files/avro-3901/test.avsc | 10 +
.../spark-native/main-0009-avro-schema-file.hwf | 235 +++++++++++++
.../0118-avro-schema-file-create-test-files.hpl | 285 ++++++++++++++++
.../transforms/0118-avro-schema-file-read.hpl | 223 +++++++++++++
.../transforms/0118-avro-schema-file-validate.hpl | 370 +++++++++++++++++++++
.../transforms/0118-avro-schema-file-write.hpl | 188 +++++++++++
.../transforms/files/avro-3901/test.avsc | 10 +
.../transforms/main-0118-avro-schema-file.hwf | 234 +++++++++++++
.../avro/transforms/avrodecode/AvroDecodeMeta.java | 3 +-
.../avro/transforms/avroencode/AvroEncodeMeta.java | 3 +-
.../transforms/avroinput/AvroFileInputMeta.java | 3 +-
.../hop/avro/transforms/avrooutput/AvroOutput.java | 45 ++-
.../transforms/avrooutput/AvroOutputDialog.java | 8 +-
.../avrooutput/messages/messages_en_US.properties | 3 +
.../avrooutput/AvroOutputSchemaFileTest.java | 272 +++++++++++++++
pom.xml | 1 +
30 files changed, 4275 insertions(+), 23 deletions(-)
diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-decode.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-decode.adoc
index abf505d4cb..630db9a15a 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-decode.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-decode.adoc
@@ -17,7 +17,7 @@ under the License.
:documentationPath: /pipeline/transforms/
:language: en_US
:description: The Avro decode transform allows you to decode an Avro field and convert it to Hop fields.
-:page-engines: Hop Engine=yes, Single Threaded=yes, Native Spark=yes, Beam Spark=yes, Beam Flink=yes, Beam Dataflow=yes
+:page-engines: Hop Engine=yes, Single Threaded=yes, Native Spark=no, Beam Spark=yes, Beam Flink=yes, Beam Dataflow=yes
:page-marketplace: hop-tech-avro
= image:transforms/icons/avro_decode.svg[Avro decode Icon, role="image-doc-icon"] Avro decode
@@ -26,6 +26,12 @@ under the License.
The Avro decode transform allows you to decode an Avro field and convert it to Hop fields.
+[WARNING]
+====
+Avro decode does not run on the native Spark engine.
+It decodes an Avro Record field, and the native Spark engine has no Spark data type to carry such a field from one transform to the next, so the pipeline is refused before it starts.
+====
+
== Options
[options="header"]
diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-encode.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-encode.adoc
index bb407a1dbf..073e64eddd 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-encode.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-encode.adoc
@@ -17,7 +17,7 @@ under the License.
:documentationPath: /pipeline/transforms/
:language: en_US
:description: The Avro encode transform allows you to encode a new Avro field using a selection of Hop fields.
-:page-engines: Hop Engine=yes, Single Threaded=yes, Native Spark=yes, Beam Spark=yes, Beam Flink=yes, Beam Dataflow=yes
+:page-engines: Hop Engine=yes, Single Threaded=yes, Native Spark=no, Beam Spark=yes, Beam Flink=yes, Beam Dataflow=yes
:page-marketplace: hop-tech-avro
= image:transforms/icons/avro_encode.svg[Avro encode Icon, role="image-doc-icon"] Avro encode
@@ -26,6 +26,13 @@ under the License.
The Avro encode transform allows you to encode a new Avro Record field using a selection of Hop fields. The Avro schema will be part of the value metadata for this Avro Record field.
+[WARNING]
+====
+Avro encode does not run on the native Spark engine.
+The native Spark engine has no Spark data type to carry the Avro Record field this transform produces from one transform to the next, so the pipeline is refused before it starts.
+To write Avro on the native Spark engine, use the xref:pipeline/transforms/avro-file-output.adoc[Avro file output] transform.
+====
+
== Options
diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-input.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-input.adoc
index 9d54360131..9b5dd36859 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-input.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-input.adoc
@@ -17,7 +17,7 @@ under the License.
:documentationPath: /pipeline/transforms/
:language: en_US
:description: The Avro file input transform simply reads Avro records from one or more files. Each record is encapsulated in an Avro field, each value has its own Schema and record.
-:page-engines: Hop Engine=yes, Single Threaded=yes, Native Spark=yes, Beam Spark=yes, Beam Flink=yes, Beam Dataflow=yes
+:page-engines: Hop Engine=yes, Single Threaded=yes, Native Spark=no, Beam Spark=yes, Beam Flink=yes, Beam Dataflow=yes
:page-marketplace: hop-tech-avro
= image:transforms/icons/avro_input.svg[Avro file input Icon, role="image-doc-icon"] Avro file input
@@ -26,6 +26,14 @@ under the License.
The Avro file input transform simply reads Avro records from one or more files.
+[WARNING]
+====
+Avro file input does not run on the native Spark engine.
+It produces an Avro Record field, and the native Spark engine has no Spark data type to carry that field from one transform to the next, so the pipeline is refused before it starts.
+
+Run the pipeline that reads the Avro files on the Hop engine or on a Beam engine instead.
+====
+
Each record is encapsulated in an Avro field, each value has its own Schema and record.
== Options
diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-output.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-output.adoc
index 00e0c2331a..5d4816e416 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-output.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-output.adoc
@@ -63,6 +63,8 @@ Note: This name has to be unique in a single pipeline.
|Schema filename
|Filepath to write or read the schema from.
+The path can contain variables, for example `${PROJECT_HOME}/schemas/customer.avsc`, and can point to any file system Hop supports (local, `s3://`, `azure://`, `gs://`, ...).
+When *Automatically create schema* is disabled, the transform reads the schema from this file before it writes the first row, and fails if the file can't be read or isn't a valid Avro schema.
|Create parent folder
|When creating a file, select this if you want to create the parent folder.
diff --git a/integration-tests/beam_directrunner/0015-avro-schema-file-create-test-files.hpl b/integration-tests/beam_directrunner/0015-avro-schema-file-create-test-files.hpl
new file mode 100644
index 0000000000..f56c061b13
--- /dev/null
+++ b/integration-tests/beam_directrunner/0015-avro-schema-file-create-test-files.hpl
@@ -0,0 +1,285 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<pipeline>
+ <info>
+ <name>0015-avro-schema-file-create-test-files</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>100 rows</from>
+ <to>id</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>id</from>
+ <to>format id</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>format id</from>
+ <to>test</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>test</from>
+ <to>${java.io.tmpdir}/0015-avro-schema-file/source/test-file</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>100 rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>prefix</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif>test</nullif>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <limit>100</limit>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field>now</row_time_field>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>id</name>
+ <type>Sequence</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <valuename>id</valuename>
+ <use_database>N</use_database>
+ <connection/>
+ <schema/>
+ <seqname>SEQ_</seqname>
+ <use_counter>Y</use_counter>
+ <counter_name/>
+ <start_at>1</start_at>
+ <increment_by>1</increment_by>
+ <max_value>999999999</max_value>
+ <attributes/>
+ <GUI>
+ <xloc>240</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <type>SelectValues</type>
+ <name>format id</name>
+ <fields>
+ <select_unspecified>N</select_unspecified>
+ <meta>
+ <name>id</name>
+ <rename>id</rename>
+ <type>String</type>
+ <length>-2</length>
+ <precision>0</precision>
+ <storage_type/>
+ <conversion_mask>000</conversion_mask>
+ <date_format_lenient>N</date_format_lenient>
+ <date_format_locale/>
+ <date_format_timezone/>
+ <lenient_string_to_number>N</lenient_string_to_number>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <currency_symbol/>
+ <encoding/>
+ <roundingType>half_even</roundingType>
+ </meta>
+ </fields>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>ConcatFields</type>
+ <name>test</name>
+ <separator>-</separator>
+ <enclosure/>
+ <force_enclosure>N</force_enclosure>
+ <fields>
+ <field>
+ <name>prefix</name>
+ <type>String</type>
+ <format/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type/>
+ </field>
+ <field>
+ <name>id</name>
+ <type>String</type>
+ <format/>
+ <length>3</length>
+ <precision>0</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type/>
+ </field>
+ </fields>
+ <ConcatFields>
+ <targetFieldName>test</targetFieldName>
+ <targetFieldLength>100</targetFieldLength>
+ <removeSelectedFields>N</removeSelectedFields>
+ </ConcatFields>
+ <skip_value_empty>N</skip_value_empty>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>528</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <name>${java.io.tmpdir}/0015-avro-schema-file/source/test-file</name>
+ <type>TextFileOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_forced>N</enclosure_forced>
+ <enclosure_fix_disabled>N</enclosure_fix_disabled>
+ <header>Y</header>
+ <footer>N</footer>
+ <format>UNIX</format>
+ <compression>None</compression>
+ <encoding>UTF-8</encoding>
+ <endedLine/>
+ <fileNameInField>N</fileNameInField>
+ <fileNameField/>
+ <create_parent_folder>Y</create_parent_folder>
+ <file>
+ <name>${java.io.tmpdir}/0015-avro-schema-file/source/test-file</name>
+ <do_not_open_new_file_init>Y</do_not_open_new_file_init>
+ <extension>csv</extension>
+ <append>N</append>
+ <split>Y</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <pad>N</pad>
+ <fast_dump>N</fast_dump>
+ <splitevery/>
+ </file>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <roundingType>half_even</roundingType>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>test</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <roundingType>half_even</roundingType>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>672</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling/>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/beam_directrunner/0015-avro-schema-file-read.hpl b/integration-tests/beam_directrunner/0015-avro-schema-file-read.hpl
new file mode 100644
index 0000000000..83271a412d
--- /dev/null
+++ b/integration-tests/beam_directrunner/0015-avro-schema-file-read.hpl
@@ -0,0 +1,223 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<pipeline>
+ <info>
+ <name>0015-avro-schema-file-read</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>avro/*.avro</from>
+ <to>Avro File Input</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Avro File Input</from>
+ <to>Avro Decode</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Avro Decode</from>
+ <to>readback/test-file</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>avro/*.avro</name>
+ <type>GetFileNames</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <file>
+ <name>${java.io.tmpdir}/0015-avro-schema-file/avro/</name>
+ <filemask>.*\.avro</filemask>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <filter>
+ <filterfiletype>all_files</filterfiletype>
+ </filter>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <filename_Field/>
+ <wildcard_Field/>
+ <exclude_wildcard_Field/>
+ <filefield>N</filefield>
+ <dynamic_include_subfolders>N</dynamic_include_subfolders>
+ <isaddresult>N</isaddresult>
+ <limit>0</limit>
+ <doNotFailIfNoFile>N</doNotFailIfNoFile>
+ <raiseAnExceptionIfNoFile>Y</raiseAnExceptionIfNoFile>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Avro File Input</name>
+ <type>AvroFileInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data_filename_field>filename</data_filename_field>
+ <output_field>avro</output_field>
+ <rows_limit/>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Avro Decode</name>
+ <type>AvroDecode</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <source_field>avro</source_field>
+ <remove_source_field>Y</remove_source_field>
+ <ignore_missing>N</ignore_missing>
+ <fields>
+ <field>
+ <source_field>id</source_field>
+ <source_avro_type>Union</source_avro_type>
+ <target_field_name>id</target_field_name>
+ <target_type>Integer</target_type>
+ </field>
+ <field>
+ <source_field>test</source_field>
+ <source_avro_type>Union</source_avro_type>
+ <target_field_name>test</target_field_name>
+ <target_type>String</target_type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>448</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>readback/test-file</name>
+ <type>TextFileOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_forced>N</enclosure_forced>
+ <enclosure_fix_disabled>N</enclosure_fix_disabled>
+ <header>Y</header>
+ <footer>N</footer>
+ <format>UNIX</format>
+ <compression>None</compression>
+ <encoding>UTF-8</encoding>
+ <endedLine/>
+ <fileNameInField>N</fileNameInField>
+ <fileNameField/>
+ <create_parent_folder>Y</create_parent_folder>
+ <file>
+ <name>${java.io.tmpdir}/0015-avro-schema-file/readback/test-file</name>
+ <do_not_open_new_file_init>Y</do_not_open_new_file_init>
+ <extension>csv</extension>
+ <append>N</append>
+ <split>Y</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <pad>N</pad>
+ <fast_dump>N</fast_dump>
+ <splitevery/>
+ </file>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <roundingType>half_even</roundingType>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>test</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <roundingType>half_even</roundingType>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>624</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling/>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/beam_directrunner/0015-avro-schema-file-validate.hpl b/integration-tests/beam_directrunner/0015-avro-schema-file-validate.hpl
new file mode 100644
index 0000000000..96e2c1f799
--- /dev/null
+++ b/integration-tests/beam_directrunner/0015-avro-schema-file-validate.hpl
@@ -0,0 +1,370 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<pipeline>
+ <info>
+ <name>0015-avro-schema-file-validate</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>readback/*.csv</from>
+ <to>read readback</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>read readback</from>
+ <to>counts</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>counts</from>
+ <to>Data validator</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Data validator</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>readback/*.csv</name>
+ <type>GetFileNames</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <file>
+ <name>${java.io.tmpdir}/0015-avro-schema-file/readback/</name>
+ <filemask>.*\.csv</filemask>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <filter>
+ <filterfiletype>all_files</filterfiletype>
+ </filter>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <filename_Field/>
+ <wildcard_Field/>
+ <exclude_wildcard_Field/>
+ <filefield>N</filefield>
+ <dynamic_include_subfolders>N</dynamic_include_subfolders>
+ <isaddresult>N</isaddresult>
+ <limit>0</limit>
+ <doNotFailIfNoFile>N</doNotFailIfNoFile>
+ <raiseAnExceptionIfNoFile>Y</raiseAnExceptionIfNoFile>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>read readback</name>
+ <type>TextFileInput2</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <accept_filenames>Y</accept_filenames>
+ <passing_through_fields>N</passing_through_fields>
+ <accept_field>filename</accept_field>
+ <accept_transform_name>readback/*.csv</accept_transform_name>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_breaks>N</enclosure_breaks>
+ <escapechar/>
+ <header>Y</header>
+ <nr_headerlines>1</nr_headerlines>
+ <footer>N</footer>
+ <nr_footerlines>1</nr_footerlines>
+ <line_wrapped>N</line_wrapped>
+ <nr_wraps>1</nr_wraps>
+ <layout_paged>N</layout_paged>
+ <nr_lines_per_page>80</nr_lines_per_page>
+ <nr_lines_doc_header>0</nr_lines_doc_header>
+ <noempty>Y</noempty>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <rownumByFile>N</rownumByFile>
+ <rownum_field/>
+ <format>Unix</format>
+ <encoding>UTF-8</encoding>
+ <length>Characters</length>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <file_type>CSV</file_type>
+ <compression>None</compression>
+ <filters>
+ </filters>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>both</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>test</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>both</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <error_ignored>N</error_ignored>
+ <skip_bad_files>N</skip_bad_files>
+ <error_line_skipped>N</error_line_skipped>
+ <date_format_lenient>Y</date_format_lenient>
+ <date_format_locale>en_US</date_format_locale>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>counts</name>
+ <type>MemoryGroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <give_back_row>N</give_back_row>
+ <group>
+ </group>
+ <fields>
+ <field>
+ <aggregate>nrRows</aggregate>
+ <subject>id</subject>
+ <type>COUNT_ALL</type>
+ <valuefield/>
+ </field>
+ <field>
+ <aggregate>countDistinctId</aggregate>
+ <subject>id</subject>
+ <type>COUNT_DISTINCT</type>
+ <valuefield/>
+ </field>
+ <field>
+ <aggregate>countDistinctTest</aggregate>
+ <subject>test</subject>
+ <type>COUNT_DISTINCT</type>
+ <valuefield/>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Data validator</name>
+ <type>Validator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <validator_field>
+ <validation_name>rows-100</validation_name>
+ <name>nrRows</name>
+ <max_length/>
+ <min_length/>
+ <null_allowed>N</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <data_type>Integer</data_type>
+ <data_type_verified>Y</data_type_verified>
+ <conversion_mask>0</conversion_mask>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <is_sourcing_values>N</is_sourcing_values>
+ <sourcing_transform/>
+ <sourcing_field/>
+ <min_value/>
+ <max_value/>
+ <start_string/>
+ <start_string_not_allowed/>
+ <end_string/>
+ <end_string_not_allowed/>
+ <regular_expression/>
+ <regular_expression_not_allowed/>
+ <error_code/>
+ <error_description/>
+ <allowed_value>
+ <value>100</value>
+ </allowed_value>
+ </validator_field>
+ <validator_field>
+ <validation_name>unique-id-100</validation_name>
+ <name>countDistinctId</name>
+ <max_length/>
+ <min_length/>
+ <null_allowed>N</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <data_type>Integer</data_type>
+ <data_type_verified>Y</data_type_verified>
+ <conversion_mask>0</conversion_mask>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <is_sourcing_values>N</is_sourcing_values>
+ <sourcing_transform/>
+ <sourcing_field/>
+ <min_value/>
+ <max_value/>
+ <start_string/>
+ <start_string_not_allowed/>
+ <end_string/>
+ <end_string_not_allowed/>
+ <regular_expression/>
+ <regular_expression_not_allowed/>
+ <error_code/>
+ <error_description/>
+ <allowed_value>
+ <value>100</value>
+ </allowed_value>
+ </validator_field>
+ <validator_field>
+ <validation_name>unique-test-100</validation_name>
+ <name>countDistinctTest</name>
+ <max_length/>
+ <min_length/>
+ <null_allowed>N</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <data_type>Integer</data_type>
+ <data_type_verified>Y</data_type_verified>
+ <conversion_mask>0</conversion_mask>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <is_sourcing_values>N</is_sourcing_values>
+ <sourcing_transform/>
+ <sourcing_field/>
+ <min_value/>
+ <max_value/>
+ <start_string/>
+ <start_string_not_allowed/>
+ <end_string/>
+ <end_string_not_allowed/>
+ <regular_expression/>
+ <regular_expression_not_allowed/>
+ <error_code/>
+ <error_description/>
+ <allowed_value>
+ <value>100</value>
+ </allowed_value>
+ </validator_field>
+ <validate_all>Y</validate_all>
+ <concat_errors>Y</concat_errors>
+ <concat_separator>, </concat_separator>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <row_threshold>0</row_threshold>
+ <message>The Avro round trip did not return exactly 100 unique rows</message>
+ <always_log_rows>Y</always_log_rows>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ <error>
+ <source_transform>Data validator</source_transform>
+ <target_transform>Abort</target_transform>
+ <is_enabled>Y</is_enabled>
+ <nr_valuename>error_row</nr_valuename>
+ <descriptions_valuename>error_description</descriptions_valuename>
+ <fields_valuename/>
+ <codes_valuename>error_code</codes_valuename>
+ <max_errors/>
+ <max_pct_errors/>
+ <min_pct_rows/>
+ </error>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/beam_directrunner/0015-avro-schema-file-write.hpl b/integration-tests/beam_directrunner/0015-avro-schema-file-write.hpl
new file mode 100644
index 0000000000..6e3207866d
--- /dev/null
+++ b/integration-tests/beam_directrunner/0015-avro-schema-file-write.hpl
@@ -0,0 +1,188 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<pipeline>
+ <info>
+ <name>0015-avro-schema-file-write</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>source/test-file_0.csv</from>
+ <to>avro/test.avro (schema from project)</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>source/test-file_0.csv</name>
+ <type>TextFileInput2</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <accept_filenames>N</accept_filenames>
+ <files>
+ <file>
+ <name>${java.io.tmpdir}/0015-avro-schema-file/source/test-file_0.csv</name>
+ <filemask/>
+ <exclude_filemask/>
+ <file_required>Y</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ </files>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_breaks>N</enclosure_breaks>
+ <escapechar/>
+ <header>Y</header>
+ <nr_headerlines>1</nr_headerlines>
+ <footer>N</footer>
+ <nr_footerlines>1</nr_footerlines>
+ <line_wrapped>N</line_wrapped>
+ <nr_wraps>1</nr_wraps>
+ <layout_paged>N</layout_paged>
+ <nr_lines_per_page>80</nr_lines_per_page>
+ <nr_lines_doc_header>0</nr_lines_doc_header>
+ <noempty>Y</noempty>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <rownumByFile>N</rownumByFile>
+ <rownum_field/>
+ <format>Unix</format>
+ <encoding>UTF-8</encoding>
+ <length>Characters</length>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <file_type>CSV</file_type>
+ <compression>None</compression>
+ <filters>
+ </filters>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>both</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>test</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>both</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <error_ignored>N</error_ignored>
+ <skip_bad_files>N</skip_bad_files>
+ <error_line_skipped>N</error_line_skipped>
+ <date_format_lenient>Y</date_format_lenient>
+ <date_format_locale>en_US</date_format_locale>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>avro/test.avro (schema from project)</name>
+ <type>AvroOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <output_type>BinaryFile</output_type>
+ <filename>${java.io.tmpdir}/0015-avro-schema-file/avro/test.avro</filename>
+ <output_field_name/>
+ <create_schema_file>N</create_schema_file>
+ <write_schema_file>N</write_schema_file>
+ <schemafilename>${PROJECT_HOME}/files/avro-3901/test.avsc</schemafilename>
+ <namespace/>
+ <recordname/>
+ <doc/>
+ <compressiontype>none</compressiontype>
+ <create_parent_folder>Y</create_parent_folder>
+ <split>N</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <fields>
+ <field>
+ <name>id</name>
+ <avroname>$.id</avroname>
+ <avrotype>5</avrotype>
+ <nullable>Y</nullable>
+ </field>
+ <field>
+ <name>test</name>
+ <avroname>$.test</avroname>
+ <avrotype>6</avrotype>
+ <nullable>Y</nullable>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>528</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling/>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/beam_directrunner/files/avro-3901/test.avsc b/integration-tests/beam_directrunner/files/avro-3901/test.avsc
new file mode 100644
index 0000000000..f391cf3a9a
--- /dev/null
+++ b/integration-tests/beam_directrunner/files/avro-3901/test.avsc
@@ -0,0 +1,10 @@
+{
+ "type": "record",
+ "name": "test",
+ "namespace": "org.apache.hop.it",
+ "doc": "Schema read from a project folder through a variable, issue #3901",
+ "fields": [
+ {"name": "id", "type": ["null", "long"], "default": null},
+ {"name": "test", "type": ["null", "string"], "default": null}
+ ]
+}
diff --git a/integration-tests/beam_directrunner/main-0015-avro-schema-file.hwf b/integration-tests/beam_directrunner/main-0015-avro-schema-file.hwf
new file mode 100644
index 0000000000..19bcbc0917
--- /dev/null
+++ b/integration-tests/beam_directrunner/main-0015-avro-schema-file.hwf
@@ -0,0 +1,234 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<workflow>
+ <name>main-0015-avro-schema-file</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>80</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>delete output</name>
+ <description/>
+ <type>DELETE_FOLDERS</type>
+ <attributes/>
+ <arg_from_previous>N</arg_from_previous>
+ <success_condition>success_if_no_errors</success_condition>
+ <limit_folders>10</limit_folders>
+ <fields>
+ <field>
+ <name>${java.io.tmpdir}/0015-avro-schema-file</name>
+ </field>
+ </fields>
+ <parallel>N</parallel>
+ <xloc>224</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0015-avro-schema-file-create-test-files</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0015-avro-schema-file-create-test-files.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>hop-local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>384</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0015-avro-schema-file-write</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0015-avro-schema-file-write.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>608</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0015-avro-schema-file-read</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0015-avro-schema-file-read.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>832</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0015-avro-schema-file-validate</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0015-avro-schema-file-validate.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>hop-local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>1056</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>delete output</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>delete output</from>
+ <to>0015-avro-schema-file-create-test-files</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0015-avro-schema-file-create-test-files</from>
+ <to>0015-avro-schema-file-write</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0015-avro-schema-file-write</from>
+ <to>0015-avro-schema-file-read</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0015-avro-schema-file-read</from>
+ <to>0015-avro-schema-file-validate</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ <notepad>
+ <backgroundcolorblue>251</backgroundcolorblue>
+ <backgroundcolorgreen>232</backgroundcolorgreen>
+ <backgroundcolorred>201</backgroundcolorred>
+ <bordercolorblue>90</bordercolorblue>
+ <bordercolorgreen>58</bordercolorgreen>
+ <bordercolorred>14</bordercolorred>
+ <fontbold>N</fontbold>
+ <fontcolorblue>90</fontcolorblue>
+ <fontcolorgreen>58</fontcolorgreen>
+ <fontcolorred>14</fontcolorred>
+ <fontitalic>N</fontitalic>
+ <fontname>Noto Sans</fontname>
+ <fontsize>10</fontsize>
+ <height>32</height>
+ <xloc>80</xloc>
+ <yloc>176</yloc>
+ <note>Issue #3901: Avro File Output reads its schema from '${PROJECT_HOME}/files/avro-3901/test.avsc'.
+Create 100 rows locally, write them to Avro and read them back with Avro File Input + Avro Decode on the Beam Direct runner,
+then verify locally that exactly 100 unique rows survived the round trip.</note>
+ <width>32</width>
+ </notepad>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/spark-native/0009-avro-schema-file-create-test-files.hpl b/integration-tests/spark-native/0009-avro-schema-file-create-test-files.hpl
new file mode 100644
index 0000000000..1745973fca
--- /dev/null
+++ b/integration-tests/spark-native/0009-avro-schema-file-create-test-files.hpl
@@ -0,0 +1,285 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<pipeline>
+ <info>
+ <name>0009-avro-schema-file-create-test-files</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>100 rows</from>
+ <to>id</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>id</from>
+ <to>format id</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>format id</from>
+ <to>test</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>test</from>
+ <to>${PROJECT_HOME}/output/0009-avro-schema-file/source/test-file</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>100 rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>prefix</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif>test</nullif>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <limit>100</limit>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field>now</row_time_field>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>id</name>
+ <type>Sequence</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <valuename>id</valuename>
+ <use_database>N</use_database>
+ <connection/>
+ <schema/>
+ <seqname>SEQ_</seqname>
+ <use_counter>Y</use_counter>
+ <counter_name/>
+ <start_at>1</start_at>
+ <increment_by>1</increment_by>
+ <max_value>999999999</max_value>
+ <attributes/>
+ <GUI>
+ <xloc>240</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <type>SelectValues</type>
+ <name>format id</name>
+ <fields>
+ <select_unspecified>N</select_unspecified>
+ <meta>
+ <name>id</name>
+ <rename>id</rename>
+ <type>String</type>
+ <length>-2</length>
+ <precision>0</precision>
+ <storage_type/>
+ <conversion_mask>000</conversion_mask>
+ <date_format_lenient>N</date_format_lenient>
+ <date_format_locale/>
+ <date_format_timezone/>
+ <lenient_string_to_number>N</lenient_string_to_number>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <currency_symbol/>
+ <encoding/>
+ <roundingType>half_even</roundingType>
+ </meta>
+ </fields>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>ConcatFields</type>
+ <name>test</name>
+ <separator>-</separator>
+ <enclosure/>
+ <force_enclosure>N</force_enclosure>
+ <fields>
+ <field>
+ <name>prefix</name>
+ <type>String</type>
+ <format/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type/>
+ </field>
+ <field>
+ <name>id</name>
+ <type>String</type>
+ <format/>
+ <length>3</length>
+ <precision>0</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type/>
+ </field>
+ </fields>
+ <ConcatFields>
+ <targetFieldName>test</targetFieldName>
+ <targetFieldLength>100</targetFieldLength>
+ <removeSelectedFields>N</removeSelectedFields>
+ </ConcatFields>
+ <skip_value_empty>N</skip_value_empty>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>528</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <name>${PROJECT_HOME}/output/0009-avro-schema-file/source/test-file</name>
+ <type>TextFileOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_forced>N</enclosure_forced>
+ <enclosure_fix_disabled>N</enclosure_fix_disabled>
+ <header>Y</header>
+ <footer>N</footer>
+ <format>UNIX</format>
+ <compression>None</compression>
+ <encoding>UTF-8</encoding>
+ <endedLine/>
+ <fileNameInField>N</fileNameInField>
+ <fileNameField/>
+ <create_parent_folder>Y</create_parent_folder>
+ <file>
+ <name>${PROJECT_HOME}/output/0009-avro-schema-file/source/test-file</name>
+ <do_not_open_new_file_init>Y</do_not_open_new_file_init>
+ <extension>csv</extension>
+ <append>N</append>
+ <split>Y</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <pad>N</pad>
+ <fast_dump>N</fast_dump>
+ <splitevery/>
+ </file>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <roundingType>half_even</roundingType>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>test</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <roundingType>half_even</roundingType>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>672</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling/>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/spark-native/0009-avro-schema-file-read.hpl b/integration-tests/spark-native/0009-avro-schema-file-read.hpl
new file mode 100644
index 0000000000..bbfc4856c0
--- /dev/null
+++ b/integration-tests/spark-native/0009-avro-schema-file-read.hpl
@@ -0,0 +1,223 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<pipeline>
+ <info>
+ <name>0009-avro-schema-file-read</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>avro/*.avro</from>
+ <to>Avro File Input</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Avro File Input</from>
+ <to>Avro Decode</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Avro Decode</from>
+ <to>readback/test-file</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>avro/*.avro</name>
+ <type>GetFileNames</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <file>
+ <name>${PROJECT_HOME}/output/0009-avro-schema-file/avro/</name>
+ <filemask>.*\.avro</filemask>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <filter>
+ <filterfiletype>all_files</filterfiletype>
+ </filter>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <filename_Field/>
+ <wildcard_Field/>
+ <exclude_wildcard_Field/>
+ <filefield>N</filefield>
+ <dynamic_include_subfolders>N</dynamic_include_subfolders>
+ <isaddresult>N</isaddresult>
+ <limit>0</limit>
+ <doNotFailIfNoFile>N</doNotFailIfNoFile>
+ <raiseAnExceptionIfNoFile>Y</raiseAnExceptionIfNoFile>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Avro File Input</name>
+ <type>AvroFileInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data_filename_field>filename</data_filename_field>
+ <output_field>avro</output_field>
+ <rows_limit/>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Avro Decode</name>
+ <type>AvroDecode</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <source_field>avro</source_field>
+ <remove_source_field>Y</remove_source_field>
+ <ignore_missing>N</ignore_missing>
+ <fields>
+ <field>
+ <source_field>id</source_field>
+ <source_avro_type>Union</source_avro_type>
+ <target_field_name>id</target_field_name>
+ <target_type>Integer</target_type>
+ </field>
+ <field>
+ <source_field>test</source_field>
+ <source_avro_type>Union</source_avro_type>
+ <target_field_name>test</target_field_name>
+ <target_type>String</target_type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>448</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>readback/test-file</name>
+ <type>TextFileOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_forced>N</enclosure_forced>
+ <enclosure_fix_disabled>N</enclosure_fix_disabled>
+ <header>Y</header>
+ <footer>N</footer>
+ <format>UNIX</format>
+ <compression>None</compression>
+ <encoding>UTF-8</encoding>
+ <endedLine/>
+ <fileNameInField>N</fileNameInField>
+ <fileNameField/>
+ <create_parent_folder>Y</create_parent_folder>
+ <file>
+ <name>${PROJECT_HOME}/output/0009-avro-schema-file/readback/test-file</name>
+ <do_not_open_new_file_init>Y</do_not_open_new_file_init>
+ <extension>csv</extension>
+ <append>N</append>
+ <split>Y</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <pad>N</pad>
+ <fast_dump>N</fast_dump>
+ <splitevery/>
+ </file>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <roundingType>half_even</roundingType>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>test</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <roundingType>half_even</roundingType>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>624</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling/>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/spark-native/0009-avro-schema-file-validate.hpl b/integration-tests/spark-native/0009-avro-schema-file-validate.hpl
new file mode 100644
index 0000000000..dd255b460c
--- /dev/null
+++ b/integration-tests/spark-native/0009-avro-schema-file-validate.hpl
@@ -0,0 +1,370 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<pipeline>
+ <info>
+ <name>0009-avro-schema-file-validate</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>readback/*.csv</from>
+ <to>read readback</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>read readback</from>
+ <to>counts</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>counts</from>
+ <to>Data validator</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Data validator</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>readback/*.csv</name>
+ <type>GetFileNames</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <file>
+ <name>${PROJECT_HOME}/output/0009-avro-schema-file/readback/</name>
+ <filemask>.*\.csv</filemask>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <filter>
+ <filterfiletype>all_files</filterfiletype>
+ </filter>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <filename_Field/>
+ <wildcard_Field/>
+ <exclude_wildcard_Field/>
+ <filefield>N</filefield>
+ <dynamic_include_subfolders>N</dynamic_include_subfolders>
+ <isaddresult>N</isaddresult>
+ <limit>0</limit>
+ <doNotFailIfNoFile>N</doNotFailIfNoFile>
+ <raiseAnExceptionIfNoFile>Y</raiseAnExceptionIfNoFile>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>read readback</name>
+ <type>TextFileInput2</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <accept_filenames>Y</accept_filenames>
+ <passing_through_fields>N</passing_through_fields>
+ <accept_field>filename</accept_field>
+ <accept_transform_name>readback/*.csv</accept_transform_name>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_breaks>N</enclosure_breaks>
+ <escapechar/>
+ <header>Y</header>
+ <nr_headerlines>1</nr_headerlines>
+ <footer>N</footer>
+ <nr_footerlines>1</nr_footerlines>
+ <line_wrapped>N</line_wrapped>
+ <nr_wraps>1</nr_wraps>
+ <layout_paged>N</layout_paged>
+ <nr_lines_per_page>80</nr_lines_per_page>
+ <nr_lines_doc_header>0</nr_lines_doc_header>
+ <noempty>Y</noempty>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <rownumByFile>N</rownumByFile>
+ <rownum_field/>
+ <format>Unix</format>
+ <encoding>UTF-8</encoding>
+ <length>Characters</length>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <file_type>CSV</file_type>
+ <compression>None</compression>
+ <filters>
+ </filters>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>both</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>test</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>both</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <error_ignored>N</error_ignored>
+ <skip_bad_files>N</skip_bad_files>
+ <error_line_skipped>N</error_line_skipped>
+ <date_format_lenient>Y</date_format_lenient>
+ <date_format_locale>en_US</date_format_locale>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>counts</name>
+ <type>MemoryGroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <give_back_row>N</give_back_row>
+ <group>
+ </group>
+ <fields>
+ <field>
+ <aggregate>nrRows</aggregate>
+ <subject>id</subject>
+ <type>COUNT_ALL</type>
+ <valuefield/>
+ </field>
+ <field>
+ <aggregate>countDistinctId</aggregate>
+ <subject>id</subject>
+ <type>COUNT_DISTINCT</type>
+ <valuefield/>
+ </field>
+ <field>
+ <aggregate>countDistinctTest</aggregate>
+ <subject>test</subject>
+ <type>COUNT_DISTINCT</type>
+ <valuefield/>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Data validator</name>
+ <type>Validator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <validator_field>
+ <validation_name>rows-100</validation_name>
+ <name>nrRows</name>
+ <max_length/>
+ <min_length/>
+ <null_allowed>N</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <data_type>Integer</data_type>
+ <data_type_verified>Y</data_type_verified>
+ <conversion_mask>0</conversion_mask>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <is_sourcing_values>N</is_sourcing_values>
+ <sourcing_transform/>
+ <sourcing_field/>
+ <min_value/>
+ <max_value/>
+ <start_string/>
+ <start_string_not_allowed/>
+ <end_string/>
+ <end_string_not_allowed/>
+ <regular_expression/>
+ <regular_expression_not_allowed/>
+ <error_code/>
+ <error_description/>
+ <allowed_value>
+ <value>100</value>
+ </allowed_value>
+ </validator_field>
+ <validator_field>
+ <validation_name>unique-id-100</validation_name>
+ <name>countDistinctId</name>
+ <max_length/>
+ <min_length/>
+ <null_allowed>N</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <data_type>Integer</data_type>
+ <data_type_verified>Y</data_type_verified>
+ <conversion_mask>0</conversion_mask>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <is_sourcing_values>N</is_sourcing_values>
+ <sourcing_transform/>
+ <sourcing_field/>
+ <min_value/>
+ <max_value/>
+ <start_string/>
+ <start_string_not_allowed/>
+ <end_string/>
+ <end_string_not_allowed/>
+ <regular_expression/>
+ <regular_expression_not_allowed/>
+ <error_code/>
+ <error_description/>
+ <allowed_value>
+ <value>100</value>
+ </allowed_value>
+ </validator_field>
+ <validator_field>
+ <validation_name>unique-test-100</validation_name>
+ <name>countDistinctTest</name>
+ <max_length/>
+ <min_length/>
+ <null_allowed>N</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <data_type>Integer</data_type>
+ <data_type_verified>Y</data_type_verified>
+ <conversion_mask>0</conversion_mask>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <is_sourcing_values>N</is_sourcing_values>
+ <sourcing_transform/>
+ <sourcing_field/>
+ <min_value/>
+ <max_value/>
+ <start_string/>
+ <start_string_not_allowed/>
+ <end_string/>
+ <end_string_not_allowed/>
+ <regular_expression/>
+ <regular_expression_not_allowed/>
+ <error_code/>
+ <error_description/>
+ <allowed_value>
+ <value>100</value>
+ </allowed_value>
+ </validator_field>
+ <validate_all>Y</validate_all>
+ <concat_errors>Y</concat_errors>
+ <concat_separator>, </concat_separator>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <row_threshold>0</row_threshold>
+ <message>The Avro round trip did not return exactly 100 unique rows</message>
+ <always_log_rows>Y</always_log_rows>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ <error>
+ <source_transform>Data validator</source_transform>
+ <target_transform>Abort</target_transform>
+ <is_enabled>Y</is_enabled>
+ <nr_valuename>error_row</nr_valuename>
+ <descriptions_valuename>error_description</descriptions_valuename>
+ <fields_valuename/>
+ <codes_valuename>error_code</codes_valuename>
+ <max_errors/>
+ <max_pct_errors/>
+ <min_pct_rows/>
+ </error>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/spark-native/0009-avro-schema-file-write.hpl b/integration-tests/spark-native/0009-avro-schema-file-write.hpl
new file mode 100644
index 0000000000..ae8aebd008
--- /dev/null
+++ b/integration-tests/spark-native/0009-avro-schema-file-write.hpl
@@ -0,0 +1,188 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<pipeline>
+ <info>
+ <name>0009-avro-schema-file-write</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>source/test-file_0.csv</from>
+ <to>avro/test.avro (schema from project)</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>source/test-file_0.csv</name>
+ <type>TextFileInput2</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <accept_filenames>N</accept_filenames>
+ <files>
+ <file>
+ <name>${PROJECT_HOME}/output/0009-avro-schema-file/source/test-file_0.csv</name>
+ <filemask/>
+ <exclude_filemask/>
+ <file_required>Y</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ </files>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_breaks>N</enclosure_breaks>
+ <escapechar/>
+ <header>Y</header>
+ <nr_headerlines>1</nr_headerlines>
+ <footer>N</footer>
+ <nr_footerlines>1</nr_footerlines>
+ <line_wrapped>N</line_wrapped>
+ <nr_wraps>1</nr_wraps>
+ <layout_paged>N</layout_paged>
+ <nr_lines_per_page>80</nr_lines_per_page>
+ <nr_lines_doc_header>0</nr_lines_doc_header>
+ <noempty>Y</noempty>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <rownumByFile>N</rownumByFile>
+ <rownum_field/>
+ <format>Unix</format>
+ <encoding>UTF-8</encoding>
+ <length>Characters</length>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <file_type>CSV</file_type>
+ <compression>None</compression>
+ <filters>
+ </filters>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>both</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>test</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>both</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <error_ignored>N</error_ignored>
+ <skip_bad_files>N</skip_bad_files>
+ <error_line_skipped>N</error_line_skipped>
+ <date_format_lenient>Y</date_format_lenient>
+ <date_format_locale>en_US</date_format_locale>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>avro/test.avro (schema from project)</name>
+ <type>AvroOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <output_type>BinaryFile</output_type>
+ <filename>${PROJECT_HOME}/output/0009-avro-schema-file/avro/test.avro</filename>
+ <output_field_name/>
+ <create_schema_file>N</create_schema_file>
+ <write_schema_file>N</write_schema_file>
+ <schemafilename>${PROJECT_HOME}/files/avro-3901/test.avsc</schemafilename>
+ <namespace/>
+ <recordname/>
+ <doc/>
+ <compressiontype>none</compressiontype>
+ <create_parent_folder>Y</create_parent_folder>
+ <split>N</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <fields>
+ <field>
+ <name>id</name>
+ <avroname>$.id</avroname>
+ <avrotype>5</avrotype>
+ <nullable>Y</nullable>
+ </field>
+ <field>
+ <name>test</name>
+ <avroname>$.test</avroname>
+ <avrotype>6</avrotype>
+ <nullable>Y</nullable>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>528</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling/>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/spark-native/files/avro-3901/test.avsc b/integration-tests/spark-native/files/avro-3901/test.avsc
new file mode 100644
index 0000000000..f391cf3a9a
--- /dev/null
+++ b/integration-tests/spark-native/files/avro-3901/test.avsc
@@ -0,0 +1,10 @@
+{
+ "type": "record",
+ "name": "test",
+ "namespace": "org.apache.hop.it",
+ "doc": "Schema read from a project folder through a variable, issue #3901",
+ "fields": [
+ {"name": "id", "type": ["null", "long"], "default": null},
+ {"name": "test", "type": ["null", "string"], "default": null}
+ ]
+}
diff --git a/integration-tests/spark-native/main-0009-avro-schema-file.hwf b/integration-tests/spark-native/main-0009-avro-schema-file.hwf
new file mode 100644
index 0000000000..dc336f9d1b
--- /dev/null
+++ b/integration-tests/spark-native/main-0009-avro-schema-file.hwf
@@ -0,0 +1,235 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<workflow>
+ <name>main-0009-avro-schema-file</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>80</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>delete output</name>
+ <description/>
+ <type>DELETE_FOLDERS</type>
+ <attributes/>
+ <arg_from_previous>N</arg_from_previous>
+ <success_condition>success_if_no_errors</success_condition>
+ <limit_folders>10</limit_folders>
+ <fields>
+ <field>
+ <name>${PROJECT_HOME}/output/0009-avro-schema-file</name>
+ </field>
+ </fields>
+ <parallel>N</parallel>
+ <xloc>224</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0009-avro-schema-file-create-test-files</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0009-avro-schema-file-create-test-files.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>384</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0009-avro-schema-file-write</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0009-avro-schema-file-write.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>spark-local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>608</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0009-avro-schema-file-read</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0009-avro-schema-file-read.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>832</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0009-avro-schema-file-validate</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0009-avro-schema-file-validate.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>1056</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>delete output</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>delete output</from>
+ <to>0009-avro-schema-file-create-test-files</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0009-avro-schema-file-create-test-files</from>
+ <to>0009-avro-schema-file-write</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0009-avro-schema-file-write</from>
+ <to>0009-avro-schema-file-read</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0009-avro-schema-file-read</from>
+ <to>0009-avro-schema-file-validate</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ <notepad>
+ <backgroundcolorblue>251</backgroundcolorblue>
+ <backgroundcolorgreen>232</backgroundcolorgreen>
+ <backgroundcolorred>201</backgroundcolorred>
+ <bordercolorblue>90</bordercolorblue>
+ <bordercolorgreen>58</bordercolorgreen>
+ <bordercolorred>14</bordercolorred>
+ <fontbold>N</fontbold>
+ <fontcolorblue>90</fontcolorblue>
+ <fontcolorgreen>58</fontcolorgreen>
+ <fontcolorred>14</fontcolorred>
+ <fontitalic>N</fontitalic>
+ <fontname>Noto Sans</fontname>
+ <fontsize>10</fontsize>
+ <height>32</height>
+ <xloc>80</xloc>
+ <yloc>176</yloc>
+ <note>Issue #3901: Avro File Output reads its schema from '${PROJECT_HOME}/files/avro-3901/test.avsc'.
+Create 100 rows locally, write them to Avro on the native Spark engine (spark-local).
+Avro File Input / Decode can't run there (no Spark type for Avro Record), so read the files back locally,
+then verify locally that exactly 100 unique rows survived the round trip.</note>
+ <width>32</width>
+ </notepad>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/transforms/0118-avro-schema-file-create-test-files.hpl b/integration-tests/transforms/0118-avro-schema-file-create-test-files.hpl
new file mode 100644
index 0000000000..72549b31eb
--- /dev/null
+++ b/integration-tests/transforms/0118-avro-schema-file-create-test-files.hpl
@@ -0,0 +1,285 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<pipeline>
+ <info>
+ <name>0118-avro-schema-file-create-test-files</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>100 rows</from>
+ <to>id</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>id</from>
+ <to>format id</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>format id</from>
+ <to>test</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>test</from>
+ <to>${PROJECT_HOME}/output/0118-avro-schema-file/source/test-file</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>100 rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>prefix</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif>test</nullif>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <limit>100</limit>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field>now</row_time_field>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>id</name>
+ <type>Sequence</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <valuename>id</valuename>
+ <use_database>N</use_database>
+ <connection/>
+ <schema/>
+ <seqname>SEQ_</seqname>
+ <use_counter>Y</use_counter>
+ <counter_name/>
+ <start_at>1</start_at>
+ <increment_by>1</increment_by>
+ <max_value>999999999</max_value>
+ <attributes/>
+ <GUI>
+ <xloc>240</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <type>SelectValues</type>
+ <name>format id</name>
+ <fields>
+ <select_unspecified>N</select_unspecified>
+ <meta>
+ <name>id</name>
+ <rename>id</rename>
+ <type>String</type>
+ <length>-2</length>
+ <precision>0</precision>
+ <storage_type/>
+ <conversion_mask>000</conversion_mask>
+ <date_format_lenient>N</date_format_lenient>
+ <date_format_locale/>
+ <date_format_timezone/>
+ <lenient_string_to_number>N</lenient_string_to_number>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <currency_symbol/>
+ <encoding/>
+ <roundingType>half_even</roundingType>
+ </meta>
+ </fields>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>ConcatFields</type>
+ <name>test</name>
+ <separator>-</separator>
+ <enclosure/>
+ <force_enclosure>N</force_enclosure>
+ <fields>
+ <field>
+ <name>prefix</name>
+ <type>String</type>
+ <format/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type/>
+ </field>
+ <field>
+ <name>id</name>
+ <type>String</type>
+ <format/>
+ <length>3</length>
+ <precision>0</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type/>
+ </field>
+ </fields>
+ <ConcatFields>
+ <targetFieldName>test</targetFieldName>
+ <targetFieldLength>100</targetFieldLength>
+ <removeSelectedFields>N</removeSelectedFields>
+ </ConcatFields>
+ <skip_value_empty>N</skip_value_empty>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>528</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <name>${PROJECT_HOME}/output/0118-avro-schema-file/source/test-file</name>
+ <type>TextFileOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_forced>N</enclosure_forced>
+ <enclosure_fix_disabled>N</enclosure_fix_disabled>
+ <header>Y</header>
+ <footer>N</footer>
+ <format>UNIX</format>
+ <compression>None</compression>
+ <encoding>UTF-8</encoding>
+ <endedLine/>
+ <fileNameInField>N</fileNameInField>
+ <fileNameField/>
+ <create_parent_folder>Y</create_parent_folder>
+ <file>
+ <name>${PROJECT_HOME}/output/0118-avro-schema-file/source/test-file</name>
+ <do_not_open_new_file_init>Y</do_not_open_new_file_init>
+ <extension>csv</extension>
+ <append>N</append>
+ <split>Y</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <pad>N</pad>
+ <fast_dump>N</fast_dump>
+ <splitevery/>
+ </file>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <roundingType>half_even</roundingType>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>test</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <roundingType>half_even</roundingType>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>672</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling/>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0118-avro-schema-file-read.hpl b/integration-tests/transforms/0118-avro-schema-file-read.hpl
new file mode 100644
index 0000000000..7465b3c330
--- /dev/null
+++ b/integration-tests/transforms/0118-avro-schema-file-read.hpl
@@ -0,0 +1,223 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<pipeline>
+ <info>
+ <name>0118-avro-schema-file-read</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>avro/*.avro</from>
+ <to>Avro File Input</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Avro File Input</from>
+ <to>Avro Decode</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Avro Decode</from>
+ <to>readback/test-file</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>avro/*.avro</name>
+ <type>GetFileNames</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <file>
+ <name>${PROJECT_HOME}/output/0118-avro-schema-file/avro/</name>
+ <filemask>.*\.avro</filemask>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <filter>
+ <filterfiletype>all_files</filterfiletype>
+ </filter>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <filename_Field/>
+ <wildcard_Field/>
+ <exclude_wildcard_Field/>
+ <filefield>N</filefield>
+ <dynamic_include_subfolders>N</dynamic_include_subfolders>
+ <isaddresult>N</isaddresult>
+ <limit>0</limit>
+ <doNotFailIfNoFile>N</doNotFailIfNoFile>
+ <raiseAnExceptionIfNoFile>Y</raiseAnExceptionIfNoFile>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Avro File Input</name>
+ <type>AvroFileInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data_filename_field>filename</data_filename_field>
+ <output_field>avro</output_field>
+ <rows_limit/>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Avro Decode</name>
+ <type>AvroDecode</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <source_field>avro</source_field>
+ <remove_source_field>Y</remove_source_field>
+ <ignore_missing>N</ignore_missing>
+ <fields>
+ <field>
+ <source_field>id</source_field>
+ <source_avro_type>Union</source_avro_type>
+ <target_field_name>id</target_field_name>
+ <target_type>Integer</target_type>
+ </field>
+ <field>
+ <source_field>test</source_field>
+ <source_avro_type>Union</source_avro_type>
+ <target_field_name>test</target_field_name>
+ <target_type>String</target_type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>448</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>readback/test-file</name>
+ <type>TextFileOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_forced>N</enclosure_forced>
+ <enclosure_fix_disabled>N</enclosure_fix_disabled>
+ <header>Y</header>
+ <footer>N</footer>
+ <format>UNIX</format>
+ <compression>None</compression>
+ <encoding>UTF-8</encoding>
+ <endedLine/>
+ <fileNameInField>N</fileNameInField>
+ <fileNameField/>
+ <create_parent_folder>Y</create_parent_folder>
+ <file>
+ <name>${PROJECT_HOME}/output/0118-avro-schema-file/readback/test-file</name>
+ <do_not_open_new_file_init>Y</do_not_open_new_file_init>
+ <extension>csv</extension>
+ <append>N</append>
+ <split>Y</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <pad>N</pad>
+ <fast_dump>N</fast_dump>
+ <splitevery/>
+ </file>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <roundingType>half_even</roundingType>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>test</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <roundingType>half_even</roundingType>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>624</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling/>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0118-avro-schema-file-validate.hpl b/integration-tests/transforms/0118-avro-schema-file-validate.hpl
new file mode 100644
index 0000000000..ca9efafac4
--- /dev/null
+++ b/integration-tests/transforms/0118-avro-schema-file-validate.hpl
@@ -0,0 +1,370 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<pipeline>
+ <info>
+ <name>0118-avro-schema-file-validate</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>readback/*.csv</from>
+ <to>read readback</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>read readback</from>
+ <to>counts</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>counts</from>
+ <to>Data validator</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Data validator</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>readback/*.csv</name>
+ <type>GetFileNames</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <file>
+ <name>${PROJECT_HOME}/output/0118-avro-schema-file/readback/</name>
+ <filemask>.*\.csv</filemask>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <filter>
+ <filterfiletype>all_files</filterfiletype>
+ </filter>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <filename_Field/>
+ <wildcard_Field/>
+ <exclude_wildcard_Field/>
+ <filefield>N</filefield>
+ <dynamic_include_subfolders>N</dynamic_include_subfolders>
+ <isaddresult>N</isaddresult>
+ <limit>0</limit>
+ <doNotFailIfNoFile>N</doNotFailIfNoFile>
+ <raiseAnExceptionIfNoFile>Y</raiseAnExceptionIfNoFile>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>read readback</name>
+ <type>TextFileInput2</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <accept_filenames>Y</accept_filenames>
+ <passing_through_fields>N</passing_through_fields>
+ <accept_field>filename</accept_field>
+ <accept_transform_name>readback/*.csv</accept_transform_name>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_breaks>N</enclosure_breaks>
+ <escapechar/>
+ <header>Y</header>
+ <nr_headerlines>1</nr_headerlines>
+ <footer>N</footer>
+ <nr_footerlines>1</nr_footerlines>
+ <line_wrapped>N</line_wrapped>
+ <nr_wraps>1</nr_wraps>
+ <layout_paged>N</layout_paged>
+ <nr_lines_per_page>80</nr_lines_per_page>
+ <nr_lines_doc_header>0</nr_lines_doc_header>
+ <noempty>Y</noempty>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <rownumByFile>N</rownumByFile>
+ <rownum_field/>
+ <format>Unix</format>
+ <encoding>UTF-8</encoding>
+ <length>Characters</length>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <file_type>CSV</file_type>
+ <compression>None</compression>
+ <filters>
+ </filters>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>both</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>test</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>both</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <error_ignored>N</error_ignored>
+ <skip_bad_files>N</skip_bad_files>
+ <error_line_skipped>N</error_line_skipped>
+ <date_format_lenient>Y</date_format_lenient>
+ <date_format_locale>en_US</date_format_locale>
+ <attributes/>
+ <GUI>
+ <xloc>256</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>counts</name>
+ <type>MemoryGroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <give_back_row>N</give_back_row>
+ <group>
+ </group>
+ <fields>
+ <field>
+ <aggregate>nrRows</aggregate>
+ <subject>id</subject>
+ <type>COUNT_ALL</type>
+ <valuefield/>
+ </field>
+ <field>
+ <aggregate>countDistinctId</aggregate>
+ <subject>id</subject>
+ <type>COUNT_DISTINCT</type>
+ <valuefield/>
+ </field>
+ <field>
+ <aggregate>countDistinctTest</aggregate>
+ <subject>test</subject>
+ <type>COUNT_DISTINCT</type>
+ <valuefield/>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Data validator</name>
+ <type>Validator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <validator_field>
+ <validation_name>rows-100</validation_name>
+ <name>nrRows</name>
+ <max_length/>
+ <min_length/>
+ <null_allowed>N</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <data_type>Integer</data_type>
+ <data_type_verified>Y</data_type_verified>
+ <conversion_mask>0</conversion_mask>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <is_sourcing_values>N</is_sourcing_values>
+ <sourcing_transform/>
+ <sourcing_field/>
+ <min_value/>
+ <max_value/>
+ <start_string/>
+ <start_string_not_allowed/>
+ <end_string/>
+ <end_string_not_allowed/>
+ <regular_expression/>
+ <regular_expression_not_allowed/>
+ <error_code/>
+ <error_description/>
+ <allowed_value>
+ <value>100</value>
+ </allowed_value>
+ </validator_field>
+ <validator_field>
+ <validation_name>unique-id-100</validation_name>
+ <name>countDistinctId</name>
+ <max_length/>
+ <min_length/>
+ <null_allowed>N</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <data_type>Integer</data_type>
+ <data_type_verified>Y</data_type_verified>
+ <conversion_mask>0</conversion_mask>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <is_sourcing_values>N</is_sourcing_values>
+ <sourcing_transform/>
+ <sourcing_field/>
+ <min_value/>
+ <max_value/>
+ <start_string/>
+ <start_string_not_allowed/>
+ <end_string/>
+ <end_string_not_allowed/>
+ <regular_expression/>
+ <regular_expression_not_allowed/>
+ <error_code/>
+ <error_description/>
+ <allowed_value>
+ <value>100</value>
+ </allowed_value>
+ </validator_field>
+ <validator_field>
+ <validation_name>unique-test-100</validation_name>
+ <name>countDistinctTest</name>
+ <max_length/>
+ <min_length/>
+ <null_allowed>N</null_allowed>
+ <only_null_allowed>N</only_null_allowed>
+ <only_numeric_allowed>N</only_numeric_allowed>
+ <data_type>Integer</data_type>
+ <data_type_verified>Y</data_type_verified>
+ <conversion_mask>0</conversion_mask>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <is_sourcing_values>N</is_sourcing_values>
+ <sourcing_transform/>
+ <sourcing_field/>
+ <min_value/>
+ <max_value/>
+ <start_string/>
+ <start_string_not_allowed/>
+ <end_string/>
+ <end_string_not_allowed/>
+ <regular_expression/>
+ <regular_expression_not_allowed/>
+ <error_code/>
+ <error_description/>
+ <allowed_value>
+ <value>100</value>
+ </allowed_value>
+ </validator_field>
+ <validate_all>Y</validate_all>
+ <concat_errors>Y</concat_errors>
+ <concat_separator>, </concat_separator>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <row_threshold>0</row_threshold>
+ <message>The Avro round trip did not return exactly 100 unique rows</message>
+ <always_log_rows>Y</always_log_rows>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ <error>
+ <source_transform>Data validator</source_transform>
+ <target_transform>Abort</target_transform>
+ <is_enabled>Y</is_enabled>
+ <nr_valuename>error_row</nr_valuename>
+ <descriptions_valuename>error_description</descriptions_valuename>
+ <fields_valuename/>
+ <codes_valuename>error_code</codes_valuename>
+ <max_errors/>
+ <max_pct_errors/>
+ <min_pct_rows/>
+ </error>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0118-avro-schema-file-write.hpl b/integration-tests/transforms/0118-avro-schema-file-write.hpl
new file mode 100644
index 0000000000..e857a51739
--- /dev/null
+++ b/integration-tests/transforms/0118-avro-schema-file-write.hpl
@@ -0,0 +1,188 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<pipeline>
+ <info>
+ <name>0118-avro-schema-file-write</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>source/test-file_0.csv</from>
+ <to>avro/test.avro (schema from project)</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>source/test-file_0.csv</name>
+ <type>TextFileInput2</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <accept_filenames>N</accept_filenames>
+ <files>
+ <file>
+ <name>${PROJECT_HOME}/output/0118-avro-schema-file/source/test-file_0.csv</name>
+ <filemask/>
+ <exclude_filemask/>
+ <file_required>Y</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ </files>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_breaks>N</enclosure_breaks>
+ <escapechar/>
+ <header>Y</header>
+ <nr_headerlines>1</nr_headerlines>
+ <footer>N</footer>
+ <nr_footerlines>1</nr_footerlines>
+ <line_wrapped>N</line_wrapped>
+ <nr_wraps>1</nr_wraps>
+ <layout_paged>N</layout_paged>
+ <nr_lines_per_page>80</nr_lines_per_page>
+ <nr_lines_doc_header>0</nr_lines_doc_header>
+ <noempty>Y</noempty>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <rownumByFile>N</rownumByFile>
+ <rownum_field/>
+ <format>Unix</format>
+ <encoding>UTF-8</encoding>
+ <length>Characters</length>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <file_type>CSV</file_type>
+ <compression>None</compression>
+ <filters>
+ </filters>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>both</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>test</name>
+ <type>String</type>
+ <format></format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>both</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <error_ignored>N</error_ignored>
+ <skip_bad_files>N</skip_bad_files>
+ <error_line_skipped>N</error_line_skipped>
+ <date_format_lenient>Y</date_format_lenient>
+ <date_format_locale>en_US</date_format_locale>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>avro/test.avro (schema from project)</name>
+ <type>AvroOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <output_type>BinaryFile</output_type>
+ <filename>${PROJECT_HOME}/output/0118-avro-schema-file/avro/test.avro</filename>
+ <output_field_name/>
+ <create_schema_file>N</create_schema_file>
+ <write_schema_file>N</write_schema_file>
+ <schemafilename>${PROJECT_HOME}/files/avro-3901/test.avsc</schemafilename>
+ <namespace/>
+ <recordname/>
+ <doc/>
+ <compressiontype>none</compressiontype>
+ <create_parent_folder>Y</create_parent_folder>
+ <split>N</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <fields>
+ <field>
+ <name>id</name>
+ <avroname>$.id</avroname>
+ <avrotype>5</avrotype>
+ <nullable>Y</nullable>
+ </field>
+ <field>
+ <name>test</name>
+ <avroname>$.test</avroname>
+ <avrotype>6</avrotype>
+ <nullable>Y</nullable>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>528</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling/>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/files/avro-3901/test.avsc b/integration-tests/transforms/files/avro-3901/test.avsc
new file mode 100644
index 0000000000..f391cf3a9a
--- /dev/null
+++ b/integration-tests/transforms/files/avro-3901/test.avsc
@@ -0,0 +1,10 @@
+{
+ "type": "record",
+ "name": "test",
+ "namespace": "org.apache.hop.it",
+ "doc": "Schema read from a project folder through a variable, issue #3901",
+ "fields": [
+ {"name": "id", "type": ["null", "long"], "default": null},
+ {"name": "test", "type": ["null", "string"], "default": null}
+ ]
+}
diff --git a/integration-tests/transforms/main-0118-avro-schema-file.hwf b/integration-tests/transforms/main-0118-avro-schema-file.hwf
new file mode 100644
index 0000000000..b6413aab3d
--- /dev/null
+++ b/integration-tests/transforms/main-0118-avro-schema-file.hwf
@@ -0,0 +1,234 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<workflow>
+ <name>main-0118-avro-schema-file</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/09/24 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/24 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>80</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>delete output</name>
+ <description/>
+ <type>DELETE_FOLDERS</type>
+ <attributes/>
+ <arg_from_previous>N</arg_from_previous>
+ <success_condition>success_if_no_errors</success_condition>
+ <limit_folders>10</limit_folders>
+ <fields>
+ <field>
+ <name>${PROJECT_HOME}/output/0118-avro-schema-file</name>
+ </field>
+ </fields>
+ <parallel>N</parallel>
+ <xloc>224</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0118-avro-schema-file-create-test-files</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0118-avro-schema-file-create-test-files.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>384</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0118-avro-schema-file-write</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0118-avro-schema-file-write.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>608</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0118-avro-schema-file-read</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0118-avro-schema-file-read.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>832</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0118-avro-schema-file-validate</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0118-avro-schema-file-validate.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>1056</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>delete output</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>delete output</from>
+ <to>0118-avro-schema-file-create-test-files</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0118-avro-schema-file-create-test-files</from>
+ <to>0118-avro-schema-file-write</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0118-avro-schema-file-write</from>
+ <to>0118-avro-schema-file-read</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0118-avro-schema-file-read</from>
+ <to>0118-avro-schema-file-validate</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ <notepad>
+ <backgroundcolorblue>251</backgroundcolorblue>
+ <backgroundcolorgreen>232</backgroundcolorgreen>
+ <backgroundcolorred>201</backgroundcolorred>
+ <bordercolorblue>90</bordercolorblue>
+ <bordercolorgreen>58</bordercolorgreen>
+ <bordercolorred>14</bordercolorred>
+ <fontbold>N</fontbold>
+ <fontcolorblue>90</fontcolorblue>
+ <fontcolorgreen>58</fontcolorgreen>
+ <fontcolorred>14</fontcolorred>
+ <fontitalic>N</fontitalic>
+ <fontname>Noto Sans</fontname>
+ <fontsize>10</fontsize>
+ <height>32</height>
+ <xloc>80</xloc>
+ <yloc>176</yloc>
+ <note>Issue #3901: Avro File Output reads its schema from '${PROJECT_HOME}/files/avro-3901/test.avsc'.
+Create 100 rows locally, write them to Avro and read them back with Avro File Input + Avro Decode on the local Hop engine,
+then verify locally that exactly 100 unique rows survived the round trip.</note>
+ <width>32</width>
+ </notepad>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrodecode/AvroDecodeMeta.java b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrodecode/AvroDecodeMeta.java
index 4a383a0113..01751836e1 100644
--- a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrodecode/AvroDecodeMeta.java
+++ b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrodecode/AvroDecodeMeta.java
@@ -37,7 +37,8 @@ import org.apache.hop.pipeline.transform.TransformMeta;
image = "avro_decode.svg",
categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Transform",
documentationUrl = "/pipeline/transforms/avro-decode.html",
- keywords = "i18n::AvroDecodeMeta.keyword")
+ keywords = "i18n::AvroDecodeMeta.keyword",
+ excludedEngines = {"SparkPipelineEngine"})
public class AvroDecodeMeta extends BaseTransformMeta<AvroDecode, AvroDecodeData> {
private static final Class<?> PKG = AvroDecodeMeta.class;
diff --git a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroencode/AvroEncodeMeta.java b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroencode/AvroEncodeMeta.java
index 742d729173..4b4236e3b4 100644
--- a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroencode/AvroEncodeMeta.java
+++ b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroencode/AvroEncodeMeta.java
@@ -43,7 +43,8 @@ import org.apache.hop.pipeline.transform.TransformMeta;
image = "avro_encode.svg",
categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Transform",
documentationUrl = "/pipeline/transforms/avro-encode.html",
- keywords = "i18n::AvroEncodeMeta.keyword")
+ keywords = "i18n::AvroEncodeMeta.keyword",
+ excludedEngines = {"SparkPipelineEngine"})
public class AvroEncodeMeta extends BaseTransformMeta<AvroEncode, AvroEncodeData> {
private static final Class<?> PKG = AvroEncodeMeta.class;
diff --git a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroinput/AvroFileInputMeta.java b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroinput/AvroFileInputMeta.java
index eb36a8dfb4..01a3c497ce 100644
--- a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroinput/AvroFileInputMeta.java
+++ b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroinput/AvroFileInputMeta.java
@@ -36,7 +36,8 @@ import org.apache.hop.pipeline.transform.TransformMeta;
image = "avro_input.svg",
categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Input",
documentationUrl = "/pipeline/transforms/avro-file-input.html",
- keywords = "i18n::AvroFileInputMeta.keyword")
+ keywords = "i18n::AvroFileInputMeta.keyword",
+ excludedEngines = {"SparkPipelineEngine"})
@InjectionSupported(localizationPrefix = "AvroInputMeta.Injection.")
public class AvroFileInputMeta extends BaseTransformMeta<AvroFileInput, AvroFileInputData> {
diff --git a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutput.java b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutput.java
index 2a98dc0b82..2d8ffcdf20 100644
--- a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutput.java
+++ b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutput.java
@@ -19,8 +19,8 @@ package org.apache.hop.avro.transforms.avrooutput;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
-import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
@@ -382,20 +382,16 @@ public class AvroOutput extends BaseTransform<AvroOutputMeta, AvroOutputData> {
private void createFileAndSchema() throws HopException {
try {
+ // Build the schema once: Beam calls this again at the start of every bundle
if (meta.isCreateSchemaFile()) {
- if (isDetailed()) {
- logDetailed("Generating Avro schema.");
- }
- writeSchemaFile();
- } else {
- if (isDetailed()) {
- logDetailed("Reading Avro schema from file.");
- }
- try {
- data.avroSchema = new Schema.Parser().parse(new File(meta.getSchemaFileName()));
- } catch (Exception e) {
- logError("Error parsing schema file", e);
+ if (data.avroSchema == null) {
+ if (isDetailed()) {
+ logDetailed("Generating Avro schema.");
+ }
+ writeSchemaFile();
}
+ } else if (data.avroSchema == null) {
+ data.avroSchema = readSchemaFile();
}
data.datumWriter = new GenericDatumWriter<>(data.avroSchema);
@@ -428,6 +424,27 @@ public class AvroOutput extends BaseTransform<AvroOutputMeta, AvroOutputData> {
}
}
+ /**
+ * Reads the Avro schema from the (variable-resolved) schema filename. Uses Hop VFS so the schema
+ * can live in a project folder, on a cloud file system, etc.
+ */
+ Schema readSchemaFile() throws HopException {
+ String schemaFileName = resolve(meta.getSchemaFileName());
+ if (Utils.isEmpty(schemaFileName)) {
+ throw new HopException(
+ BaseMessages.getString(PKG, "AvroOutput.Exception.SchemaFileNameNotSet"));
+ }
+ if (isDetailed()) {
+ logDetailed("Reading Avro schema from file [" + schemaFileName + "]");
+ }
+ try (InputStream inputStream = HopVfs.getInputStream(schemaFileName, variables)) {
+ return new Schema.Parser().parse(inputStream);
+ } catch (Exception e) {
+ throw new HopException(
+ BaseMessages.getString(PKG, "AvroOutput.Exception.ReadingSchemaFile", schemaFileName), e);
+ }
+ }
+
private void closeOutput() throws HopException {
if (AvroOutputMeta.OUTPUT_TYPES[AvroOutputMeta.OUTPUT_TYPE_FIELD].equals(
meta.getOutputType())) {
@@ -446,7 +463,6 @@ public class AvroOutput extends BaseTransform<AvroOutputMeta, AvroOutputData> {
closeFile();
}
data.datumWriter = null;
- data.avroSchema = null;
}
public Object getValue(
@@ -600,7 +616,6 @@ public class AvroOutput extends BaseTransform<AvroOutputMeta, AvroOutputData> {
}
}
data.datumWriter = null;
- data.avroSchema = null;
retval = true;
} catch (Exception e) {
diff --git a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutputDialog.java b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutputDialog.java
index 43a29b7cc5..d308df2a6f 100644
--- a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutputDialog.java
+++ b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutputDialog.java
@@ -17,7 +17,7 @@
package org.apache.hop.avro.transforms.avrooutput;
-import java.io.File;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -29,6 +29,7 @@ import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
@@ -824,7 +825,10 @@ public class AvroOutputDialog extends BaseTransformDialog {
try {
avroSchema = null;
- avroSchema = new Schema.Parser().parse(new File(variables.resolve(wSchema.getText())));
+ try (InputStream inputStream =
+ HopVfs.getInputStream(variables.resolve(wSchema.getText()), variables)) {
+ avroSchema = new Schema.Parser().parse(inputStream);
+ }
validSchema = true;
diff --git a/plugins/tech/avro/src/main/resources/org/apache/hop/avro/transforms/avrooutput/messages/messages_en_US.properties b/plugins/tech/avro/src/main/resources/org/apache/hop/avro/transforms/avrooutput/messages/messages_en_US.properties
index cbfe9438b4..e9b3a90575 100644
--- a/plugins/tech/avro/src/main/resources/org/apache/hop/avro/transforms/avrooutput/messages/messages_en_US.properties
+++ b/plugins/tech/avro/src/main/resources/org/apache/hop/avro/transforms/avrooutput/messages/messages_en_US.properties
@@ -17,6 +17,9 @@
AvroFileOutputMeta.keyword=avro,write,serialize,apache,hadoop
+AvroOutput.Exception.FileNameNotSet=No output filename was specified.
+AvroOutput.Exception.ReadingSchemaFile=Unable to read the Avro schema from file ''{0}''
+AvroOutput.Exception.SchemaFileNameNotSet=No schema filename was specified. Either specify one or enable ''Automatically create schema''.
AvroOutput.Injection.ADD_TO_RESULT=Add output filename to result? (Y/N)
AvroOutput.Injection.AUTO_CREATE_SCHEMA=Automatically generate the Avro schema? (Y/N)
AvroOutput.Injection.AVRO_DOC=The documentation for the Avro schema.
diff --git a/plugins/tech/avro/src/test/java/org/apache/hop/avro/transforms/avrooutput/AvroOutputSchemaFileTest.java b/plugins/tech/avro/src/test/java/org/apache/hop/avro/transforms/avrooutput/AvroOutputSchemaFileTest.java
new file mode 100644
index 0000000000..93f18f5c19
--- /dev/null
+++ b/plugins/tech/avro/src/test/java/org/apache/hop/avro/transforms/avrooutput/AvroOutputSchemaFileTest.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.avro.transforms.avrooutput;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Issue #3901: a schema filename containing variables (e.g. {@code
+ * ${PROJECT_HOME}/schemas/customer.avsc}) was handed to {@code java.io.File} unresolved, so only a
+ * bare filename relative to the Hop installation folder could be used.
+ */
+class AvroOutputSchemaFileTest {
+
+ private static final String SCHEMA =
+ "{\"type\":\"record\",\"name\":\"customer\",\"namespace\":\"org.apache.hop.test\","
+ + "\"fields\":["
+ + "{\"name\":\"id\",\"type\":[\"null\",\"long\"]},"
+ + "{\"name\":\"name\",\"type\":[\"null\",\"string\"]}]}";
+
+ @TempDir Path tempDir;
+
+ private TransformMockHelper<AvroOutputMeta, AvroOutputData> mockHelper;
+ private AvroOutputData data;
+
+ @BeforeAll
+ static void initEnv() throws Exception {
+ HopEnvironment.init();
+ }
+
+ @AfterAll
+ static void resetEnv() {
+ HopEnvironment.reset();
+ }
+
+ @BeforeEach
+ void setUp() {
+ mockHelper =
+ new TransformMockHelper<>("AvroOutput", AvroOutputMeta.class, AvroOutputData.class);
+ when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(mockHelper.iLogChannel);
+ data = new AvroOutputData();
+ }
+
+ @AfterEach
+ void tearDown() {
+ mockHelper.cleanUp();
+ }
+
+ @Test
+ void schemaFilenameWithVariablesIsResolved() throws Exception {
+ Files.writeString(tempDir.resolve("customer.avsc"), SCHEMA);
+ AvroOutputMeta meta = new AvroOutputMeta();
+ meta.setSchemaFileName("${SCHEMA_FOLDER}/customer.avsc");
+ AvroOutput transform = createTransform(meta);
+ transform.setVariable("SCHEMA_FOLDER", tempDir.toString());
+
+ Schema schema = transform.readSchemaFile();
+
+ assertEquals("org.apache.hop.test.customer", schema.getFullName());
+ }
+
+ @Test
+ void schemaFileIsReadThroughVfs() throws Exception {
+ String schemaFileName = "ram:///avro-3901/customer.avsc";
+ HopVfs.getFileObject("ram:///avro-3901").createFolder();
+ try (OutputStream outputStream = HopVfs.getOutputStream(schemaFileName, false)) {
+ outputStream.write(SCHEMA.getBytes(StandardCharsets.UTF_8));
+ }
+ AvroOutputMeta meta = new AvroOutputMeta();
+ meta.setSchemaFileName(schemaFileName);
+
+ Schema schema = createTransform(meta).readSchemaFile();
+
+ assertEquals(2, schema.getFields().size());
+ }
+
+ @Test
+ void missingSchemaFileFailsWithTheResolvedFilename() {
+ AvroOutputMeta meta = new AvroOutputMeta();
+ meta.setSchemaFileName("${SCHEMA_FOLDER}/does-not-exist.avsc");
+ AvroOutput transform = createTransform(meta);
+ transform.setVariable("SCHEMA_FOLDER", tempDir.toString());
+
+ HopException e = assertThrows(HopException.class, transform::readSchemaFile);
+
+ assertTrue(
+ e.getMessage().contains(tempDir.resolve("does-not-exist.avsc").toString()), e.getMessage());
+ }
+
+ @Test
+ void emptySchemaFilenameFails() {
+ AvroOutputMeta meta = new AvroOutputMeta();
+ meta.setSchemaFileName("");
+
+ assertThrows(HopException.class, () -> createTransform(meta).readSchemaFile());
+ }
+
+ @Test
+ void invalidSchemaFileFails() throws Exception {
+ Files.writeString(tempDir.resolve("broken.avsc"), "{ this is not a schema");
+ AvroOutputMeta meta = new AvroOutputMeta();
+ meta.setSchemaFileName(tempDir.resolve("broken.avsc").toString());
+
+ assertThrows(HopException.class, () -> createTransform(meta).readSchemaFile());
+ }
+
+ /** The whole transform: rows written to a binary file with the schema from a variable path. */
+ @Test
+ void writesBinaryFileUsingSchemaFromVariablePath() throws Exception {
+ Files.writeString(tempDir.resolve("customer.avsc"), SCHEMA);
+ AvroOutputMeta meta = binaryFileMeta("${OUT}/customer.avsc", "${OUT}/customers.avro");
+ AvroOutput transform =
+ spyWithRows(meta, new Object[] {1L, "Alice"}, new Object[] {2L, "Bob"}, null);
+ transform.setVariable("OUT", tempDir.toString());
+
+ assertTrue(transform.processRow());
+ assertTrue(transform.processRow());
+ assertFalse(transform.processRow());
+
+ List<GenericRecord> records = readAvroFile(tempDir.resolve("customers.avro"));
+ assertEquals(2, records.size());
+ assertEquals(1L, records.get(0).get("id"));
+ assertEquals("Bob", records.get(1).get("name").toString());
+ }
+
+ /** Before #3901 an unreadable schema was only logged and the transform carried on. */
+ @Test
+ void unreadableSchemaStopsTheTransform() throws Exception {
+ AvroOutputMeta meta = binaryFileMeta("${OUT}/does-not-exist.avsc", "${OUT}/customers.avro");
+ AvroOutput transform = spyWithRows(meta, new Object[] {1L, "Alice"});
+ transform.setVariable("OUT", tempDir.toString());
+
+ assertThrows(HopException.class, transform::processRow);
+ assertFalse(Files.exists(tempDir.resolve("customers.avro")));
+ }
+
+ /** Beam calls startBundle() for every bundle; the schema file is only parsed once. */
+ @Test
+ void schemaIsReadOnlyOnceAcrossBundles() throws Exception {
+ Path schemaFile = tempDir.resolve("customer.avsc");
+ Files.writeString(schemaFile, SCHEMA);
+ AvroOutputMeta meta = binaryFileMeta(schemaFile.toString(), "${OUT}/customers.avro");
+ meta.setOutputType(AvroOutputMeta.OUTPUT_TYPES[AvroOutputMeta.OUTPUT_TYPE_FIELD]);
+ meta.setOutputFieldName("avro");
+ AvroOutput transform = spyWithRows(meta, new Object[] {1L, "Alice"});
+ transform.setVariable("OUT", tempDir.toString());
+
+ assertTrue(transform.processRow());
+ Schema first = data.avroSchema;
+ Files.delete(schemaFile);
+ transform.finishBundle();
+ transform.startBundle();
+
+ assertEquals(first, data.avroSchema);
+ }
+
+ /** A generated schema file is written once, not rewritten at the start of every bundle. */
+ @Test
+ void generatedSchemaFileIsWrittenOnceAcrossBundles() throws Exception {
+ Path schemaFile = tempDir.resolve("generated.avsc");
+ AvroOutputMeta meta = binaryFileMeta("${OUT}/generated.avsc", "${OUT}/customers.avro");
+ meta.setOutputType(AvroOutputMeta.OUTPUT_TYPES[AvroOutputMeta.OUTPUT_TYPE_FIELD]);
+ meta.setOutputFieldName("avro");
+ meta.setCreateSchemaFile(true);
+ meta.setWriteSchemaFile(true);
+ AvroOutput transform = spyWithRows(meta, new Object[] {1L, "Alice"});
+ transform.setVariable("OUT", tempDir.toString());
+
+ assertTrue(transform.processRow());
+ Schema first = data.avroSchema;
+ assertTrue(Files.exists(schemaFile));
+ Files.delete(schemaFile);
+ transform.finishBundle();
+ transform.startBundle();
+
+ assertEquals(first, data.avroSchema);
+ assertFalse(Files.exists(schemaFile));
+ }
+
+ private AvroOutputMeta binaryFileMeta(String schemaFileName, String fileName) {
+ AvroOutputMeta meta = new AvroOutputMeta();
+ meta.setDefault();
+ meta.setOutputType(AvroOutputMeta.OUTPUT_TYPES[AvroOutputMeta.OUTPUT_TYPE_BINARY_FILE]);
+ meta.setCreateSchemaFile(false);
+ meta.setWriteSchemaFile(false);
+ meta.setSchemaFileName(schemaFileName);
+ meta.setFileName(fileName);
+ List<AvroOutputField> fields = new ArrayList<>();
+ fields.add(new AvroOutputField("id", "id", AvroOutputField.AVRO_TYPE_LONG, true));
+ fields.add(new AvroOutputField("name", "name", AvroOutputField.AVRO_TYPE_STRING, true));
+ meta.setOutputFields(fields);
+ return meta;
+ }
+
+ private AvroOutput spyWithRows(AvroOutputMeta meta, Object[] row, Object[]... moreRows)
+ throws Exception {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("id"));
+ rowMeta.addValueMeta(new ValueMetaString("name"));
+
+ AvroOutput transform = spy(createTransform(meta));
+ doReturn(rowMeta).when(transform).getInputRowMeta();
+ doReturn(row, (Object[]) moreRows).when(transform).getRow();
+ doAnswer(inv -> null).when(transform).putRow(any(), any());
+ return transform;
+ }
+
+ private AvroOutput createTransform(AvroOutputMeta meta) {
+ return new AvroOutput(
+ mockHelper.transformMeta, meta, data, 0, mockHelper.pipelineMeta, mockHelper.pipeline);
+ }
+
+ private static List<GenericRecord> readAvroFile(Path file) throws Exception {
+ List<GenericRecord> records = new ArrayList<>();
+ try (InputStream inputStream = Files.newInputStream(file);
+ DataFileStream<GenericRecord> reader =
+ new DataFileStream<>(inputStream, new GenericDatumReader<>())) {
+ reader.forEach(records::add);
+ }
+ return records;
+ }
+}
diff --git a/pom.xml b/pom.xml
index 2a1f3847e8..dfe4ab9054 100644
--- a/pom.xml
+++ b/pom.xml
@@ -513,6 +513,7 @@
<exclude>**/integration-tests/**/*.md</exclude>
<exclude>**/integration-tests/**/*.csv</exclude>
<exclude>**/integration-tests/**/*.avro</exclude>
+ <exclude>**/integration-tests/**/*.avsc</exclude>
<exclude>**/integration-tests/**/*.xls</exclude>
<exclude>**/integration-tests/**/*.xlsx</exclude>
<exclude>**/integration-tests/**/*.json</exclude>