You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by li...@apache.org on 2026/06/02 03:55:57 UTC
(seatunnel) branch dev updated: [Feature][Connector-File] Add schema evolution support (ADD/DROP/RENAME/UPDATE column) for all file formats (#10744)
This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new e592175b50 [Feature][Connector-File] Add schema evolution support (ADD/DROP/RENAME/UPDATE column) for all file formats (#10744)
e592175b50 is described below
commit e592175b50839a18ee34da6e382f60127301b693
Author: Ricky Makhija <96...@users.noreply.github.com>
AuthorDate: Tue Jun 2 09:25:51 2026 +0530
[Feature][Connector-File] Add schema evolution support (ADD/DROP/RENAME/UPDATE column) for all file formats (#10744)
---
docs/en/connectors/sink/CosFile.md | 30 +
docs/en/connectors/sink/FtpFile.md | 30 +
docs/en/connectors/sink/HdfsFile.md | 30 +
docs/en/connectors/sink/LocalFile.md | 30 +
docs/en/connectors/sink/ObsFile.md | 30 +
docs/en/connectors/sink/OssFile.md | 30 +
docs/en/connectors/sink/OssJindoFile.md | 30 +
docs/en/connectors/sink/S3File.md | 30 +
docs/en/connectors/sink/SftpFile.md | 30 +
docs/zh/connectors/sink/CosFile.md | 30 +
docs/zh/connectors/sink/FtpFile.md | 30 +
docs/zh/connectors/sink/HdfsFile.md | 30 +
docs/zh/connectors/sink/LocalFile.md | 30 +
docs/zh/connectors/sink/ObsFile.md | 30 +
docs/zh/connectors/sink/OssFile.md | 30 +
docs/zh/connectors/sink/OssJindoFile.md | 30 +
docs/zh/connectors/sink/S3File.md | 30 +
docs/zh/connectors/sink/SftpFile.md | 30 +
.../api/transform/SeaTunnelTransform.java | 15 +
.../seatunnel/file/config/FileBaseSinkOptions.java | 10 +
.../seatunnel/file/sink/BaseFileSinkWriter.java | 10 +-
.../file/sink/BaseMultipleTableFileSink.java | 21 +-
.../seatunnel/file/sink/config/FileSinkConfig.java | 17 +
.../seatunnel/file/sink/util/ExcelGenerator.java | 2 +-
.../seatunnel/file/sink/util/XmlWriter.java | 4 +-
.../file/sink/writer/AbstractWriteStrategy.java | 262 +++++++-
.../file/sink/writer/BinaryWriteStrategy.java | 9 +
.../file/sink/writer/CanalJsonWriteStrategy.java | 16 +-
.../file/sink/writer/CsvWriteStrategy.java | 22 +-
.../sink/writer/DebeziumJsonWriteStrategy.java | 16 +-
.../file/sink/writer/JsonWriteStrategy.java | 14 +-
.../file/sink/writer/MaxWellJsonWriteStrategy.java | 16 +-
.../file/sink/writer/OrcWriteStrategy.java | 23 +-
.../file/sink/writer/ParquetWriteStrategy.java | 52 +-
.../file/sink/writer/TextWriteStrategy.java | 21 +-
.../seatunnel/file/sink/writer/WriteStrategy.java | 16 +
.../file/writer/FileSchemaEvolutionTest.java | 693 +++++++++++++++++++++
.../file/writer/ParquetTypeCoercionTest.java | 263 ++++++++
.../writer/ParquetWriteStrategyEvolutionTest.java | 264 ++++++++
.../server/task/flow/TransformFlowLifeCycle.java | 11 +-
.../common/AbstractCatalogSupportMapTransform.java | 48 ++
.../common/AbstractMultiCatalogTransform.java | 70 +++
.../common/AbstractSeaTunnelTransform.java | 11 +
.../seatunnel/transform/sql/SQLTransform.java | 42 +-
.../metadata/ChainTimestampPreservationTest.java | 268 ++++++++
.../MetadataMultiCatalogSchemaChangeTest.java | 184 ++++++
.../ProductionPipelineSchemaChangeTest.java | 276 ++++++++
.../metadata/TransformChainLiveAlterTest.java | 376 +++++++++++
.../sql/SQLMultiCatalogSchemaChangeTest.java | 135 ++++
49 files changed, 3664 insertions(+), 63 deletions(-)
diff --git a/docs/en/connectors/sink/CosFile.md b/docs/en/connectors/sink/CosFile.md
index c090b20e38..a77b6b249c 100644
--- a/docs/en/connectors/sink/CosFile.md
+++ b/docs/en/connectors/sink/CosFile.md
@@ -80,6 +80,7 @@ To use this connector you need put hadoop-cos-{hadoop.version}-{version}.jar and
| parquet_avro_write_fixed_as_int96 | array | no | - | Only used when file_format is parquet. |
| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
| merge_update_event | boolean | no | false | Only used when file_format_type is canal_json,debezium_json or maxwell_json. When value is true, the UPDATE_AFTER and UPDATE_BEFORE event will be merged into UPDATE event data |
+| schema_evolution_enabled | boolean | no | false | Enable schema evolution support for CDC pipelines. When true, ADD/DROP/RENAME/MODIFY column events from the source are applied to the sink without a job restart. Not supported for binary format. |
### path [string]
@@ -317,6 +318,35 @@ For orc file format simple config
```
+
+### schema_evolution_enabled [boolean]
+
+When set to `true`, the file sink handles CDC schema change events (ADD COLUMN, DROP COLUMN, RENAME COLUMN, MODIFY COLUMN type) at runtime without requiring a job restart. On each schema change the current output file is closed and a new file is opened with the updated schema.
+
+**Supported formats:** All file formats except `binary`. Enabling this option with `file_format_type = binary` will fail at job startup with a config validation error.
+
+**Partition constraint:** When `have_partition = true`, dropping a column listed in `partition_by` is not allowed and will fail fast. Partition columns must remain stable across schema changes.
+
+**When `schema_evolution_enabled = false` (default):** If the upstream CDC source has `schema-changes.enabled = true` and an `AlterTableEvent` arrives at the sink, the job will throw immediately with an actionable error:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+Users on the default CDC source config (`schema-changes.enabled = false`) are completely unaffected.
+
+**Known limitation:** Schema changes are not atomic with checkpointing. If the job crashes in the narrow window between file rotation and schema metadata update, rows written after restore may use the pre-change schema. This is a known architectural gap shared across other SeaTunnel sinks. For full restart-with-DDL correctness, a follow-up CDC source fix is required (tracked separately).
+
+Example usage in a CDC pipeline:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## Changelog
<ChangeLog />
diff --git a/docs/en/connectors/sink/FtpFile.md b/docs/en/connectors/sink/FtpFile.md
index f0f07a8b4d..d305c5a851 100644
--- a/docs/en/connectors/sink/FtpFile.md
+++ b/docs/en/connectors/sink/FtpFile.md
@@ -136,6 +136,7 @@ When the format in the `file_name_expression` parameter is `xxxx-${now}` , `file
| d | Day of month |
| H | Hour in day (0-23) |
| m | Minute in hour |
+| schema_evolution_enabled | boolean | no | false | Enable schema evolution support for CDC pipelines. When true, ADD/DROP/RENAME/MODIFY column events from the source are applied to the sink without a job restart. Not supported for binary format. |
| s | Second in minute |
### file_format_type [string]
@@ -353,6 +354,35 @@ FtpFile {
```
+
+### schema_evolution_enabled [boolean]
+
+When set to `true`, the file sink handles CDC schema change events (ADD COLUMN, DROP COLUMN, RENAME COLUMN, MODIFY COLUMN type) at runtime without requiring a job restart. On each schema change the current output file is closed and a new file is opened with the updated schema.
+
+**Supported formats:** All file formats except `binary`. Enabling this option with `file_format_type = binary` will fail at job startup with a config validation error.
+
+**Partition constraint:** When `have_partition = true`, dropping a column listed in `partition_by` is not allowed and will fail fast. Partition columns must remain stable across schema changes.
+
+**When `schema_evolution_enabled = false` (default):** If the upstream CDC source has `schema-changes.enabled = true` and an `AlterTableEvent` arrives at the sink, the job will throw immediately with an actionable error:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+Users on the default CDC source config (`schema-changes.enabled = false`) are completely unaffected.
+
+**Known limitation:** Schema changes are not atomic with checkpointing. If the job crashes in the narrow window between file rotation and schema metadata update, rows written after restore may use the pre-change schema. This is a known architectural gap shared across other SeaTunnel sinks. For full restart-with-DDL correctness, a follow-up CDC source fix is required (tracked separately).
+
+Example usage in a CDC pipeline:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## Changelog
<ChangeLog />
diff --git a/docs/en/connectors/sink/HdfsFile.md b/docs/en/connectors/sink/HdfsFile.md
index 000474ea08..2d80255d1f 100644
--- a/docs/en/connectors/sink/HdfsFile.md
+++ b/docs/en/connectors/sink/HdfsFile.md
@@ -91,6 +91,7 @@ Output data to hdfs file
| schema_save_mode | string | no | CREATE_SCHEMA_WHEN_NOT_EXIST | Existing dir processing method [...]
| data_save_mode | string | no | APPEND_DATA | Existing data processing method [...]
| merge_update_event | boolean | no | false | Only used when file_format_type is canal_json,debezium_json or maxwell_json. When value is true, the UPDATE_AFTER and UPDATE_BEFORE event will be merged into UPDATE event data [...]
+| schema_evolution_enabled | boolean | no | false | Enable schema evolution support for CDC pipelines. When true, ADD/DROP/RENAME/MODIFY column events from the source are applied to the sink without a job restart. Not supported for binary format. |
### Tips
@@ -291,6 +292,35 @@ Configure mount table in `core-site.xml`:
</configuration>
```
+
+### schema_evolution_enabled [boolean]
+
+When set to `true`, the file sink handles CDC schema change events (ADD COLUMN, DROP COLUMN, RENAME COLUMN, MODIFY COLUMN type) at runtime without requiring a job restart. On each schema change the current output file is closed and a new file is opened with the updated schema.
+
+**Supported formats:** All file formats except `binary`. Enabling this option with `file_format_type = binary` will fail at job startup with a config validation error.
+
+**Partition constraint:** When `have_partition = true`, dropping a column listed in `partition_by` is not allowed and will fail fast. Partition columns must remain stable across schema changes.
+
+**When `schema_evolution_enabled = false` (default):** If the upstream CDC source has `schema-changes.enabled = true` and an `AlterTableEvent` arrives at the sink, the job will throw immediately with an actionable error:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+Users on the default CDC source config (`schema-changes.enabled = false`) are completely unaffected.
+
+**Known limitation:** Schema changes are not atomic with checkpointing. If the job crashes in the narrow window between file rotation and schema metadata update, rows written after restore may use the pre-change schema. This is a known architectural gap shared across other SeaTunnel sinks. For full restart-with-DDL correctness, a follow-up CDC source fix is required (tracked separately).
+
+Example usage in a CDC pipeline:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## Changelog
<ChangeLog />
\ No newline at end of file
diff --git a/docs/en/connectors/sink/LocalFile.md b/docs/en/connectors/sink/LocalFile.md
index 115159756b..661b0d439a 100644
--- a/docs/en/connectors/sink/LocalFile.md
+++ b/docs/en/connectors/sink/LocalFile.md
@@ -78,6 +78,7 @@ By default, we use 2PC commit to ensure `exactly-once`
| schema_save_mode | string | no | CREATE_SCHEMA_WHEN_NOT_EXIST | Existing dir processing method |
| data_save_mode | string | no | APPEND_DATA | Existing data processing method |
| merge_update_event | boolean | no | false | Only used when file_format_type is canal_json,debezium_json or maxwell_json. When value is true, the UPDATE_AFTER and UPDATE_BEFORE event will be merged into UPDATE event data |
+| schema_evolution_enabled | boolean | no | false | Enable schema evolution support for CDC pipelines. When true, ADD/DROP/RENAME/MODIFY column events from the source are applied to the sink without a job restart. Not supported for binary format. |
### path [string]
@@ -348,6 +349,35 @@ LocalFile {
```
+
+### schema_evolution_enabled [boolean]
+
+When set to `true`, the file sink handles CDC schema change events (ADD COLUMN, DROP COLUMN, RENAME COLUMN, MODIFY COLUMN type) at runtime without requiring a job restart. On each schema change the current output file is closed and a new file is opened with the updated schema.
+
+**Supported formats:** All file formats except `binary`. Enabling this option with `file_format_type = binary` will fail at job startup with a config validation error.
+
+**Partition constraint:** When `have_partition = true`, dropping a column listed in `partition_by` is not allowed and will fail fast. Partition columns must remain stable across schema changes.
+
+**When `schema_evolution_enabled = false` (default):** If the upstream CDC source has `schema-changes.enabled = true` and an `AlterTableEvent` arrives at the sink, the job will throw immediately with an actionable error:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+Users on the default CDC source config (`schema-changes.enabled = false`) are completely unaffected.
+
+**Known limitation:** Schema changes are not atomic with checkpointing. If the job crashes in the narrow window between file rotation and schema metadata update, rows written after restore may use the pre-change schema. This is a known architectural gap shared across other SeaTunnel sinks. For full restart-with-DDL correctness, a follow-up CDC source fix is required (tracked separately).
+
+Example usage in a CDC pipeline:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## Changelog
<ChangeLog />
diff --git a/docs/en/connectors/sink/ObsFile.md b/docs/en/connectors/sink/ObsFile.md
index e092ce6015..9445d35083 100644
--- a/docs/en/connectors/sink/ObsFile.md
+++ b/docs/en/connectors/sink/ObsFile.md
@@ -87,6 +87,7 @@ It only supports hadoop version **2.9.X+**.
| sheet_name | string | no | Sheet${Random number} | Writer the sheet of the workbook. Only used when file_format is excel. |
| sheet_max_rows | int | no | 1048576 | Only used when file format_type is excel. |
| merge_update_event | boolean | no | false | Only used when file_format_type is canal_json,debezium_json or maxwell_json. When value is true, the UPDATE_AFTER and UPDATE_BEFORE event will be merged into UPDATE event data |
+| schema_evolution_enabled | boolean | no | false | Enable schema evolution support for CDC pipelines. When true, ADD/DROP/RENAME/MODIFY column events from the source are applied to the sink without a job restart. Not supported for binary format. |
### Tips
@@ -297,6 +298,35 @@ Please note that excel type does not support any compression format
```
+
+### schema_evolution_enabled [boolean]
+
+When set to `true`, the file sink handles CDC schema change events (ADD COLUMN, DROP COLUMN, RENAME COLUMN, MODIFY COLUMN type) at runtime without requiring a job restart. On each schema change the current output file is closed and a new file is opened with the updated schema.
+
+**Supported formats:** All file formats except `binary`. Enabling this option with `file_format_type = binary` will fail at job startup with a config validation error.
+
+**Partition constraint:** When `have_partition = true`, dropping a column listed in `partition_by` is not allowed and will fail fast. Partition columns must remain stable across schema changes.
+
+**When `schema_evolution_enabled = false` (default):** If the upstream CDC source has `schema-changes.enabled = true` and an `AlterTableEvent` arrives at the sink, the job will throw immediately with an actionable error:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+Users on the default CDC source config (`schema-changes.enabled = false`) are completely unaffected.
+
+**Known limitation:** Schema changes are not atomic with checkpointing. If the job crashes in the narrow window between file rotation and schema metadata update, rows written after restore may use the pre-change schema. This is a known architectural gap shared across other SeaTunnel sinks. For full restart-with-DDL correctness, a follow-up CDC source fix is required (tracked separately).
+
+Example usage in a CDC pipeline:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## Changelog
<ChangeLog />
diff --git a/docs/en/connectors/sink/OssFile.md b/docs/en/connectors/sink/OssFile.md
index 02754eb78d..46fb4c3f86 100644
--- a/docs/en/connectors/sink/OssFile.md
+++ b/docs/en/connectors/sink/OssFile.md
@@ -135,6 +135,7 @@ If write to `csv`, `text`, `json` file type, All column will be string.
| schema_save_mode | Enum | no | CREATE_SCHEMA_WHEN_NOT_EXIST | Before turning on the synchronous task, do different treatment of the target path |
| data_save_mode | Enum | no | APPEND_DATA | Before opening the synchronous task, the data file in the target path is differently processed |
| merge_update_event | boolean | no | false | Only used when file_format_type is canal_json,debezium_json or maxwell_json. When value is true, the UPDATE_AFTER and UPDATE_BEFORE event will be merged into UPDATE event data |
+| schema_evolution_enabled | boolean | no | false | Enable schema evolution support for CDC pipelines. When true, ADD/DROP/RENAME/MODIFY column events from the source are applied to the sink without a job restart. Not supported for binary format. |
### path [string]
@@ -582,6 +583,35 @@ sink {
> 1.[SeaTunnel Deployment Document](../../getting-started/locally/deployment.md).
+
+### schema_evolution_enabled [boolean]
+
+When set to `true`, the file sink handles CDC schema change events (ADD COLUMN, DROP COLUMN, RENAME COLUMN, MODIFY COLUMN type) at runtime without requiring a job restart. On each schema change the current output file is closed and a new file is opened with the updated schema.
+
+**Supported formats:** All file formats except `binary`. Enabling this option with `file_format_type = binary` will fail at job startup with a config validation error.
+
+**Partition constraint:** When `have_partition = true`, dropping a column listed in `partition_by` is not allowed and will fail fast. Partition columns must remain stable across schema changes.
+
+**When `schema_evolution_enabled = false` (default):** If the upstream CDC source has `schema-changes.enabled = true` and an `AlterTableEvent` arrives at the sink, the job will throw immediately with an actionable error:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+Users on the default CDC source config (`schema-changes.enabled = false`) are completely unaffected.
+
+**Known limitation:** Schema changes are not atomic with checkpointing. If the job crashes in the narrow window between file rotation and schema metadata update, rows written after restore may use the pre-change schema. This is a known architectural gap shared across other SeaTunnel sinks. For full restart-with-DDL correctness, a follow-up CDC source fix is required (tracked separately).
+
+Example usage in a CDC pipeline:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## Changelog
<ChangeLog />
diff --git a/docs/en/connectors/sink/OssJindoFile.md b/docs/en/connectors/sink/OssJindoFile.md
index 35f3021172..bfed2c17fb 100644
--- a/docs/en/connectors/sink/OssJindoFile.md
+++ b/docs/en/connectors/sink/OssJindoFile.md
@@ -84,6 +84,7 @@ It only supports hadoop version **2.9.X+**.
| parquet_avro_write_fixed_as_int96 | array | no | - | Only used when file_format is parquet. |
| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
| merge_update_event | boolean | no | false | Only used when file_format_type is canal_json,debezium_json or maxwell_json. When value is true, the UPDATE_AFTER and UPDATE_BEFORE event will be merged into UPDATE event data |
+| schema_evolution_enabled | boolean | no | false | Enable schema evolution support for CDC pipelines. When true, ADD/DROP/RENAME/MODIFY column events from the source are applied to the sink without a job restart. Not supported for binary format. |
### path [string]
@@ -318,6 +319,35 @@ For orc file format simple config
```
+
+### schema_evolution_enabled [boolean]
+
+When set to `true`, the file sink handles CDC schema change events (ADD COLUMN, DROP COLUMN, RENAME COLUMN, MODIFY COLUMN type) at runtime without requiring a job restart. On each schema change the current output file is closed and a new file is opened with the updated schema.
+
+**Supported formats:** All file formats except `binary`. Enabling this option with `file_format_type = binary` will fail at job startup with a config validation error.
+
+**Partition constraint:** When `have_partition = true`, dropping a column listed in `partition_by` is not allowed and will fail fast. Partition columns must remain stable across schema changes.
+
+**When `schema_evolution_enabled = false` (default):** If the upstream CDC source has `schema-changes.enabled = true` and an `AlterTableEvent` arrives at the sink, the job will throw immediately with an actionable error:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+Users on the default CDC source config (`schema-changes.enabled = false`) are completely unaffected.
+
+**Known limitation:** Schema changes are not atomic with checkpointing. If the job crashes in the narrow window between file rotation and schema metadata update, rows written after restore may use the pre-change schema. This is a known architectural gap shared across other SeaTunnel sinks. For full restart-with-DDL correctness, a follow-up CDC source fix is required (tracked separately).
+
+Example usage in a CDC pipeline:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## Changelog
<ChangeLog />
diff --git a/docs/en/connectors/sink/S3File.md b/docs/en/connectors/sink/S3File.md
index 4c7a1394cb..694539e2bf 100644
--- a/docs/en/connectors/sink/S3File.md
+++ b/docs/en/connectors/sink/S3File.md
@@ -145,6 +145,7 @@ If write to `csv`, `text` file type, All column will be string.
| enable_header_write | boolean | no | false | Only used when file_format_type is text,csv.<br/> false:don't write header,true:write header. |
| encoding | string | no | "UTF-8" | Only used when file_format_type is json,text,csv,xml. |
| merge_update_event | boolean | no | false | Only used when file_format_type is canal_json,debezium_json or maxwell_json. When value is true, the UPDATE_AFTER and UPDATE_BEFORE event will be merged into UPDATE event data |
+| schema_evolution_enabled | boolean | no | false | Enable schema evolution support for CDC pipelines. When true, ADD/DROP/RENAME/MODIFY column events from the source are applied to the sink without a job restart. Not supported for binary format. |
### path [string]
@@ -528,6 +529,35 @@ sink {
Only used when file_format_type is text,csv.false:don't write header,true:write header.
+
+### schema_evolution_enabled [boolean]
+
+When set to `true`, the file sink handles CDC schema change events (ADD COLUMN, DROP COLUMN, RENAME COLUMN, MODIFY COLUMN type) at runtime without requiring a job restart. On each schema change the current output file is closed and a new file is opened with the updated schema.
+
+**Supported formats:** All file formats except `binary`. Enabling this option with `file_format_type = binary` will fail at job startup with a config validation error.
+
+**Partition constraint:** When `have_partition = true`, dropping a column listed in `partition_by` is not allowed and will fail fast. Partition columns must remain stable across schema changes.
+
+**When `schema_evolution_enabled = false` (default):** If the upstream CDC source has `schema-changes.enabled = true` and an `AlterTableEvent` arrives at the sink, the job will throw immediately with an actionable error:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+Users on the default CDC source config (`schema-changes.enabled = false`) are completely unaffected.
+
+**Known limitation:** Schema changes are not atomic with checkpointing. If the job crashes in the narrow window between file rotation and schema metadata update, rows written after restore may use the pre-change schema. This is a known architectural gap shared across other SeaTunnel sinks. For full restart-with-DDL correctness, a follow-up CDC source fix is required (tracked separately).
+
+Example usage in a CDC pipeline:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## Changelog
<ChangeLog />
\ No newline at end of file
diff --git a/docs/en/connectors/sink/SftpFile.md b/docs/en/connectors/sink/SftpFile.md
index 33e60f79b2..683918c299 100644
--- a/docs/en/connectors/sink/SftpFile.md
+++ b/docs/en/connectors/sink/SftpFile.md
@@ -81,6 +81,7 @@ If you use SeaTunnel Engine, It automatically integrated the hadoop jar when you
| schema_save_mode | string | no | CREATE_SCHEMA_WHEN_NOT_EXIST | Existing dir processing method |
| data_save_mode | string | no | APPEND_DATA | Existing data processing method |
| merge_update_event | boolean | no | false | Only used when file_format_type is canal_json,debezium_json or maxwell_json. When value is true, the UPDATE_AFTER and UPDATE_BEFORE event will be merged into UPDATE event data |
+| schema_evolution_enabled | boolean | no | false | Enable schema evolution support for CDC pipelines. When true, ADD/DROP/RENAME/MODIFY column events from the source are applied to the sink without a job restart. Not supported for binary format. |
### host [string]
@@ -335,6 +336,35 @@ SftpFile {
```
+
+### schema_evolution_enabled [boolean]
+
+When set to `true`, the file sink handles CDC schema change events (ADD COLUMN, DROP COLUMN, RENAME COLUMN, MODIFY COLUMN type) at runtime without requiring a job restart. On each schema change the current output file is closed and a new file is opened with the updated schema.
+
+**Supported formats:** All file formats except `binary`. Enabling this option with `file_format_type = binary` will fail at job startup with a config validation error.
+
+**Partition constraint:** When `have_partition = true`, dropping a column listed in `partition_by` is not allowed and will fail fast. Partition columns must remain stable across schema changes.
+
+**When `schema_evolution_enabled = false` (default):** If the upstream CDC source has `schema-changes.enabled = true` and an `AlterTableEvent` arrives at the sink, the job will throw immediately with an actionable error:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+Users on the default CDC source config (`schema-changes.enabled = false`) are completely unaffected.
+
+**Known limitation:** Schema changes are not atomic with checkpointing. If the job crashes in the narrow window between file rotation and schema metadata update, rows written after restore may use the pre-change schema. This is a known architectural gap shared across other SeaTunnel sinks. For full restart-with-DDL correctness, a follow-up CDC source fix is required (tracked separately).
+
+Example usage in a CDC pipeline:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## Changelog
<ChangeLog />
diff --git a/docs/zh/connectors/sink/CosFile.md b/docs/zh/connectors/sink/CosFile.md
index aa5fca7a46..97402c64dc 100644
--- a/docs/zh/connectors/sink/CosFile.md
+++ b/docs/zh/connectors/sink/CosFile.md
@@ -79,6 +79,7 @@ import ChangeLog from '../changelog/connector-file-cos.md';
| parquet_avro_write_fixed_as_int96 | array | 否 | - | 仅在file_format为parquet时使用. |
| encoding | string | 否 | "UTF-8" | 仅当file_format_type为json、text、csv、xml时使用. |
| merge_update_event | boolean | 否 | false | 仅当file_format_type为canal_json、debezium_json、maxwell_json. |
+| schema_evolution_enabled | boolean | 否 | false | 开启 Schema 演变支持,适用于 CDC 管道。为 true 时,来自上游的 ADD/DROP/RENAME/MODIFY 列事件无需重启作业即可应用到 Sink。不支持 binary 格式。 |
### path [string]
@@ -312,6 +313,35 @@ Tips: excel 类型不支持任何压缩格式
```
+
+### schema_evolution_enabled [boolean]
+
+设置为 `true` 时,文件 Sink 可在运行时处理 CDC Schema 变更事件(ADD COLUMN、DROP COLUMN、RENAME COLUMN、MODIFY COLUMN 类型),无需重启作业。每次 Schema 变更时,当前输出文件会被关闭,并以新 Schema 打开一个新文件。
+
+**支持的格式:** 除 `binary` 外的所有文件格式。将此选项与 `file_format_type = binary` 一起使用时,作业启动时会抛出配置校验错误。
+
+**分区约束:** 当 `have_partition = true` 时,不允许删除 `partition_by` 中列出的列,违反时会立即抛出异常。分区列在 Schema 变更过程中必须保持稳定。
+
+**当 `schema_evolution_enabled = false`(默认值)时:** 若上游 CDC Source 配置了 `schema-changes.enabled = true` 且 Sink 收到 `AlterTableEvent`,作业会立即抛出如下错误:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+使用默认 CDC Source 配置(`schema-changes.enabled = false`)的用户不受影响。
+
+**已知限制:** Schema 变更与 Checkpoint 不是原子操作。若作业在文件轮转与 Schema 元数据更新之间的窗口期崩溃,恢复后写入的数据行可能使用变更前的 Schema。这是与其他 SeaTunnel Sink 共同存在的已知架构限制。完整的重启后 DDL 正确性支持需要配套的 CDC Source 修复(另行跟踪)。
+
+CDC 管道中的使用示例:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## 变更日志
<ChangeLog />
diff --git a/docs/zh/connectors/sink/FtpFile.md b/docs/zh/connectors/sink/FtpFile.md
index 2c77b53b14..fa93f91d42 100644
--- a/docs/zh/connectors/sink/FtpFile.md
+++ b/docs/zh/connectors/sink/FtpFile.md
@@ -136,6 +136,7 @@ import ChangeLog from '../changelog/connector-file-ftp.md';
| d | Day of month |
| H | Hour in day (0-23) |
| m | Minute in hour |
+| schema_evolution_enabled | boolean | 否 | false | 开启 Schema 演变支持,适用于 CDC 管道。为 true 时,来自上游的 ADD/DROP/RENAME/MODIFY 列事件无需重启作业即可应用到 Sink。不支持 binary 格式。 |
| s | Second in minute |
### file_format_type [string]
@@ -356,6 +357,35 @@ FtpFile {
```
+
+### schema_evolution_enabled [boolean]
+
+设置为 `true` 时,文件 Sink 可在运行时处理 CDC Schema 变更事件(ADD COLUMN、DROP COLUMN、RENAME COLUMN、MODIFY COLUMN 类型),无需重启作业。每次 Schema 变更时,当前输出文件会被关闭,并以新 Schema 打开一个新文件。
+
+**支持的格式:** 除 `binary` 外的所有文件格式。将此选项与 `file_format_type = binary` 一起使用时,作业启动时会抛出配置校验错误。
+
+**分区约束:** 当 `have_partition = true` 时,不允许删除 `partition_by` 中列出的列,违反时会立即抛出异常。分区列在 Schema 变更过程中必须保持稳定。
+
+**当 `schema_evolution_enabled = false`(默认值)时:** 若上游 CDC Source 配置了 `schema-changes.enabled = true` 且 Sink 收到 `AlterTableEvent`,作业会立即抛出如下错误:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+使用默认 CDC Source 配置(`schema-changes.enabled = false`)的用户不受影响。
+
+**已知限制:** Schema 变更与 Checkpoint 不是原子操作。若作业在文件轮转与 Schema 元数据更新之间的窗口期崩溃,恢复后写入的数据行可能使用变更前的 Schema。这是与其他 SeaTunnel Sink 共同存在的已知架构限制。完整的重启后 DDL 正确性支持需要配套的 CDC Source 修复(另行跟踪)。
+
+CDC 管道中的使用示例:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## 变更日志
<ChangeLog />
\ No newline at end of file
diff --git a/docs/zh/connectors/sink/HdfsFile.md b/docs/zh/connectors/sink/HdfsFile.md
index 1f98ea7099..bf2756d4a6 100644
--- a/docs/zh/connectors/sink/HdfsFile.md
+++ b/docs/zh/connectors/sink/HdfsFile.md
@@ -82,6 +82,7 @@ import ChangeLog from '../changelog/connector-file-hadoop.md';
| schema_save_mode | string | 否 | CREATE_SCHEMA_WHEN_NOT_EXIST | 现有目录处理方式 |
| data_save_mode | string | 否 | APPEND_DATA | 现有数据处理方式 |
| merge_update_event | boolean | 否 | false | 仅当file_format_type为canal_json、debezium_json、maxwell_json. |
+| schema_evolution_enabled | boolean | 否 | false | 开启 Schema 演变支持,适用于 CDC 管道。为 true 时,来自上游的 ADD/DROP/RENAME/MODIFY 列事件无需重启作业即可应用到 Sink。不支持 binary 格式。 |
### 提示
@@ -287,6 +288,35 @@ HdfsFile {
</configuration>
```
+
+### schema_evolution_enabled [boolean]
+
+设置为 `true` 时,文件 Sink 可在运行时处理 CDC Schema 变更事件(ADD COLUMN、DROP COLUMN、RENAME COLUMN、MODIFY COLUMN 类型),无需重启作业。每次 Schema 变更时,当前输出文件会被关闭,并以新 Schema 打开一个新文件。
+
+**支持的格式:** 除 `binary` 外的所有文件格式。将此选项与 `file_format_type = binary` 一起使用时,作业启动时会抛出配置校验错误。
+
+**分区约束:** 当 `have_partition = true` 时,不允许删除 `partition_by` 中列出的列,违反时会立即抛出异常。分区列在 Schema 变更过程中必须保持稳定。
+
+**当 `schema_evolution_enabled = false`(默认值)时:** 若上游 CDC Source 配置了 `schema-changes.enabled = true` 且 Sink 收到 `AlterTableEvent`,作业会立即抛出如下错误:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+使用默认 CDC Source 配置(`schema-changes.enabled = false`)的用户不受影响。
+
+**已知限制:** Schema 变更与 Checkpoint 不是原子操作。若作业在文件轮转与 Schema 元数据更新之间的窗口期崩溃,恢复后写入的数据行可能使用变更前的 Schema。这是与其他 SeaTunnel Sink 共同存在的已知架构限制。完整的重启后 DDL 正确性支持需要配套的 CDC Source 修复(另行跟踪)。
+
+CDC 管道中的使用示例:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## 变更日志
<ChangeLog />
\ No newline at end of file
diff --git a/docs/zh/connectors/sink/LocalFile.md b/docs/zh/connectors/sink/LocalFile.md
index 0e1549a751..26cf245093 100644
--- a/docs/zh/connectors/sink/LocalFile.md
+++ b/docs/zh/connectors/sink/LocalFile.md
@@ -76,6 +76,7 @@ import ChangeLog from '../changelog/connector-file-local.md';
| schema_save_mode | string | 否 | CREATE_SCHEMA_WHEN_NOT_EXIST | 现有目录处理方式 |
| data_save_mode | string | 否 | APPEND_DATA | 现有数据处理方式 |
| merge_update_event | boolean | 否 | false | 仅当file_format_type为canal_json、debezium_json、maxwell_json. |
+| schema_evolution_enabled | boolean | 否 | false | 开启 Schema 演变支持,适用于 CDC 管道。为 true 时,来自上游的 ADD/DROP/RENAME/MODIFY 列事件无需重启作业即可应用到 Sink。不支持 binary 格式。 |
### path [string]
@@ -340,6 +341,35 @@ LocalFile {
```
+
+### schema_evolution_enabled [boolean]
+
+设置为 `true` 时,文件 Sink 可在运行时处理 CDC Schema 变更事件(ADD COLUMN、DROP COLUMN、RENAME COLUMN、MODIFY COLUMN 类型),无需重启作业。每次 Schema 变更时,当前输出文件会被关闭,并以新 Schema 打开一个新文件。
+
+**支持的格式:** 除 `binary` 外的所有文件格式。将此选项与 `file_format_type = binary` 一起使用时,作业启动时会抛出配置校验错误。
+
+**分区约束:** 当 `have_partition = true` 时,不允许删除 `partition_by` 中列出的列,违反时会立即抛出异常。分区列在 Schema 变更过程中必须保持稳定。
+
+**当 `schema_evolution_enabled = false`(默认值)时:** 若上游 CDC Source 配置了 `schema-changes.enabled = true` 且 Sink 收到 `AlterTableEvent`,作业会立即抛出如下错误:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+使用默认 CDC Source 配置(`schema-changes.enabled = false`)的用户不受影响。
+
+**已知限制:** Schema 变更与 Checkpoint 不是原子操作。若作业在文件轮转与 Schema 元数据更新之间的窗口期崩溃,恢复后写入的数据行可能使用变更前的 Schema。这是与其他 SeaTunnel Sink 共同存在的已知架构限制。完整的重启后 DDL 正确性支持需要配套的 CDC Source 修复(另行跟踪)。
+
+CDC 管道中的使用示例:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## 变更日志
<ChangeLog />
diff --git a/docs/zh/connectors/sink/ObsFile.md b/docs/zh/connectors/sink/ObsFile.md
index 03945f9dd4..df1ff7afd7 100644
--- a/docs/zh/connectors/sink/ObsFile.md
+++ b/docs/zh/connectors/sink/ObsFile.md
@@ -86,6 +86,7 @@ import ChangeLog from '../changelog/connector-file-obs.md';
| max_rows_in_memory | int | 否 | - | 当文件格式为Excel时,内存中可以缓存的最大数据项数。仅在file_format为excel时使用。 |
| sheet_name | string | 否 | Sheet${Random number} | 标签页。仅在file_format为excel时使用。 |
| merge_update_event | boolean | 否 | false | 仅当file_format_type为canal_json、debezium_json、maxwell_json. |
+| schema_evolution_enabled | boolean | 否 | false | 开启 Schema 演变支持,适用于 CDC 管道。为 true 时,来自上游的 ADD/DROP/RENAME/MODIFY 列事件无需重启作业即可应用到 Sink。不支持 binary 格式。 |
### 提示
@@ -296,6 +297,35 @@ import ChangeLog from '../changelog/connector-file-obs.md';
```
+
+### schema_evolution_enabled [boolean]
+
+设置为 `true` 时,文件 Sink 可在运行时处理 CDC Schema 变更事件(ADD COLUMN、DROP COLUMN、RENAME COLUMN、MODIFY COLUMN 类型),无需重启作业。每次 Schema 变更时,当前输出文件会被关闭,并以新 Schema 打开一个新文件。
+
+**支持的格式:** 除 `binary` 外的所有文件格式。将此选项与 `file_format_type = binary` 一起使用时,作业启动时会抛出配置校验错误。
+
+**分区约束:** 当 `have_partition = true` 时,不允许删除 `partition_by` 中列出的列,违反时会立即抛出异常。分区列在 Schema 变更过程中必须保持稳定。
+
+**当 `schema_evolution_enabled = false`(默认值)时:** 若上游 CDC Source 配置了 `schema-changes.enabled = true` 且 Sink 收到 `AlterTableEvent`,作业会立即抛出如下错误:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+使用默认 CDC Source 配置(`schema-changes.enabled = false`)的用户不受影响。
+
+**已知限制:** Schema 变更与 Checkpoint 不是原子操作。若作业在文件轮转与 Schema 元数据更新之间的窗口期崩溃,恢复后写入的数据行可能使用变更前的 Schema。这是与其他 SeaTunnel Sink 共同存在的已知架构限制。完整的重启后 DDL 正确性支持需要配套的 CDC Source 修复(另行跟踪)。
+
+CDC 管道中的使用示例:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## 变更日志
<ChangeLog />
\ No newline at end of file
diff --git a/docs/zh/connectors/sink/OssFile.md b/docs/zh/connectors/sink/OssFile.md
index 2407b49e9b..962533728e 100644
--- a/docs/zh/connectors/sink/OssFile.md
+++ b/docs/zh/connectors/sink/OssFile.md
@@ -135,6 +135,7 @@ import ChangeLog from '../changelog/connector-file-oss.md';
| schema_save_mode | Enum | 否 | CREATE_SCHEMA_WHEN_NOT_EXIST | 在开启同步任务之前,对目标路径进行不同的处理 |
| data_save_mode | Enum | 否 | APPEND_DATA | 在开启同步任务之前,对目标路径中的数据文件进行不同的处理 |
| merge_update_event | boolean | 否 | false | 仅当file_format_type为canal_json、debezium_json、maxwell_json. |
+| schema_evolution_enabled | boolean | 否 | false | 开启 Schema 演变支持,适用于 CDC 管道。为 true 时,来自上游的 ADD/DROP/RENAME/MODIFY 列事件无需重启作业即可应用到 Sink。不支持 binary 格式。 |
### path [string]
@@ -578,6 +579,35 @@ sink {
> 1.[SeaTunnel部署方案](../../getting-started/locally/deployment.md).
+
+### schema_evolution_enabled [boolean]
+
+设置为 `true` 时,文件 Sink 可在运行时处理 CDC Schema 变更事件(ADD COLUMN、DROP COLUMN、RENAME COLUMN、MODIFY COLUMN 类型),无需重启作业。每次 Schema 变更时,当前输出文件会被关闭,并以新 Schema 打开一个新文件。
+
+**支持的格式:** 除 `binary` 外的所有文件格式。将此选项与 `file_format_type = binary` 一起使用时,作业启动时会抛出配置校验错误。
+
+**分区约束:** 当 `have_partition = true` 时,不允许删除 `partition_by` 中列出的列,违反时会立即抛出异常。分区列在 Schema 变更过程中必须保持稳定。
+
+**当 `schema_evolution_enabled = false`(默认值)时:** 若上游 CDC Source 配置了 `schema-changes.enabled = true` 且 Sink 收到 `AlterTableEvent`,作业会立即抛出如下错误:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+使用默认 CDC Source 配置(`schema-changes.enabled = false`)的用户不受影响。
+
+**已知限制:** Schema 变更与 Checkpoint 不是原子操作。若作业在文件轮转与 Schema 元数据更新之间的窗口期崩溃,恢复后写入的数据行可能使用变更前的 Schema。这是与其他 SeaTunnel Sink 共同存在的已知架构限制。完整的重启后 DDL 正确性支持需要配套的 CDC Source 修复(另行跟踪)。
+
+CDC 管道中的使用示例:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## 变更日志
<ChangeLog />
diff --git a/docs/zh/connectors/sink/OssJindoFile.md b/docs/zh/connectors/sink/OssJindoFile.md
index 4d2d9c346a..9d715056c1 100644
--- a/docs/zh/connectors/sink/OssJindoFile.md
+++ b/docs/zh/connectors/sink/OssJindoFile.md
@@ -81,6 +81,7 @@ import ChangeLog from '../changelog/connector-file-oss-jindo.md';
| parquet_avro_write_fixed_as_int96 | array | 否 | - | 仅在file_format为parquet时使用。 |
| encoding | string | 否 | "UTF-8" | 仅当file_format_type为json、text、csv、xml时使用。 |
| merge_update_event | boolean | 否 | false | 仅当file_format_type为canal_json、debezium_json、maxwell_json. |
+| schema_evolution_enabled | boolean | 否 | false | 开启 Schema 演变支持,适用于 CDC 管道。为 true 时,来自上游的 ADD/DROP/RENAME/MODIFY 列事件无需重启作业即可应用到 Sink。不支持 binary 格式。 |
### path [string]
@@ -313,6 +314,35 @@ Sink插件常用参数,请参考[Sink common Options](../common-options/sink
```
+
+### schema_evolution_enabled [boolean]
+
+设置为 `true` 时,文件 Sink 可在运行时处理 CDC Schema 变更事件(ADD COLUMN、DROP COLUMN、RENAME COLUMN、MODIFY COLUMN 类型),无需重启作业。每次 Schema 变更时,当前输出文件会被关闭,并以新 Schema 打开一个新文件。
+
+**支持的格式:** 除 `binary` 外的所有文件格式。将此选项与 `file_format_type = binary` 一起使用时,作业启动时会抛出配置校验错误。
+
+**分区约束:** 当 `have_partition = true` 时,不允许删除 `partition_by` 中列出的列,违反时会立即抛出异常。分区列在 Schema 变更过程中必须保持稳定。
+
+**当 `schema_evolution_enabled = false`(默认值)时:** 若上游 CDC Source 配置了 `schema-changes.enabled = true` 且 Sink 收到 `AlterTableEvent`,作业会立即抛出如下错误:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+使用默认 CDC Source 配置(`schema-changes.enabled = false`)的用户不受影响。
+
+**已知限制:** Schema 变更与 Checkpoint 不是原子操作。若作业在文件轮转与 Schema 元数据更新之间的窗口期崩溃,恢复后写入的数据行可能使用变更前的 Schema。这是与其他 SeaTunnel Sink 共同存在的已知架构限制。完整的重启后 DDL 正确性支持需要配套的 CDC Source 修复(另行跟踪)。
+
+CDC 管道中的使用示例:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## 变更日志
<ChangeLog />
\ No newline at end of file
diff --git a/docs/zh/connectors/sink/S3File.md b/docs/zh/connectors/sink/S3File.md
index 0cd641633c..c73c78e2dd 100644
--- a/docs/zh/connectors/sink/S3File.md
+++ b/docs/zh/connectors/sink/S3File.md
@@ -142,6 +142,7 @@ import ChangeLog from '../changelog/connector-file-s3.md';
| enable_header_write | boolean | 否 | false | 仅当 file_format_type 为 text,csv 时使用。<br/> false: 不写入表头, true: 写入表头。 |
| encoding | string | 否 | "UTF-8" | 仅当 file_format_type 为 json,text,csv,xml 时使用。 |
| merge_update_event | boolean | 否 | false | 仅当file_format_type为canal_json、debezium_json、maxwell_json. |
+| schema_evolution_enabled | boolean | 否 | false | 开启 Schema 演变支持,适用于 CDC 管道。为 true 时,来自上游的 ADD/DROP/RENAME/MODIFY 列事件无需重启作业即可应用到 Sink。不支持 binary 格式。 |
### path [string]
@@ -516,6 +517,35 @@ sink {
### enable_header_write [boolean]
仅在 file_format_type 为 text 或 csv 时使用。false:不写入表头,true:写入表头。
+
+### schema_evolution_enabled [boolean]
+
+设置为 `true` 时,文件 Sink 可在运行时处理 CDC Schema 变更事件(ADD COLUMN、DROP COLUMN、RENAME COLUMN、MODIFY COLUMN 类型),无需重启作业。每次 Schema 变更时,当前输出文件会被关闭,并以新 Schema 打开一个新文件。
+
+**支持的格式:** 除 `binary` 外的所有文件格式。将此选项与 `file_format_type = binary` 一起使用时,作业启动时会抛出配置校验错误。
+
+**分区约束:** 当 `have_partition = true` 时,不允许删除 `partition_by` 中列出的列,违反时会立即抛出异常。分区列在 Schema 变更过程中必须保持稳定。
+
+**当 `schema_evolution_enabled = false`(默认值)时:** 若上游 CDC Source 配置了 `schema-changes.enabled = true` 且 Sink 收到 `AlterTableEvent`,作业会立即抛出如下错误:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+使用默认 CDC Source 配置(`schema-changes.enabled = false`)的用户不受影响。
+
+**已知限制:** Schema 变更与 Checkpoint 不是原子操作。若作业在文件轮转与 Schema 元数据更新之间的窗口期崩溃,恢复后写入的数据行可能使用变更前的 Schema。这是与其他 SeaTunnel Sink 共同存在的已知架构限制。完整的重启后 DDL 正确性支持需要配套的 CDC Source 修复(另行跟踪)。
+
+CDC 管道中的使用示例:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## 变更日志
<ChangeLog />
\ No newline at end of file
diff --git a/docs/zh/connectors/sink/SftpFile.md b/docs/zh/connectors/sink/SftpFile.md
index 5ee015e5b6..7a4c791fdf 100644
--- a/docs/zh/connectors/sink/SftpFile.md
+++ b/docs/zh/connectors/sink/SftpFile.md
@@ -78,6 +78,7 @@ import ChangeLog from '../changelog/connector-file-sftp.md';
| schema_save_mode | string | 否 | CREATE_SCHEMA_WHEN_NOT_EXIST | 现有目录处理方式 |
| data_save_mode | string | 否 | APPEND_DATA | 现有数据处理方式 |
| merge_update_event | boolean | 否 | false | 仅当file_format_type为canal_json、debezium_json、maxwell_json. |
+| schema_evolution_enabled | boolean | 否 | false | 开启 Schema 演变支持,适用于 CDC 管道。为 true 时,来自上游的 ADD/DROP/RENAME/MODIFY 列事件无需重启作业即可应用到 Sink。不支持 binary 格式。 |
### host [string]
@@ -327,6 +328,35 @@ SftpFile {
```
+
+### schema_evolution_enabled [boolean]
+
+设置为 `true` 时,文件 Sink 可在运行时处理 CDC Schema 变更事件(ADD COLUMN、DROP COLUMN、RENAME COLUMN、MODIFY COLUMN 类型),无需重启作业。每次 Schema 变更时,当前输出文件会被关闭,并以新 Schema 打开一个新文件。
+
+**支持的格式:** 除 `binary` 外的所有文件格式。将此选项与 `file_format_type = binary` 一起使用时,作业启动时会抛出配置校验错误。
+
+**分区约束:** 当 `have_partition = true` 时,不允许删除 `partition_by` 中列出的列,违反时会立即抛出异常。分区列在 Schema 变更过程中必须保持稳定。
+
+**当 `schema_evolution_enabled = false`(默认值)时:** 若上游 CDC Source 配置了 `schema-changes.enabled = true` 且 Sink 收到 `AlterTableEvent`,作业会立即抛出如下错误:
+> `Received AlterTableEvent but schema_evolution_enabled=false at this sink. Either set schema_evolution_enabled=true to handle schema changes, or set schema-changes.enabled=false at the CDC source to suppress them.`
+
+使用默认 CDC Source 配置(`schema-changes.enabled = false`)的用户不受影响。
+
+**已知限制:** Schema 变更与 Checkpoint 不是原子操作。若作业在文件轮转与 Schema 元数据更新之间的窗口期崩溃,恢复后写入的数据行可能使用变更前的 Schema。这是与其他 SeaTunnel Sink 共同存在的已知架构限制。完整的重启后 DDL 正确性支持需要配套的 CDC Source 修复(另行跟踪)。
+
+CDC 管道中的使用示例:
+
+```hocon
+LocalFile {
+ path = "/tmp/cdc/${table_name}"
+ file_format_type = "parquet"
+ schema_evolution_enabled = true
+ have_partition = true
+ partition_by = ["updated_at_month"]
+}
+```
+
+
## 变更日志
<ChangeLog />
\ No newline at end of file
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/transform/SeaTunnelTransform.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/transform/SeaTunnelTransform.java
index b12d69bafb..cfa7d50f30 100644
--- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/transform/SeaTunnelTransform.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/transform/SeaTunnelTransform.java
@@ -52,6 +52,21 @@ public interface SeaTunnelTransform<T>
return schemaChangeEvent;
}
+ /**
+ * Called by the engine when an upstream transform's produced schema changes (e.g., after a
+ * schema-change event flows through the chain). Allows this transform to re-derive its state
+ * from the new upstream layout instead of applying schema changes to a stale local view.
+ *
+ * <p>Without this, downstream transforms in a chain accumulate divergence: each one applies the
+ * ALTER event locally (appending new cols at end of its own catalog), but the actual data row
+ * from upstream has new cols at the position upstream put them. The result is row-vs-catalog
+ * order divergence that breaks name-based field access (SQL projections, FilterField excludes).
+ *
+ * <p>Default implementation is a no-op for backwards compatibility with transforms that don't
+ * carry upstream-position-dependent state.
+ */
+ default void setInputCatalogTables(List<CatalogTable> inputCatalogTables) {}
+
/** call it when Transformer completed */
default void close() {}
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSinkOptions.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSinkOptions.java
index 462d467127..ef89f84153 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSinkOptions.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSinkOptions.java
@@ -336,4 +336,14 @@ public class FileBaseSinkOptions extends FileBaseOptions {
.defaultValue(false)
.withDescription(
"Only used when file_format_type is canal_json,debezium_json,maxwell_json. set true,then when serialize data,UPDATE_AFTER and UPDATE_BEFORE event will merge into UPDATE data;if set false, when serialize data will get UPDATE_AFTER and UPDATE_BEFORE event ");
+
+ public static final Option<Boolean> SCHEMA_EVOLUTION_ENABLED =
+ Options.key("schema_evolution_enabled")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Whether to enable schema evolution for the file sink. "
+ + "When true, ALTER TABLE events (ADD/DROP/RENAME/UPDATE column) "
+ + "from CDC sources are applied to the sink schema at runtime. "
+ + "Files are rotated at each schema change boundary. Default: false.");
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/BaseFileSinkWriter.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/BaseFileSinkWriter.java
index 6849205009..7fc9f6b135 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/BaseFileSinkWriter.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/BaseFileSinkWriter.java
@@ -21,6 +21,8 @@ import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils;
import org.apache.seatunnel.api.sink.SinkWriter;
import org.apache.seatunnel.api.sink.SupportMultiTableSinkWriter;
+import org.apache.seatunnel.api.sink.SupportSchemaEvolutionSinkWriter;
+import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.Constants;
import org.apache.seatunnel.common.exception.CommonError;
@@ -52,7 +54,8 @@ import static org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSink
public class BaseFileSinkWriter
implements SinkWriter<SeaTunnelRow, FileCommitInfo, FileSinkState>,
- SupportMultiTableSinkWriter<WriteStrategy> {
+ SupportMultiTableSinkWriter<WriteStrategy>,
+ SupportSchemaEvolutionSinkWriter {
protected final WriteStrategy writeStrategy;
@@ -199,6 +202,11 @@ public class BaseFileSinkWriter
return writeStrategy.snapshotState(checkpointId);
}
+ @Override
+ public void applySchemaChange(SchemaChangeEvent event) throws IOException {
+ writeStrategy.applySchemaChange(event);
+ }
+
@Override
public void close() throws IOException {
if (writeStrategy != null) {
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/BaseMultipleTableFileSink.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/BaseMultipleTableFileSink.java
index cd2f9b1f10..075cdf84c2 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/BaseMultipleTableFileSink.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/BaseMultipleTableFileSink.java
@@ -30,9 +30,11 @@ import org.apache.seatunnel.api.sink.SinkAggregatedCommitter;
import org.apache.seatunnel.api.sink.SinkWriter;
import org.apache.seatunnel.api.sink.SupportMultiTableSink;
import org.apache.seatunnel.api.sink.SupportSaveMode;
+import org.apache.seatunnel.api.sink.SupportSchemaEvolutionSink;
import org.apache.seatunnel.api.table.catalog.Catalog;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.seatunnel.api.table.factory.CatalogFactory;
+import org.apache.seatunnel.api.table.schema.SchemaChangeType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSinkOptions;
import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf;
@@ -44,6 +46,8 @@ import org.apache.seatunnel.connectors.seatunnel.file.sink.state.FileSinkState;
import org.apache.seatunnel.connectors.seatunnel.file.sink.writer.WriteStrategy;
import org.apache.seatunnel.connectors.seatunnel.file.sink.writer.WriteStrategyFactory;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Optional;
@@ -53,7 +57,10 @@ public abstract class BaseMultipleTableFileSink
implements SeaTunnelSink<
SeaTunnelRow, FileSinkState, FileCommitInfo, FileAggregatedCommitInfo>,
SupportMultiTableSink,
- SupportSaveMode {
+ SupportSaveMode,
+ SupportSchemaEvolutionSink {
+
+ private static final long serialVersionUID = 1L;
private final HadoopConf hadoopConf;
private final CatalogTable catalogTable;
@@ -121,6 +128,18 @@ public abstract class BaseMultipleTableFileSink
return Optional.of(new DefaultSerializer<>());
}
+ @Override
+ public List<SchemaChangeType> supports() {
+ if (!fileSinkConfig.isSchemaEvolutionEnabled()) {
+ return Collections.emptyList();
+ }
+ return Arrays.asList(
+ SchemaChangeType.ADD_COLUMN,
+ SchemaChangeType.DROP_COLUMN,
+ SchemaChangeType.RENAME_COLUMN,
+ SchemaChangeType.UPDATE_COLUMN);
+ }
+
protected WriteStrategy createWriteStrategy() {
WriteStrategy writeStrategy =
WriteStrategyFactory.of(fileSinkConfig.getFileFormat(), fileSinkConfig);
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/config/FileSinkConfig.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/config/FileSinkConfig.java
index 3a884c6e4b..400ceba119 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/config/FileSinkConfig.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/config/FileSinkConfig.java
@@ -26,6 +26,7 @@ import org.apache.seatunnel.connectors.seatunnel.file.config.BaseFileSinkConfig;
import org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSinkOptions;
import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat;
import org.apache.seatunnel.connectors.seatunnel.file.config.PartitionConfig;
+import org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorErrorCode;
import org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorException;
import org.apache.seatunnel.format.csv.constant.CsvStringQuoteMode;
@@ -88,6 +89,9 @@ public class FileSinkConfig extends BaseFileSinkConfig implements PartitionConfi
private Boolean mergeUpdateEvent;
+ private boolean schemaEvolutionEnabled =
+ FileBaseSinkOptions.SCHEMA_EVOLUTION_ENABLED.defaultValue();
+
public FileSinkConfig(
@NonNull ReadonlyConfig pluginConfig, @NonNull SeaTunnelRowType seaTunnelRowTypeInfo) {
super(pluginConfig);
@@ -217,5 +221,18 @@ public class FileSinkConfig extends BaseFileSinkConfig implements PartitionConfi
|| FileFormat.MAXWELL_JSON.equals(this.fileFormat)) {
this.mergeUpdateEvent = pluginConfig.get(FileBaseSinkOptions.MERGE_UPDATE_EVENT);
}
+
+ this.schemaEvolutionEnabled =
+ pluginConfig.get(FileBaseSinkOptions.SCHEMA_EVOLUTION_ENABLED);
+ if (this.schemaEvolutionEnabled && FileFormat.BINARY.equals(this.fileFormat)) {
+ throw new FileConnectorException(
+ FileConnectorErrorCode.FORMAT_NOT_SUPPORT,
+ "schema_evolution_enabled=true is not supported for file_format_type=binary."
+ + " Binary format has a fixed schema and cannot apply schema changes.");
+ }
+ // schema_evolution + partition_by is supported: AbstractWriteStrategy.applySchemaChange
+ // rebuilds both sinkColumnsIndexInRow and partitionFieldsIndexInRow from column NAMES
+ // against the post-ALTER row type. Dropping a partition column itself is rejected at
+ // rebuild time (throws IllegalStateException) so the partition tree never corrupts.
}
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/util/ExcelGenerator.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/util/ExcelGenerator.java
index 4498e1718f..d69e26e323 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/util/ExcelGenerator.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/util/ExcelGenerator.java
@@ -122,7 +122,7 @@ public class ExcelGenerator {
int index = 0;
for (Integer i : sinkColumnsIndexInRow) {
Cell cell = excelRow.createCell(index++);
- Object value = seaTunnelRow.getField(i);
+ Object value = i < seaTunnelRow.getArity() ? seaTunnelRow.getField(i) : null;
setCellValue(fieldTypes[i], seaTunnelRowType.getFieldName(i), value, cell);
}
currentRowInSheet++;
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/util/XmlWriter.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/util/XmlWriter.java
index b548359442..19a35b2142 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/util/XmlWriter.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/util/XmlWriter.java
@@ -75,7 +75,9 @@ public class XmlWriter {
new AbstractMap.SimpleEntry<>(
seaTunnelRowType.getFieldName(index),
convertToXmlString(
- seaTunnelRow.getField(index),
+ index < seaTunnelRow.getArity()
+ ? seaTunnelRow.getField(index)
+ : null,
seaTunnelRowType.getFieldType(index))))
.forEach(
entry -> {
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java
index e4fcfa9bbb..b029c75144 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java
@@ -21,6 +21,15 @@ import org.apache.seatunnel.shade.com.google.common.collect.Lists;
import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.TableSchema;
+import org.apache.seatunnel.api.table.schema.event.AlterTableAddColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableChangeColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableColumnsEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableDropColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableModifyColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent;
+import org.apache.seatunnel.api.table.schema.handler.AlterTableSchemaEventHandler;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
@@ -65,7 +74,14 @@ public abstract class AbstractWriteStrategy<T> implements WriteStrategy<T> {
protected final Logger log = LoggerFactory.getLogger(this.getClass());
protected final FileSinkConfig fileSinkConfig;
protected final CompressFormat compressFormat;
- protected final List<Integer> sinkColumnsIndexInRow;
+ // Non-final: rebuilt on each applySchemaChange call. Defensive copy in constructor.
+ protected List<Integer> sinkColumnsIndexInRow;
+ // Tracks which column names to write. Updated on each schema change event.
+ protected List<String> sinkColumnNames;
+ // Writer-local partition indices: rebuilt on each applySchemaChange so that ADD COLUMN before
+ // a partition column does not leave the partition key reading stale row positions. Defensive
+ // copy in constructor — fileSinkConfig is shared across writer instances.
+ protected List<Integer> partitionFieldsIndexInRow;
protected String jobId;
protected int subTaskIndex;
protected HadoopConf hadoopConf;
@@ -79,6 +95,7 @@ public abstract class AbstractWriteStrategy<T> implements WriteStrategy<T> {
protected LinkedHashMap<String, String> beingWrittenFile = new LinkedHashMap<>();
private LinkedHashMap<String, List<String>> partitionDirAndValuesMap;
protected SeaTunnelRowType seaTunnelRowType;
+ protected TableSchema tableSchema;
// Checkpoint id from engine is start with 1
protected Long checkpointId = 0L;
@@ -89,7 +106,14 @@ public abstract class AbstractWriteStrategy<T> implements WriteStrategy<T> {
public AbstractWriteStrategy(FileSinkConfig fileSinkConfig) {
this.fileSinkConfig = fileSinkConfig;
- this.sinkColumnsIndexInRow = fileSinkConfig.getSinkColumnsIndexInRow();
+ // Defensive copies — mutations in applySchemaChange must not affect FileSinkConfig
+ // (FileSinkConfig instance is shared across writer instances in multi-writer mode)
+ this.sinkColumnsIndexInRow = new ArrayList<>(fileSinkConfig.getSinkColumnsIndexInRow());
+ this.sinkColumnNames = new ArrayList<>(fileSinkConfig.getSinkColumnList());
+ this.partitionFieldsIndexInRow =
+ fileSinkConfig.getPartitionFieldsIndexInRow() == null
+ ? new ArrayList<>()
+ : new ArrayList<>(fileSinkConfig.getPartitionFieldsIndexInRow());
this.batchSize = fileSinkConfig.getBatchSize();
this.compressFormat = fileSinkConfig.getCompressFormat();
this.singleFileMode = fileSinkConfig.isSingleFileMode();
@@ -160,9 +184,239 @@ public abstract class AbstractWriteStrategy<T> implements WriteStrategy<T> {
*/
@Override
public void setCatalogTable(CatalogTable catalogTable) {
+ this.tableSchema = catalogTable.getTableSchema();
this.seaTunnelRowType = catalogTable.getSeaTunnelRowType();
}
+ // ── Schema Evolution ──────────────────────────────────────────────────────────
+
+ @Override
+ public void applySchemaChange(SchemaChangeEvent event) throws IOException {
+ if (!fileSinkConfig.isSchemaEvolutionEnabled()) {
+ throw new UnsupportedOperationException(
+ "Received AlterTableEvent but schema_evolution_enabled=false at this sink. "
+ + "Either set schema_evolution_enabled=true to handle schema changes, "
+ + "or set schema-changes.enabled=false at the CDC source to suppress them.");
+ }
+ log.info(
+ "[FileSchemaEvolution] applying {} — before: rowType={}, sinkColumns={}, indices={}",
+ event.getClass().getSimpleName(),
+ seaTunnelRowType,
+ sinkColumnNames,
+ sinkColumnsIndexInRow);
+
+ // Step 1: flush + close all open writers; register partial files for commit.
+ // Reuses the subclass finishAndCloseFile() — Parquet and ORC each close their own
+ // beingWrittenWriter map entries and add to needMoveFiles there.
+ // beingWrittenFile.clear() is in a finally block to guarantee it runs even if a writer
+ // fails to close, so stale path entries never accumulate in the cache.
+ try {
+ finishAndCloseFile();
+ } finally {
+ beingWrittenFile.clear();
+ }
+
+ // Step 2: adopt the upstream's actual produced schema, propagated through the chain via
+ // event.changeAfter (each transform's mapSchemaChangeEvent updates this field with its own
+ // getProducedCatalogTable). This avoids order-divergence — applying ALTER locally would
+ // append new cols at the end of the sink's catalog, but upstream's actual row has new
+ // cols at the position upstream put them. Reading changeAfter directly aligns the sink's
+ // catalog with the actual row layout.
+ if (event.getChangeAfter() != null) {
+ this.tableSchema = event.getChangeAfter().getTableSchema();
+ } else {
+ // Fallback for events without changeAfter (older sources or events that bypass the
+ // transform chain). Apply ALTER locally as before.
+ this.tableSchema = new AlterTableSchemaEventHandler().reset(tableSchema).apply(event);
+ }
+ this.seaTunnelRowType = tableSchema.toPhysicalRowDataType();
+
+ // Step 3: update sinkColumnNames to reflect the structural change.
+ updateSinkColumnNames(event);
+
+ // Step 4: rebuild sinkColumnsIndexInRow from sinkColumnNames + new seaTunnelRowType.
+ this.sinkColumnsIndexInRow = rebuildSinkColumnsIndex();
+
+ // Step 4b: rebuild partitionFieldsIndexInRow so partition keys read from correct row
+ // positions after ADD/DROP column shifts. Without this, ADD COLUMN before a partition
+ // field causes generatorPartitionDir() to read the wrong field for the partition value.
+ this.partitionFieldsIndexInRow = rebuildPartitionFieldsIndex();
+
+ // Step 5: let subclasses invalidate any format-specific cached schema objects.
+ onSchemaChanged();
+
+ log.info(
+ "[FileSchemaEvolution] applied — after: rowType={}, sinkColumns={}, sinkIndices={}, partitionIndices={}",
+ seaTunnelRowType,
+ sinkColumnNames,
+ sinkColumnsIndexInRow,
+ partitionFieldsIndexInRow);
+ }
+
+ /**
+ * Hook for subclasses to invalidate format-specific cached schema objects after a schema
+ * change. Default is a no-op. Override in ParquetWriteStrategy to null the cached Avro schema.
+ * ORC does not need this because it calls buildSchemaWithRowType() on every write().
+ */
+ protected void onSchemaChanged() {
+ // no-op by default
+ }
+
+ /**
+ * Bounds-safe row field accessor. Returns null when {@code index >= row.getArity()}.
+ *
+ * <p>Used in write() loops of all format strategies to handle in-flight rows that were
+ * serialised against an old (shorter) schema before a schema change event was processed.
+ * Without this guard, row.getField(index) would throw ArrayIndexOutOfBoundsException.
+ *
+ * @param row the SeaTunnelRow being written
+ * @param index the field index from sinkColumnsIndexInRow
+ * @return the field value, or null if the row predates the schema change
+ */
+ protected Object getFieldSafe(SeaTunnelRow row, int index) {
+ if (index >= row.getArity()) {
+ return null;
+ }
+ return row.getField(index);
+ }
+
+ /**
+ * Projects a row to only the sink columns, substituting null for any column whose index exceeds
+ * the row's arity. This mirrors {@link SeaTunnelRow#copy(int[])} but is safe for in-flight
+ * old-schema rows arriving after an ADD_COLUMN schema change event.
+ */
+ protected SeaTunnelRow safeProjectedRow(SeaTunnelRow row) {
+ int[] indexMapping = sinkColumnsIndexInRow.stream().mapToInt(Integer::intValue).toArray();
+ Object[] newFields = new Object[indexMapping.length];
+ for (int i = 0; i < indexMapping.length; i++) {
+ newFields[i] = indexMapping[i] < row.getArity() ? row.getField(indexMapping[i]) : null;
+ }
+ SeaTunnelRow newRow = new SeaTunnelRow(newFields);
+ newRow.setRowKind(row.getRowKind());
+ newRow.setTableId(row.getTableId());
+ newRow.setOptions(row.getOptions());
+ return newRow;
+ }
+
+ private void updateSinkColumnNames(SchemaChangeEvent event) {
+ if (event instanceof AlterTableAddColumnEvent) {
+ AlterTableAddColumnEvent e = (AlterTableAddColumnEvent) event;
+ String newCol = e.getColumn().getName();
+ // Case-insensitive duplicate check — CDC sources may send names in any case
+ boolean alreadyPresent =
+ sinkColumnNames.stream().anyMatch(c -> c.equalsIgnoreCase(newCol));
+ if (!alreadyPresent) {
+ if (e.isFirst()) {
+ sinkColumnNames.add(0, newCol);
+ } else if (e.getAfterColumn() != null) {
+ int pos = indexOfIgnoreCase(sinkColumnNames, e.getAfterColumn());
+ sinkColumnNames.add(pos >= 0 ? pos + 1 : sinkColumnNames.size(), newCol);
+ } else {
+ sinkColumnNames.add(newCol);
+ }
+ }
+ } else if (event instanceof AlterTableDropColumnEvent) {
+ // Case-insensitive removal: CDC sources may send column names in any case
+ String toDrop = ((AlterTableDropColumnEvent) event).getColumn();
+ boolean removed = sinkColumnNames.removeIf(c -> c.equalsIgnoreCase(toDrop));
+ if (!removed) {
+ log.warn(
+ "[FileSchemaEvolution] DROP event references unknown column '{}' — ignored",
+ toDrop);
+ }
+ } else if (event instanceof AlterTableChangeColumnEvent) {
+ // RENAME (and optional MOVE): remove old name, insert new name at target position.
+ // AlterTableChangeColumnEvent covers MySQL's CHANGE COLUMN which can rename and
+ // reorder in one statement. Both the rename and the position change are applied.
+ AlterTableChangeColumnEvent e = (AlterTableChangeColumnEvent) event;
+ int idx = indexOfIgnoreCase(sinkColumnNames, e.getOldColumn());
+ if (idx >= 0) {
+ sinkColumnNames.remove(idx);
+ String newName = e.getColumn().getName();
+ if (e.isFirst()) {
+ sinkColumnNames.add(0, newName);
+ } else if (e.getAfterColumn() != null) {
+ int afterIdx = indexOfIgnoreCase(sinkColumnNames, e.getAfterColumn());
+ sinkColumnNames.add(
+ afterIdx >= 0 ? afterIdx + 1 : sinkColumnNames.size(), newName);
+ } else {
+ // No position change — restore at the same index
+ sinkColumnNames.add(idx, newName);
+ }
+ }
+ } else if (event instanceof AlterTableModifyColumnEvent) {
+ // TYPE change only — column name is unchanged, no list update needed
+ } else if (event instanceof AlterTableColumnsEvent) {
+ // Batch: apply each sub-event in order
+ for (AlterTableColumnEvent sub : ((AlterTableColumnsEvent) event).getEvents()) {
+ updateSinkColumnNames(sub);
+ }
+ }
+ }
+
+ /** Case-insensitive indexOf for a list of column names. Returns -1 if not found. */
+ private static int indexOfIgnoreCase(List<String> list, String name) {
+ for (int i = 0; i < list.size(); i++) {
+ if (list.get(i).equalsIgnoreCase(name)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ private List<Integer> rebuildSinkColumnsIndex() {
+ String[] fieldNames = seaTunnelRowType.getFieldNames();
+ Map<String, Integer> nameToIndex = new HashMap<>(fieldNames.length);
+ for (int i = 0; i < fieldNames.length; i++) {
+ nameToIndex.put(fieldNames[i].toLowerCase(), i);
+ }
+ List<Integer> newIndex = new ArrayList<>(sinkColumnNames.size());
+ for (String col : sinkColumnNames) {
+ Integer idx = nameToIndex.get(col.toLowerCase());
+ if (idx != null) {
+ newIndex.add(idx);
+ }
+ // Column not found in new rowType (e.g. just dropped) — skip silently
+ }
+ return newIndex;
+ }
+
+ /**
+ * Rebuilds partition-field row indices from the configured partition column names against the
+ * post-ALTER {@link #seaTunnelRowType}. Partition column NAMES are immutable (set once at job
+ * start); only their row positions can shift when other columns are added/dropped.
+ *
+ * <p>Throws if a configured partition column has been dropped from the schema — a dropped
+ * partition key would corrupt the partition tree, so we fail fast rather than silently writing
+ * to the wrong directory.
+ */
+ private List<Integer> rebuildPartitionFieldsIndex() {
+ List<String> partitionFieldList = fileSinkConfig.getPartitionFieldList();
+ if (partitionFieldList == null || partitionFieldList.isEmpty()) {
+ return new ArrayList<>();
+ }
+ String[] fieldNames = seaTunnelRowType.getFieldNames();
+ Map<String, Integer> nameToIndex = new HashMap<>(fieldNames.length);
+ for (int i = 0; i < fieldNames.length; i++) {
+ nameToIndex.put(fieldNames[i].toLowerCase(), i);
+ }
+ List<Integer> newIndex = new ArrayList<>(partitionFieldList.size());
+ for (String col : partitionFieldList) {
+ Integer idx = nameToIndex.get(col.toLowerCase());
+ if (idx == null) {
+ throw new IllegalStateException(
+ "Schema evolution: partition column ["
+ + col
+ + "] is missing from the post-ALTER schema. "
+ + "Dropping or renaming a partition column is not supported. "
+ + "Available columns: "
+ + java.util.Arrays.toString(fieldNames));
+ }
+ newIndex.add(idx);
+ }
+ return newIndex;
+ }
+
/**
* use seaTunnelRow generate partition directory
*
@@ -171,7 +425,9 @@ public abstract class AbstractWriteStrategy<T> implements WriteStrategy<T> {
*/
@Override
public LinkedHashMap<String, List<String>> generatorPartitionDir(SeaTunnelRow seaTunnelRow) {
- List<Integer> partitionFieldsIndexInRow = fileSinkConfig.getPartitionFieldsIndexInRow();
+ // Use writer-local indices (not fileSinkConfig.getPartitionFieldsIndexInRow()): these get
+ // rebuilt in applySchemaChange so they stay aligned with the post-ALTER row layout.
+ List<Integer> partitionFieldsIndexInRow = this.partitionFieldsIndexInRow;
LinkedHashMap<String, List<String>> partitionDirAndValuesMap = new LinkedHashMap<>(1);
if (CollectionUtils.isEmpty(partitionFieldsIndexInRow)) {
partitionDirAndValuesMap.put(FileBaseSinkOptions.NON_PARTITION, null);
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/BinaryWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/BinaryWriteStrategy.java
index d1904f3c94..eb27a17a9a 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/BinaryWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/BinaryWriteStrategy.java
@@ -18,6 +18,7 @@
package org.apache.seatunnel.connectors.seatunnel.file.sink.writer;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.exception.CommonError;
import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated;
@@ -142,6 +143,14 @@ public class BinaryWriteStrategy extends AbstractWriteStrategy<FSDataOutputStrea
return fsDataOutputStream;
}
+ @Override
+ public void applySchemaChange(SchemaChangeEvent event) {
+ throw new FileConnectorException(
+ FileConnectorErrorCode.FORMAT_NOT_SUPPORT,
+ "BinaryWriteStrategy does not support schema evolution. "
+ + "Binary format requires a fixed schema.");
+ }
+
@Override
public void finishAndCloseFile() {
beingWrittenOutputStream.forEach(
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/CanalJsonWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/CanalJsonWriteStrategy.java
index f40e2fce90..a7e1082d15 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/CanalJsonWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/CanalJsonWriteStrategy.java
@@ -67,18 +67,22 @@ public class CanalJsonWriteStrategy extends AbstractWriteStrategy<FSDataOutputSt
mergeUpdateEventFlag);
}
+ @Override
+ protected void onSchemaChanged() {
+ this.serializationSchema =
+ new CanalJsonSerializationSchema(
+ buildSchemaWithRowType(seaTunnelRowType, sinkColumnsIndexInRow),
+ charset,
+ mergeUpdateEventFlag);
+ }
+
@Override
public void write(@NonNull SeaTunnelRow seaTunnelRow) {
super.write(seaTunnelRow);
String filePath = getOrCreateFilePathBeingWritten(seaTunnelRow);
FSDataOutputStream fsDataOutputStream = getOrCreateOutputStream(filePath);
try {
- byte[] rowBytes =
- serializationSchema.serialize(
- seaTunnelRow.copy(
- sinkColumnsIndexInRow.stream()
- .mapToInt(Integer::intValue)
- .toArray()));
+ byte[] rowBytes = serializationSchema.serialize(safeProjectedRow(seaTunnelRow));
if (rowBytes == null) {
return;
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/CsvWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/CsvWriteStrategy.java
index 9c0dabf708..74e9bd0789 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/CsvWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/CsvWriteStrategy.java
@@ -91,6 +91,21 @@ public class CsvWriteStrategy extends AbstractWriteStrategy<FSDataOutputStream>
.build();
}
+ @Override
+ protected void onSchemaChanged() {
+ this.serializationSchema =
+ CsvSerializationSchema.builder()
+ .seaTunnelRowType(
+ buildSchemaWithRowType(seaTunnelRowType, sinkColumnsIndexInRow))
+ .delimiter(fieldDelimiter)
+ .dateFormatter(dateFormat)
+ .dateTimeFormatter(dateTimeFormat)
+ .timeFormatter(timeFormat)
+ .charset(charset)
+ .quoteMode(csvStringQuoteMode)
+ .build();
+ }
+
@Override
public void write(@NonNull SeaTunnelRow seaTunnelRow) {
super.write(seaTunnelRow);
@@ -102,12 +117,7 @@ public class CsvWriteStrategy extends AbstractWriteStrategy<FSDataOutputStream>
} else {
fsDataOutputStream.write(rowDelimiter.getBytes(charset));
}
- fsDataOutputStream.write(
- serializationSchema.serialize(
- seaTunnelRow.copy(
- sinkColumnsIndexInRow.stream()
- .mapToInt(Integer::intValue)
- .toArray())));
+ fsDataOutputStream.write(serializationSchema.serialize(safeProjectedRow(seaTunnelRow)));
} catch (IOException e) {
throw CommonError.fileOperationFailed("CsvFile", "write", filePath, e);
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/DebeziumJsonWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/DebeziumJsonWriteStrategy.java
index 5aa0a86445..8c26b57c30 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/DebeziumJsonWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/DebeziumJsonWriteStrategy.java
@@ -67,18 +67,22 @@ public class DebeziumJsonWriteStrategy extends AbstractWriteStrategy<FSDataOutpu
mergeUpdateEventFlag);
}
+ @Override
+ protected void onSchemaChanged() {
+ this.serializationSchema =
+ new DebeziumJsonSerializationSchema(
+ buildSchemaWithRowType(seaTunnelRowType, sinkColumnsIndexInRow),
+ charset,
+ mergeUpdateEventFlag);
+ }
+
@Override
public void write(@NonNull SeaTunnelRow seaTunnelRow) {
super.write(seaTunnelRow);
String filePath = getOrCreateFilePathBeingWritten(seaTunnelRow);
FSDataOutputStream fsDataOutputStream = getOrCreateOutputStream(filePath);
try {
- byte[] rowBytes =
- serializationSchema.serialize(
- seaTunnelRow.copy(
- sinkColumnsIndexInRow.stream()
- .mapToInt(Integer::intValue)
- .toArray()));
+ byte[] rowBytes = serializationSchema.serialize(safeProjectedRow(seaTunnelRow));
if (rowBytes == null) {
return;
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/JsonWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/JsonWriteStrategy.java
index bc4a08b2d8..4b4bf421f2 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/JsonWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/JsonWriteStrategy.java
@@ -64,18 +64,20 @@ public class JsonWriteStrategy extends AbstractWriteStrategy<FSDataOutputStream>
charset);
}
+ @Override
+ protected void onSchemaChanged() {
+ this.serializationSchema =
+ new JsonSerializationSchema(
+ buildSchemaWithRowType(seaTunnelRowType, sinkColumnsIndexInRow), charset);
+ }
+
@Override
public void write(@NonNull SeaTunnelRow seaTunnelRow) {
super.write(seaTunnelRow);
String filePath = getOrCreateFilePathBeingWritten(seaTunnelRow);
FSDataOutputStream fsDataOutputStream = getOrCreateOutputStream(filePath);
try {
- byte[] rowBytes =
- serializationSchema.serialize(
- seaTunnelRow.copy(
- sinkColumnsIndexInRow.stream()
- .mapToInt(Integer::intValue)
- .toArray()));
+ byte[] rowBytes = serializationSchema.serialize(safeProjectedRow(seaTunnelRow));
if (isFirstWrite.get(filePath)) {
isFirstWrite.put(filePath, false);
} else {
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/MaxWellJsonWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/MaxWellJsonWriteStrategy.java
index 693e07fc1e..bf69d02618 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/MaxWellJsonWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/MaxWellJsonWriteStrategy.java
@@ -67,18 +67,22 @@ public class MaxWellJsonWriteStrategy extends AbstractWriteStrategy<FSDataOutput
mergeUpdateEventFlag);
}
+ @Override
+ protected void onSchemaChanged() {
+ this.serializationSchema =
+ new MaxWellJsonSerializationSchema(
+ buildSchemaWithRowType(seaTunnelRowType, sinkColumnsIndexInRow),
+ charset,
+ mergeUpdateEventFlag);
+ }
+
@Override
public void write(@NonNull SeaTunnelRow seaTunnelRow) {
super.write(seaTunnelRow);
String filePath = getOrCreateFilePathBeingWritten(seaTunnelRow);
FSDataOutputStream fsDataOutputStream = getOrCreateOutputStream(filePath);
try {
- byte[] rowBytes =
- serializationSchema.serialize(
- seaTunnelRow.copy(
- sinkColumnsIndexInRow.stream()
- .mapToInt(Integer::intValue)
- .toArray()));
+ byte[] rowBytes = serializationSchema.serialize(safeProjectedRow(seaTunnelRow));
if (rowBytes == null) {
return;
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/OrcWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/OrcWriteStrategy.java
index 339067e1c8..50bb69eea8 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/OrcWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/OrcWriteStrategy.java
@@ -55,6 +55,7 @@ import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.temporal.ChronoField;
+import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -79,7 +80,7 @@ public class OrcWriteStrategy extends AbstractWriteStrategy<Writer> {
int i = 0;
int row = rowBatch.size++;
for (Integer index : sinkColumnsIndexInRow) {
- Object value = seaTunnelRow.getField(index);
+ Object value = getFieldSafe(seaTunnelRow, index);
ColumnVector vector = rowBatch.cols[i];
setColumn(value, vector, row);
i++;
@@ -97,6 +98,7 @@ public class OrcWriteStrategy extends AbstractWriteStrategy<Writer> {
@Override
public void finishAndCloseFile() {
+ List<FileConnectorException> closeErrors = new ArrayList<>();
this.beingWrittenWriter.forEach(
(k, v) -> {
try {
@@ -106,18 +108,23 @@ public class OrcWriteStrategy extends AbstractWriteStrategy<Writer> {
rowBatch.reset();
}
v.close();
+ needMoveFiles.put(k, getTargetLocation(k));
} catch (IOException e) {
- String errorMsg =
- String.format(
- "Close file [%s] orc writer failed, error msg: [%s]",
- k, e.getMessage());
- throw new FileConnectorException(
- CommonErrorCodeDeprecated.WRITER_OPERATION_FAILED, errorMsg, e);
+ closeErrors.add(
+ new FileConnectorException(
+ CommonErrorCodeDeprecated.WRITER_OPERATION_FAILED,
+ String.format(
+ "Close file [%s] orc writer failed, error"
+ + " msg: [%s]",
+ k, e.getMessage()),
+ e));
}
- needMoveFiles.put(k, getTargetLocation(k));
});
this.vectorizedRowBatches.clear();
this.beingWrittenWriter.clear();
+ if (!closeErrors.isEmpty()) {
+ throw closeErrors.get(0);
+ }
}
private VectorizedRowBatch getOrCreateVectorizedRowBatch(@NonNull String filePath) {
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
index 41155fd829..d8ed92d5fb 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
@@ -79,6 +79,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
private AvroSchemaConverter schemaConverter;
private Schema schema;
private Set<String> writePathsAsInt96;
+
public static final int[] PRECISION_TO_BYTE_COUNT = new int[38];
static {
@@ -123,7 +124,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
for (Integer integer : sinkColumnsIndexInRow) {
String fieldName = seaTunnelRowType.getFieldName(integer);
- Object field = seaTunnelRow.getField(integer);
+ Object field = getFieldSafe(seaTunnelRow, integer);
recordBuilder.set(
fieldName.toLowerCase(),
resolveObject(fieldName, field, seaTunnelRowType.getFieldType(integer)));
@@ -138,21 +139,27 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
@Override
public void finishAndCloseFile() {
+ List<FileConnectorException> closeErrors = new ArrayList<>();
this.beingWrittenWriter.forEach(
(k, v) -> {
try {
v.close();
+ needMoveFiles.put(k, getTargetLocation(k));
} catch (IOException e) {
- String errorMsg =
- String.format(
- "Close file [%s] parquet writer failed, error msg: [%s]",
- k, e.getMessage());
- throw new FileConnectorException(
- CommonErrorCodeDeprecated.WRITER_OPERATION_FAILED, errorMsg, e);
+ closeErrors.add(
+ new FileConnectorException(
+ CommonErrorCodeDeprecated.WRITER_OPERATION_FAILED,
+ String.format(
+ "Close file [%s] parquet writer failed, error"
+ + " msg: [%s]",
+ k, e.getMessage()),
+ e));
}
- needMoveFiles.put(k, getTargetLocation(k));
});
this.beingWrittenWriter.clear();
+ if (!closeErrors.isEmpty()) {
+ throw closeErrors.get(0);
+ }
}
@Override
@@ -221,6 +228,11 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
case MAP:
case STRING:
case BOOLEAN:
+ // CDC maps tinyint(1) to BOOLEAN but sends Byte — coerce to Boolean for Avro
+ if (data instanceof Byte) {
+ return ((Byte) data) != 0;
+ }
+ return data;
case TINYINT:
case SMALLINT:
case INT:
@@ -252,7 +264,8 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
calendar.get(Calendar.MILLISECOND));
NanoTime nanoTime = new NanoTime(julianDays, timeOfDayNanos);
return new GenericData.Fixed(
- schema.getField(name).schema(), nanoTime.toBinary().getBytes());
+ schema.getField(name.toLowerCase()).schema(),
+ nanoTime.toBinary().getBytes());
}
return ((LocalDateTime) data)
.atZone(ZoneId.systemDefault())
@@ -260,7 +273,8 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
.toEpochMilli();
case BYTES:
if (writePathsAsInt96.contains(name)) {
- return new GenericData.Fixed(schema.getField(name).schema(), (byte[]) data);
+ return new GenericData.Fixed(
+ schema.getField(name.toLowerCase()).schema(), (byte[]) data);
}
return ByteBuffer.wrap((byte[]) data);
case ROW:
@@ -409,6 +423,24 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
}
}
+ @Override
+ protected void onSchemaChanged() {
+ this.schema = null;
+ // Rebuild writePathsAsInt96 so that any TIMESTAMP columns added via schema evolution
+ // are included. Without this, new TIMESTAMP columns get INT64 encoding instead of INT96
+ // when parquetWriteTimestampAsInt96=true — silent data corruption.
+ if (fileSinkConfig.getParquetWriteTimestampAsInt96()) {
+ writePathsAsInt96 = new HashSet<>(fileSinkConfig.getParquetAvroWriteFixedAsInt96());
+ for (int i = 0; i < seaTunnelRowType.getTotalFields(); i++) {
+ if (SqlType.TIMESTAMP.equals(seaTunnelRowType.getFieldType(i).getSqlType())) {
+ writePathsAsInt96.add(seaTunnelRowType.getFieldName(i));
+ }
+ }
+ }
+ // Note: if only parquetAvroWriteFixedAsInt96 (BYTES columns) is configured,
+ // those are static column names from config and don't need rebuilding.
+ }
+
private Schema buildAvroSchemaWithRowType(
SeaTunnelRowType seaTunnelRowType, List<Integer> sinkColumnsIndex) {
ArrayList<Type> types = new ArrayList<>();
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/TextWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/TextWriteStrategy.java
index 903b3c027c..df3ab9c4f1 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/TextWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/TextWriteStrategy.java
@@ -86,6 +86,20 @@ public class TextWriteStrategy extends AbstractWriteStrategy<FSDataOutputStream>
.build();
}
+ @Override
+ protected void onSchemaChanged() {
+ this.serializationSchema =
+ TextSerializationSchema.builder()
+ .seaTunnelRowType(
+ buildSchemaWithRowType(seaTunnelRowType, sinkColumnsIndexInRow))
+ .delimiter(fieldDelimiter)
+ .dateFormatter(dateFormat)
+ .dateTimeFormatter(dateTimeFormat)
+ .timeFormatter(timeFormat)
+ .charset(charset)
+ .build();
+ }
+
@Override
public void write(@NonNull SeaTunnelRow seaTunnelRow) {
super.write(seaTunnelRow);
@@ -97,12 +111,7 @@ public class TextWriteStrategy extends AbstractWriteStrategy<FSDataOutputStream>
} else {
fsDataOutputStream.write(rowDelimiter.getBytes(charset));
}
- fsDataOutputStream.write(
- serializationSchema.serialize(
- seaTunnelRow.copy(
- sinkColumnsIndexInRow.stream()
- .mapToInt(Integer::intValue)
- .toArray())));
+ fsDataOutputStream.write(serializationSchema.serialize(safeProjectedRow(seaTunnelRow)));
} catch (IOException e) {
throw CommonError.fileOperationFailed("TextFile", "write", filePath, e);
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/WriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/WriteStrategy.java
index 25b84714d4..ddd5e99fb3 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/WriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/WriteStrategy.java
@@ -18,6 +18,7 @@
package org.apache.seatunnel.connectors.seatunnel.file.sink.writer;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf;
import org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorException;
@@ -104,4 +105,19 @@ public interface WriteStrategy<T> extends Transaction, Serializable, Closeable {
* @return file system utils
*/
HadoopFileSystemProxy getHadoopFileSystemProxy();
+
+ /**
+ * Apply a schema change event to this write strategy. Implementations must: 1. Close and flush
+ * all currently open writers (resource-safe: attempt all closes) 2. Update the row type and
+ * column index mapping 3. Invalidate any cached format-specific schema objects
+ *
+ * <p>The next {@link #write(SeaTunnelRow)} call after this method returns will open new writers
+ * with the updated schema.
+ *
+ * <p>This method is a no-op when {@code schema_evolution_enabled=false}.
+ *
+ * @param event the schema change event (ADD/DROP/RENAME/UPDATE column, or batch)
+ * @throws IOException if closing any open writer fails
+ */
+ void applySchemaChange(SchemaChangeEvent event) throws IOException;
}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSchemaEvolutionTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSchemaEvolutionTest.java
new file mode 100644
index 0000000000..ac68705f36
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSchemaEvolutionTest.java
@@ -0,0 +1,693 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.file.writer;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.table.catalog.PhysicalColumn;
+import org.apache.seatunnel.api.table.catalog.TableIdentifier;
+import org.apache.seatunnel.api.table.schema.event.AlterTableAddColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableChangeColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableColumnsEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableDropColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableModifyColumnEvent;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf;
+import org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorException;
+import org.apache.seatunnel.connectors.seatunnel.file.sink.config.FileSinkConfig;
+import org.apache.seatunnel.connectors.seatunnel.file.sink.writer.AbstractWriteStrategy;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+/**
+ * Unit tests for schema evolution logic in AbstractWriteStrategy. Uses a minimal NoOpWriteStrategy
+ * inner class that avoids any filesystem I/O.
+ */
+public class FileSchemaEvolutionTest {
+
+ private static final TableIdentifier TABLE_ID =
+ TableIdentifier.of("catalog", "db", "test_table");
+
+ /** Minimal row type: id (INT), name (STRING), age (INT) */
+ private static final SeaTunnelRowType BASE_ROW_TYPE =
+ new SeaTunnelRowType(
+ new String[] {"id", "name", "age"},
+ new SeaTunnelDataType[] {
+ BasicType.INT_TYPE, BasicType.STRING_TYPE, BasicType.INT_TYPE
+ });
+
+ private static FileSinkConfig schemaEvolutionEnabledConfig() {
+ Config config =
+ ConfigFactory.parseString(
+ "path = \"/tmp/test\"\n"
+ + "file_format_type = \"parquet\"\n"
+ + "schema_evolution_enabled = true");
+ return new FileSinkConfig(ReadonlyConfig.fromConfig(config), BASE_ROW_TYPE);
+ }
+
+ private static FileSinkConfig schemaEvolutionDisabledConfig() {
+ Config config =
+ ConfigFactory.parseString(
+ "path = \"/tmp/test\"\n" + "file_format_type = \"parquet\"");
+ return new FileSinkConfig(ReadonlyConfig.fromConfig(config), BASE_ROW_TYPE);
+ }
+
+ /**
+ * Config where sink_columns uses mixed case (e.g., "AGE" instead of "age"). This tests the
+ * case-insensitive matching in updateSinkColumnNames: CDC events carry schema-exact names, but
+ * user-configured sink_columns may have different capitalisation.
+ */
+ private static FileSinkConfig schemaEvolutionEnabledConfigWithUppercaseSinkColumns() {
+ // "AGE" uppercase in config, but schema has "age" — real mismatch scenario
+ Config config =
+ ConfigFactory.parseString(
+ "path = \"/tmp/test\"\n"
+ + "file_format_type = \"parquet\"\n"
+ + "sink_columns = [\"id\", \"NAME\", \"AGE\"]\n"
+ + "schema_evolution_enabled = true");
+ return new FileSinkConfig(ReadonlyConfig.fromConfig(config), BASE_ROW_TYPE);
+ }
+
+ // ── Helper to build a NoOpWriteStrategy with a given FileSinkConfig ──────────
+
+ private static NoOpWriteStrategy buildStrategy(FileSinkConfig config) {
+ NoOpWriteStrategy strategy = new NoOpWriteStrategy(config);
+ // applySchemaChange (post 01cc356f2) reads tableSchema, not just seaTunnelRowType.
+ // Use the production setCatalogTable() path so both fields are wired up correctly.
+ strategy.setCatalogTable(
+ org.apache.seatunnel.api.table.catalog.CatalogTableUtil.getCatalogTable(
+ TABLE_ID.getCatalogName(),
+ TABLE_ID.getDatabaseName(),
+ TABLE_ID.getSchemaName(),
+ TABLE_ID.getTableName(),
+ BASE_ROW_TYPE));
+ return strategy;
+ }
+
+ // ── ADD_COLUMN ────────────────────────────────────────────────────────────────
+
+ @Test
+ public void testAddColumnAppendsToSinkColumnNamesAndRebuildsIndex() throws IOException {
+ NoOpWriteStrategy strategy = buildStrategy(schemaEvolutionEnabledConfig());
+
+ PhysicalColumn newCol =
+ PhysicalColumn.builder()
+ .name("email")
+ .dataType(BasicType.STRING_TYPE)
+ .nullable(true)
+ .build();
+ AlterTableAddColumnEvent event = AlterTableAddColumnEvent.add(TABLE_ID, newCol);
+ strategy.applySchemaChange(event);
+
+ List<String> names = strategy.getSinkColumnNames();
+ Assertions.assertTrue(names.contains("email"), "email should be in sinkColumnNames");
+ Assertions.assertEquals(4, names.size(), "should now have 4 columns");
+
+ // Index for email should be 3 (last position in new rowType)
+ SeaTunnelRowType updated = strategy.getSeaTunnelRowType();
+ int emailIdx = Arrays.asList(updated.getFieldNames()).indexOf("email");
+ Assertions.assertTrue(strategy.getSinkColumnsIndexInRow().contains(emailIdx));
+ }
+
+ @Test
+ public void testAddColumnFirstPrependsToSinkColumnNames() throws IOException {
+ NoOpWriteStrategy strategy = buildStrategy(schemaEvolutionEnabledConfig());
+
+ PhysicalColumn newCol =
+ PhysicalColumn.builder()
+ .name("prefix")
+ .dataType(BasicType.STRING_TYPE)
+ .nullable(true)
+ .build();
+ AlterTableAddColumnEvent event = AlterTableAddColumnEvent.addFirst(TABLE_ID, newCol);
+ strategy.applySchemaChange(event);
+
+ Assertions.assertEquals("prefix", strategy.getSinkColumnNames().get(0));
+ }
+
+ // ── DROP_COLUMN ───────────────────────────────────────────────────────────────
+
+ @Test
+ public void testDropColumnRemovesFromSinkColumnNamesAndRebuildsIndex() throws IOException {
+ NoOpWriteStrategy strategy = buildStrategy(schemaEvolutionEnabledConfig());
+
+ AlterTableDropColumnEvent event = new AlterTableDropColumnEvent(TABLE_ID, "age");
+ strategy.applySchemaChange(event);
+
+ Assertions.assertFalse(
+ strategy.getSinkColumnNames().contains("age"), "age should be removed");
+ Assertions.assertEquals(2, strategy.getSinkColumnNames().size());
+
+ // No index for 'age' any more
+ SeaTunnelRowType updated = strategy.getSeaTunnelRowType();
+ Assertions.assertFalse(
+ Arrays.asList(updated.getFieldNames()).contains("age"),
+ "age should not be in updated rowType");
+ }
+
+ // ── RENAME_COLUMN (AlterTableChangeColumnEvent) ───────────────────────────────
+
+ @Test
+ public void testRenameColumnUpdatesNameInSinkColumnNames() throws IOException {
+ NoOpWriteStrategy strategy = buildStrategy(schemaEvolutionEnabledConfig());
+
+ PhysicalColumn renamedCol =
+ PhysicalColumn.builder()
+ .name("full_name")
+ .dataType(BasicType.STRING_TYPE)
+ .nullable(true)
+ .build();
+ AlterTableChangeColumnEvent event =
+ AlterTableChangeColumnEvent.change(TABLE_ID, "name", renamedCol);
+ strategy.applySchemaChange(event);
+
+ Assertions.assertFalse(
+ strategy.getSinkColumnNames().contains("name"), "old name should be removed");
+ Assertions.assertTrue(
+ strategy.getSinkColumnNames().contains("full_name"), "new name should be present");
+
+ SeaTunnelRowType updated = strategy.getSeaTunnelRowType();
+ Assertions.assertTrue(
+ Arrays.asList(updated.getFieldNames()).contains("full_name"),
+ "full_name should be in updated rowType");
+ }
+
+ // ── UPDATE_COLUMN / MODIFY_COLUMN ─────────────────────────────────────────────
+
+ @Test
+ public void testModifyColumnTypeDoesNotChangeSinkColumnNames() throws IOException {
+ NoOpWriteStrategy strategy = buildStrategy(schemaEvolutionEnabledConfig());
+
+ // Change 'age' from INT to LONG
+ PhysicalColumn modifiedCol =
+ PhysicalColumn.builder()
+ .name("age")
+ .dataType(BasicType.LONG_TYPE)
+ .nullable(true)
+ .build();
+ AlterTableModifyColumnEvent event =
+ AlterTableModifyColumnEvent.modify(TABLE_ID, modifiedCol);
+ strategy.applySchemaChange(event);
+
+ // Column names should be unchanged
+ Assertions.assertEquals(Arrays.asList("id", "name", "age"), strategy.getSinkColumnNames());
+
+ // But the type in rowType should be updated to LONG
+ int ageIdx = Arrays.asList(strategy.getSeaTunnelRowType().getFieldNames()).indexOf("age");
+ Assertions.assertEquals(
+ BasicType.LONG_TYPE, strategy.getSeaTunnelRowType().getFieldType(ageIdx));
+ }
+
+ // ── Batch / AlterTableColumnsEvent ────────────────────────────────────────────
+
+ @Test
+ public void testBatchColumnsEventAppliedAsOneSchemaChange() throws IOException {
+ NoOpWriteStrategy strategy = buildStrategy(schemaEvolutionEnabledConfig());
+
+ PhysicalColumn newScore =
+ PhysicalColumn.builder()
+ .name("score")
+ .dataType(BasicType.INT_TYPE)
+ .nullable(true)
+ .build();
+
+ AlterTableColumnsEvent batch =
+ new AlterTableColumnsEvent(
+ TABLE_ID,
+ Arrays.asList(
+ new AlterTableDropColumnEvent(TABLE_ID, "age"),
+ AlterTableAddColumnEvent.add(TABLE_ID, newScore)));
+ strategy.applySchemaChange(batch);
+
+ // age dropped, score added
+ Assertions.assertFalse(strategy.getSinkColumnNames().contains("age"));
+ Assertions.assertTrue(strategy.getSinkColumnNames().contains("score"));
+ // finishAndCloseFile called exactly once (batch counts as single schema change)
+ Assertions.assertEquals(1, strategy.finishAndCloseFileCalls);
+ }
+
+ // ── Disabled flag: applySchemaChange fails fast with actionable message ────────
+
+ @Test
+ public void testDisabledFlagThrowsOnAlterTableEvent() throws IOException {
+ NoOpWriteStrategy strategy = buildStrategy(schemaEvolutionDisabledConfig());
+
+ AlterTableDropColumnEvent event = new AlterTableDropColumnEvent(TABLE_ID, "age");
+ UnsupportedOperationException ex =
+ Assertions.assertThrows(
+ UnsupportedOperationException.class,
+ () -> strategy.applySchemaChange(event));
+ Assertions.assertTrue(ex.getMessage().contains("schema_evolution_enabled=false"));
+ Assertions.assertTrue(ex.getMessage().contains("schema-changes.enabled=false"));
+ }
+
+ // ── finishAndCloseFile called on each schema change ───────────────────────────
+
+ @Test
+ public void testFinishAndCloseFileCalledOnSchemaChange() throws IOException {
+ NoOpWriteStrategy strategy = buildStrategy(schemaEvolutionEnabledConfig());
+
+ strategy.applySchemaChange(new AlterTableDropColumnEvent(TABLE_ID, "age"));
+ Assertions.assertEquals(1, strategy.finishAndCloseFileCalls);
+
+ // Rebuild seaTunnelRowType to have only id + name for the second event
+ PhysicalColumn newCol =
+ PhysicalColumn.builder()
+ .name("extra")
+ .dataType(BasicType.STRING_TYPE)
+ .nullable(true)
+ .build();
+ strategy.applySchemaChange(AlterTableAddColumnEvent.add(TABLE_ID, newCol));
+ Assertions.assertEquals(2, strategy.finishAndCloseFileCalls);
+ }
+
+ // ── getFieldSafe bounds check ─────────────────────────────────────────────────
+
+ @Test
+ public void testGetFieldSafeReturnsNullWhenIndexOutOfBounds() {
+ NoOpWriteStrategy strategy = buildStrategy(schemaEvolutionEnabledConfig());
+ SeaTunnelRow row = new SeaTunnelRow(new Object[] {"val0", "val1", 42});
+
+ // In-bounds: normal access
+ Assertions.assertEquals("val0", strategy.callGetFieldSafe(row, 0));
+ Assertions.assertEquals(42, strategy.callGetFieldSafe(row, 2));
+
+ // Out-of-bounds: returns null instead of ArrayIndexOutOfBoundsException
+ Assertions.assertNull(strategy.callGetFieldSafe(row, 5));
+ Assertions.assertNull(strategy.callGetFieldSafe(row, 3));
+ }
+
+ // ── IndexInRow stays consistent after ADD then DROP ───────────────────────────
+
+ @Test
+ public void testIndexInRowConsistencyAfterAddThenDrop() throws IOException {
+ NoOpWriteStrategy strategy = buildStrategy(schemaEvolutionEnabledConfig());
+
+ // Add email
+ PhysicalColumn emailCol =
+ PhysicalColumn.builder()
+ .name("email")
+ .dataType(BasicType.STRING_TYPE)
+ .nullable(true)
+ .build();
+ strategy.applySchemaChange(AlterTableAddColumnEvent.add(TABLE_ID, emailCol));
+
+ // Drop id
+ strategy.applySchemaChange(new AlterTableDropColumnEvent(TABLE_ID, "id"));
+
+ // Remaining: name, age, email
+ Assertions.assertFalse(strategy.getSinkColumnNames().contains("id"));
+ Assertions.assertTrue(strategy.getSinkColumnNames().contains("name"));
+ Assertions.assertTrue(strategy.getSinkColumnNames().contains("age"));
+ Assertions.assertTrue(strategy.getSinkColumnNames().contains("email"));
+
+ // Every index in sinkColumnsIndexInRow should be within bounds of the updated rowType
+ int numFields = strategy.getSeaTunnelRowType().getTotalFields();
+ for (int idx : strategy.getSinkColumnsIndexInRow()) {
+ Assertions.assertTrue(
+ idx >= 0 && idx < numFields,
+ "Index " + idx + " out of range [0, " + numFields + ")");
+ }
+ }
+
+ // ── Case-insensitive DROP ─────────────────────────────────────────────────────
+
+ @Test
+ public void testDropColumnCaseInsensitiveSinkColumns() throws IOException {
+ // sink_columns configured as ["id", "NAME", "AGE"] — user typed uppercase
+ // CDC event carries "age" (exact schema case) — dispatcher finds it, we must
+ // match it against "AGE" in sinkColumnNames via equalsIgnoreCase
+ NoOpWriteStrategy strategy =
+ buildStrategy(schemaEvolutionEnabledConfigWithUppercaseSinkColumns());
+
+ AlterTableDropColumnEvent event = new AlterTableDropColumnEvent(TABLE_ID, "age");
+ strategy.applySchemaChange(event);
+
+ // "AGE" must be removed even though sinkColumnNames stored it as uppercase
+ Assertions.assertFalse(
+ strategy.getSinkColumnNames().stream().anyMatch(c -> c.equalsIgnoreCase("age")),
+ "AGE should be removed via case-insensitive match");
+ Assertions.assertEquals(2, strategy.getSinkColumnNames().size());
+ }
+
+ // ── RENAME with position change ───────────────────────────────────────────────
+
+ @Test
+ public void testRenameColumnWithPositionMoveToFirst() throws IOException {
+ NoOpWriteStrategy strategy = buildStrategy(schemaEvolutionEnabledConfig());
+ // Original: [id, name, age]. Move-rename "age" → "score" and put it first.
+ PhysicalColumn renamedCol =
+ PhysicalColumn.builder()
+ .name("score")
+ .dataType(BasicType.INT_TYPE)
+ .nullable(true)
+ .build();
+ AlterTableChangeColumnEvent event =
+ AlterTableChangeColumnEvent.changeFirst(TABLE_ID, "age", renamedCol);
+ strategy.applySchemaChange(event);
+
+ // "score" (was "age") should now be at index 0
+ Assertions.assertEquals("score", strategy.getSinkColumnNames().get(0));
+ Assertions.assertFalse(strategy.getSinkColumnNames().contains("age"));
+ }
+
+ @Test
+ public void testRenameCaseInsensitiveSinkColumns() throws IOException {
+ // sink_columns configured as ["id", "NAME", "AGE"] — user typed "NAME" uppercase.
+ // CDC RENAME event sends oldColumn="name" (exact schema case), new name="full_name".
+ // Our updateSinkColumnNames must find "NAME" in sinkColumnNames via equalsIgnoreCase.
+ NoOpWriteStrategy strategy =
+ buildStrategy(schemaEvolutionEnabledConfigWithUppercaseSinkColumns());
+
+ PhysicalColumn renamedCol =
+ PhysicalColumn.builder()
+ .name("full_name")
+ .dataType(BasicType.STRING_TYPE)
+ .nullable(true)
+ .build();
+ // Event uses schema-exact lowercase "name" — dispatcher can find it
+ AlterTableChangeColumnEvent event =
+ AlterTableChangeColumnEvent.change(TABLE_ID, "name", renamedCol);
+ strategy.applySchemaChange(event);
+
+ // "NAME" (uppercase) should be replaced by "full_name"
+ Assertions.assertFalse(
+ strategy.getSinkColumnNames().stream().anyMatch(c -> c.equalsIgnoreCase("name")),
+ "old name should be removed regardless of case in sinkColumnNames");
+ Assertions.assertTrue(strategy.getSinkColumnNames().contains("full_name"));
+ }
+
+ // ── beingWrittenFile cleared even when finishAndCloseFile throws ──────────────
+
+ @Test
+ public void testBeingWrittenFileClearedEvenOnFinishAndCloseFileException() {
+ // Strategy whose finishAndCloseFile throws to simulate a writer-close failure
+ ThrowingWriteStrategy strategy = new ThrowingWriteStrategy(schemaEvolutionEnabledConfig());
+ strategy.setCatalogTable(
+ org.apache.seatunnel.api.table.catalog.CatalogTableUtil.getCatalogTable(
+ TABLE_ID.getCatalogName(),
+ TABLE_ID.getDatabaseName(),
+ TABLE_ID.getSchemaName(),
+ TABLE_ID.getTableName(),
+ BASE_ROW_TYPE));
+ // Simulate a file already being tracked
+ strategy.exposeBeingWrittenFile().put("NON_PARTITION", "/tmp/test/some-file.parquet");
+
+ AlterTableDropColumnEvent event = new AlterTableDropColumnEvent(TABLE_ID, "age");
+ try {
+ strategy.applySchemaChange(event);
+ Assertions.fail("Expected exception from finishAndCloseFile was not thrown");
+ } catch (Exception e) {
+ // Expected — finishAndCloseFile threw (FileConnectorException is a RuntimeException)
+ }
+
+ // Despite the exception, beingWrittenFile must be cleared (try-finally guarantee)
+ Assertions.assertTrue(
+ strategy.exposeBeingWrittenFile().isEmpty(),
+ "beingWrittenFile must be cleared even when finishAndCloseFile throws");
+ // Schema state must NOT be updated (schema change did not complete successfully)
+ Assertions.assertEquals(
+ 3,
+ strategy.getSeaTunnelRowType().getTotalFields(),
+ "seaTunnelRowType must remain unchanged after failed schema change");
+ }
+
+ // ── Type-zoo coverage ─────────────────────────────────────────────────────────
+ //
+ // Production tables include BIGINT (often unsigned), DECIMAL, TIMESTAMP, BOOLEAN
+ // (tinyint(1)) and TINYINT — schema-evolution logic must preserve all of them.
+
+ /**
+ * Type zoo mirroring the columns we see across the bookkeeping / settlement / static_inventory
+ * tables: id (BIGINT), is_active (BOOLEAN), op (TINYINT), status (SMALLINT), version (INT),
+ * amount (DECIMAL), score (DOUBLE), name (STRING), created_at (TIMESTAMP), birth (DATE).
+ */
+ private static final SeaTunnelRowType TYPE_ZOO_ROW_TYPE =
+ new SeaTunnelRowType(
+ new String[] {
+ "id",
+ "is_active",
+ "op",
+ "status",
+ "version",
+ "amount",
+ "score",
+ "name",
+ "created_at",
+ "birth"
+ },
+ new SeaTunnelDataType[] {
+ BasicType.LONG_TYPE,
+ BasicType.BOOLEAN_TYPE,
+ BasicType.BYTE_TYPE,
+ BasicType.SHORT_TYPE,
+ BasicType.INT_TYPE,
+ new org.apache.seatunnel.api.table.type.DecimalType(20, 6),
+ BasicType.DOUBLE_TYPE,
+ BasicType.STRING_TYPE,
+ org.apache.seatunnel.api.table.type.LocalTimeType.LOCAL_DATE_TIME_TYPE,
+ org.apache.seatunnel.api.table.type.LocalTimeType.LOCAL_DATE_TYPE
+ });
+
+ private static NoOpWriteStrategy buildTypeZooStrategy() {
+ Config config =
+ ConfigFactory.parseString(
+ "path = \"/tmp/test\"\n"
+ + "file_format_type = \"parquet\"\n"
+ + "schema_evolution_enabled = true");
+ FileSinkConfig sinkConfig =
+ new FileSinkConfig(ReadonlyConfig.fromConfig(config), TYPE_ZOO_ROW_TYPE);
+ NoOpWriteStrategy strategy = new NoOpWriteStrategy(sinkConfig);
+ strategy.setCatalogTable(
+ org.apache.seatunnel.api.table.catalog.CatalogTableUtil.getCatalogTable(
+ TABLE_ID.getCatalogName(),
+ TABLE_ID.getDatabaseName(),
+ TABLE_ID.getSchemaName(),
+ TABLE_ID.getTableName(),
+ TYPE_ZOO_ROW_TYPE));
+ return strategy;
+ }
+
+ @Test
+ public void testTypeZooAddBooleanColumnPreservesAllOriginalTypes() throws IOException {
+ NoOpWriteStrategy strategy = buildTypeZooStrategy();
+
+ PhysicalColumn newCol =
+ PhysicalColumn.builder()
+ .name("is_archived")
+ .dataType(BasicType.BOOLEAN_TYPE)
+ .nullable(true)
+ .build();
+ strategy.applySchemaChange(AlterTableAddColumnEvent.add(TABLE_ID, newCol));
+
+ SeaTunnelRowType after = strategy.getSeaTunnelRowType();
+ Assertions.assertEquals(11, after.getTotalFields());
+ Assertions.assertEquals("is_archived", after.getFieldNames()[10]);
+ // All original types must still match (no widening / narrowing).
+ for (int i = 0; i < TYPE_ZOO_ROW_TYPE.getTotalFields(); i++) {
+ Assertions.assertEquals(
+ TYPE_ZOO_ROW_TYPE.getFieldType(i),
+ after.getFieldType(i),
+ "type at index " + i + " (" + after.getFieldNames()[i] + ") must be unchanged");
+ }
+ }
+
+ @Test
+ public void testTypeZooDropDecimalColumnLeavesAllOtherTypesIntact() throws IOException {
+ NoOpWriteStrategy strategy = buildTypeZooStrategy();
+
+ strategy.applySchemaChange(new AlterTableDropColumnEvent(TABLE_ID, "amount"));
+
+ SeaTunnelRowType after = strategy.getSeaTunnelRowType();
+ Assertions.assertEquals(9, after.getTotalFields());
+ Assertions.assertFalse(Arrays.asList(after.getFieldNames()).contains("amount"));
+ // Surviving columns must keep exact types — DECIMAL drop must not collapse other DECIMALs.
+ Assertions.assertEquals(BasicType.LONG_TYPE, after.getFieldType(0));
+ Assertions.assertEquals(BasicType.BOOLEAN_TYPE, after.getFieldType(1));
+ Assertions.assertEquals(BasicType.BYTE_TYPE, after.getFieldType(2));
+ Assertions.assertEquals(BasicType.SHORT_TYPE, after.getFieldType(3));
+ Assertions.assertEquals(BasicType.INT_TYPE, after.getFieldType(4));
+ Assertions.assertEquals(BasicType.DOUBLE_TYPE, after.getFieldType(5));
+ Assertions.assertEquals(BasicType.STRING_TYPE, after.getFieldType(6));
+ }
+
+ @Test
+ public void testTypeZooRenameTimestampColumnKeepsType() throws IOException {
+ NoOpWriteStrategy strategy = buildTypeZooStrategy();
+
+ PhysicalColumn renamed =
+ PhysicalColumn.builder()
+ .name("event_time")
+ .dataType(
+ org.apache.seatunnel.api.table.type.LocalTimeType
+ .LOCAL_DATE_TIME_TYPE)
+ .nullable(true)
+ .build();
+ strategy.applySchemaChange(
+ AlterTableChangeColumnEvent.change(TABLE_ID, "created_at", renamed));
+
+ SeaTunnelRowType after = strategy.getSeaTunnelRowType();
+ Assertions.assertFalse(Arrays.asList(after.getFieldNames()).contains("created_at"));
+ Assertions.assertTrue(Arrays.asList(after.getFieldNames()).contains("event_time"));
+ int idx = Arrays.asList(after.getFieldNames()).indexOf("event_time");
+ Assertions.assertEquals(
+ org.apache.seatunnel.api.table.type.LocalTimeType.LOCAL_DATE_TIME_TYPE,
+ after.getFieldType(idx),
+ "rename must preserve TIMESTAMP type");
+ }
+
+ // ── Inner NoOp strategy ───────────────────────────────────────────────────────
+
+ /**
+ * Minimal concrete write strategy for unit-testing AbstractWriteStrategy logic. Tracks how many
+ * times finishAndCloseFile() was called. No filesystem I/O is performed.
+ *
+ * <p>Accessor methods expose protected fields from the parent class, which is required because
+ * the test class lives in a different package from AbstractWriteStrategy.
+ */
+ static class NoOpWriteStrategy extends AbstractWriteStrategy<Object> {
+ int finishAndCloseFileCalls = 0;
+ boolean onSchemaChangedCalled = false;
+
+ NoOpWriteStrategy(FileSinkConfig config) {
+ super(config);
+ // Stub out needMoveFiles which is initialised in beginTransaction
+ this.needMoveFiles = new LinkedHashMap<>();
+ }
+
+ // Accessors for protected fields (required when test is in a different package)
+ List<String> getSinkColumnNames() {
+ return sinkColumnNames;
+ }
+
+ List<Integer> getSinkColumnsIndexInRow() {
+ return sinkColumnsIndexInRow;
+ }
+
+ SeaTunnelRowType getSeaTunnelRowType() {
+ return seaTunnelRowType;
+ }
+
+ void setSeaTunnelRowTypeForTest(SeaTunnelRowType rowType) {
+ this.seaTunnelRowType = rowType;
+ }
+
+ LinkedHashMap<String, String> exposeBeingWrittenFile() {
+ return beingWrittenFile;
+ }
+
+ Object callGetFieldSafe(SeaTunnelRow row, int index) {
+ return getFieldSafe(row, index);
+ }
+
+ @Override
+ public void finishAndCloseFile() {
+ finishAndCloseFileCalls++;
+ // Don't touch needMoveFiles here — nothing to move in unit tests
+ }
+
+ @Override
+ public Object getOrCreateOutputStream(String path) {
+ return new Object();
+ }
+
+ @Override
+ protected void onSchemaChanged() {
+ onSchemaChangedCalled = true;
+ }
+
+ @Override
+ public void init(
+ org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf conf,
+ String jobId,
+ String uuidPrefix,
+ int subTaskIndex) {
+ // no-op — no filesystem in unit tests
+ }
+
+ @Override
+ public Configuration getConfiguration(HadoopConf conf) {
+ return new Configuration();
+ }
+ }
+
+ // ── Config validation ─────────────────────────────────────────────────────
+
+ @Test
+ public void testBinaryWithSchemaEvolutionEnabledThrowsAtConfig() {
+ Config config =
+ ConfigFactory.parseString(
+ "path = \"/tmp/test\"\n"
+ + "file_format_type = \"binary\"\n"
+ + "schema_evolution_enabled = true");
+ Assertions.assertThrows(
+ FileConnectorException.class,
+ () -> new FileSinkConfig(ReadonlyConfig.fromConfig(config), BASE_ROW_TYPE),
+ "binary format must reject schema_evolution_enabled=true at config time");
+ }
+
+ @Test
+ public void testPartitionByWithSchemaEvolutionEnabledIsAccepted() {
+ // partition_by + schema_evolution_enabled is supported: applySchemaChange rebuilds
+ // partitionFieldsIndexInRow on every ALTER via name-based lookup against the post-ALTER
+ // row type, mirroring sinkColumnsIndexInRow. Drop/rename of a partition column itself is
+ // rejected at rebuild time with an explicit IllegalStateException (covered separately).
+ Config config =
+ ConfigFactory.parseString(
+ "path = \"/tmp/test\"\n"
+ + "file_format_type = \"text\"\n"
+ + "partition_by = [\"age\"]\n"
+ + "schema_evolution_enabled = true");
+ Assertions.assertDoesNotThrow(
+ () -> new FileSinkConfig(ReadonlyConfig.fromConfig(config), BASE_ROW_TYPE),
+ "partition_by + schema_evolution_enabled=true must be supported");
+ }
+
+ /**
+ * Variant that throws on finishAndCloseFile — used to verify the try-finally guarantee in
+ * applySchemaChange (beingWrittenFile must be cleared even when the close fails).
+ */
+ static class ThrowingWriteStrategy extends NoOpWriteStrategy {
+ ThrowingWriteStrategy(FileSinkConfig config) {
+ super(config);
+ }
+
+ @Override
+ public void finishAndCloseFile() {
+ // Simulate a writer that fails to close (e.g., HDFS timeout)
+ throw new org.apache.seatunnel.connectors.seatunnel.file.exception
+ .FileConnectorException(
+ org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated
+ .WRITER_OPERATION_FAILED,
+ "Simulated writer close failure for test");
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetTypeCoercionTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetTypeCoercionTest.java
new file mode 100644
index 0000000000..006f530b50
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetTypeCoercionTest.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.file.writer;
+
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.source.Collector;
+import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.DecimalType;
+import org.apache.seatunnel.api.table.type.LocalTimeType;
+import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat;
+import org.apache.seatunnel.connectors.seatunnel.file.sink.config.FileSinkConfig;
+import org.apache.seatunnel.connectors.seatunnel.file.sink.writer.ParquetWriteStrategy;
+import org.apache.seatunnel.connectors.seatunnel.file.source.reader.ParquetReadStrategy;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
+
+/**
+ * Asserts that {@link ParquetWriteStrategy} accepts the canonical Java type for every SeaTunnel
+ * SqlType AND defensively coerces the most likely "looks-wrong" types Debezium may emit. The
+ * canonical regression case is BOOLEAN ← Byte, which crashed v21 in stage when MySQL {@code
+ * tinyint(1)} columns flowed through the Parquet sink as raw Byte values.
+ */
+@Slf4j
+public class ParquetTypeCoercionTest {
+
+ private static final String BASE_PATH = "file:///tmp/seatunnel/parquet/coercion";
+
+ /**
+ * v21 stage regression. MySQL {@code tinyint(1)} arrives as Byte but the SeaTunnel schema says
+ * BOOLEAN; Avro requires Boolean. Without coercion in resolveObject, this throws {@code
+ * java.lang.Byte cannot be cast to java.lang.Boolean} at AvroWriteSupport.
+ */
+ @DisabledOnOs(OS.WINDOWS)
+ @Test
+ public void testBooleanColumnAcceptsByteFromCdcTinyintOne() throws Exception {
+ SeaTunnelRowType rowType =
+ new SeaTunnelRowType(
+ new String[] {"id", "is_active"},
+ new SeaTunnelDataType[] {BasicType.INT_TYPE, BasicType.BOOLEAN_TYPE});
+
+ SeaTunnelRow row1 = new SeaTunnelRow(new Object[] {1, (byte) 1});
+ SeaTunnelRow row0 = new SeaTunnelRow(new Object[] {2, (byte) 0});
+ SeaTunnelRow rowBoolean = new SeaTunnelRow(new Object[] {3, Boolean.TRUE});
+
+ List<Object[]> readBack =
+ writeAndReadBack(
+ "boolean-from-byte",
+ rowType,
+ java.util.Arrays.asList(row1, row0, rowBoolean));
+
+ Assertions.assertEquals(3, readBack.size());
+ // After coercion every BOOLEAN cell must be a Boolean — never a Byte.
+ for (Object[] r : readBack) {
+ Assertions.assertTrue(
+ r[1] instanceof Boolean,
+ "BOOLEAN column read back as " + r[1].getClass() + " (value=" + r[1] + ")");
+ }
+ // Byte 1 → true, Byte 0 → false, Boolean.TRUE → true.
+ Assertions.assertEquals(Boolean.TRUE, readBack.get(0)[1]);
+ Assertions.assertEquals(Boolean.FALSE, readBack.get(1)[1]);
+ Assertions.assertEquals(Boolean.TRUE, readBack.get(2)[1]);
+ }
+
+ /**
+ * Type fidelity for the most common types we see in production (settlement / bookkeeping
+ * pipelines): INT, BIGINT, SMALLINT, TINYINT, FLOAT, DOUBLE, DECIMAL, STRING, DATE, TIMESTAMP,
+ * BYTES. Each column gets the canonical Java type — this test guards against regressions in
+ * {@code resolveObject}'s case statements.
+ */
+ @DisabledOnOs(OS.WINDOWS)
+ @Test
+ public void testCanonicalTypesRoundTripFaithfully() throws Exception {
+ SeaTunnelRowType rowType =
+ new SeaTunnelRowType(
+ new String[] {
+ "c_tinyint",
+ "c_smallint",
+ "c_int",
+ "c_bigint",
+ "c_float",
+ "c_double",
+ "c_decimal",
+ "c_string",
+ "c_date",
+ "c_timestamp",
+ "c_bytes"
+ },
+ new SeaTunnelDataType[] {
+ BasicType.BYTE_TYPE,
+ BasicType.SHORT_TYPE,
+ BasicType.INT_TYPE,
+ BasicType.LONG_TYPE,
+ BasicType.FLOAT_TYPE,
+ BasicType.DOUBLE_TYPE,
+ new DecimalType(20, 6),
+ BasicType.STRING_TYPE,
+ LocalTimeType.LOCAL_DATE_TYPE,
+ LocalTimeType.LOCAL_DATE_TIME_TYPE,
+ PrimitiveByteArrayType.INSTANCE
+ });
+
+ LocalDate testDate = LocalDate.of(2026, 4, 26);
+ LocalDateTime testTimestamp = LocalDateTime.of(2026, 4, 26, 14, 30, 15);
+ BigDecimal testDecimal = new BigDecimal("12345678901234.567890");
+
+ SeaTunnelRow row =
+ new SeaTunnelRow(
+ new Object[] {
+ (byte) 7,
+ (short) 1234,
+ 42,
+ 9_999_999_999L,
+ 3.14f,
+ 2.71828,
+ testDecimal,
+ "hello",
+ testDate,
+ testTimestamp,
+ new byte[] {1, 2, 3, 4}
+ });
+
+ List<Object[]> readBack =
+ writeAndReadBack("canonical-types", rowType, java.util.Arrays.asList(row));
+
+ Assertions.assertEquals(1, readBack.size());
+ Object[] r = readBack.get(0);
+ Assertions.assertEquals((byte) 7, ((Number) r[0]).byteValue());
+ Assertions.assertEquals((short) 1234, ((Number) r[1]).shortValue());
+ Assertions.assertEquals(42, ((Number) r[2]).intValue());
+ Assertions.assertEquals(9_999_999_999L, ((Number) r[3]).longValue());
+ Assertions.assertEquals(3.14f, ((Number) r[4]).floatValue(), 0.0001f);
+ Assertions.assertEquals(2.71828, ((Number) r[5]).doubleValue(), 0.0000001);
+ Assertions.assertEquals(0, testDecimal.compareTo((BigDecimal) r[6]));
+ Assertions.assertEquals("hello", r[7]);
+ Assertions.assertEquals(testDate, r[8]);
+ Assertions.assertEquals(testTimestamp, r[9]);
+ Assertions.assertArrayEquals(new byte[] {1, 2, 3, 4}, (byte[]) r[10]);
+ }
+
+ /**
+ * Null values must round-trip as null for every nullable column type — otherwise downstream
+ * consumers see surprising defaults or NPE.
+ */
+ @DisabledOnOs(OS.WINDOWS)
+ @Test
+ public void testNullValuesPreserved() throws Exception {
+ SeaTunnelRowType rowType =
+ new SeaTunnelRowType(
+ new String[] {"id", "name", "amount", "is_active"},
+ new SeaTunnelDataType[] {
+ BasicType.INT_TYPE,
+ BasicType.STRING_TYPE,
+ new DecimalType(10, 2),
+ BasicType.BOOLEAN_TYPE
+ });
+
+ SeaTunnelRow row = new SeaTunnelRow(new Object[] {1, null, null, null});
+
+ List<Object[]> readBack = writeAndReadBack("nulls", rowType, java.util.Arrays.asList(row));
+
+ Assertions.assertEquals(1, readBack.size());
+ Object[] r = readBack.get(0);
+ Assertions.assertEquals(1, ((Number) r[0]).intValue());
+ Assertions.assertNull(r[1]);
+ Assertions.assertNull(r[2]);
+ Assertions.assertNull(r[3]);
+ }
+
+ // ── helper ─────────────────────────────────────────────────────────────────
+
+ private static List<Object[]> writeAndReadBack(
+ String testName, SeaTunnelRowType rowType, List<SeaTunnelRow> rows) throws Exception {
+ Map<String, Object> writeConfig = new HashMap<>();
+ String tmpPath = BASE_PATH + "/" + testName + "/tmp";
+ String finalPath = BASE_PATH + "/" + testName + "/out";
+ writeConfig.put("tmp_path", tmpPath);
+ writeConfig.put("path", finalPath);
+ writeConfig.put("file_format_type", FileFormat.PARQUET.name());
+
+ FileSinkConfig sinkConfig =
+ new FileSinkConfig(
+ ReadonlyConfig.fromConfig(ConfigFactory.parseMap(writeConfig)), rowType);
+ ParquetWriteStrategy strategy = new ParquetWriteStrategy(sinkConfig);
+ ParquetReadStrategyTest.LocalConf hadoopConf =
+ new ParquetReadStrategyTest.LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ strategy.setCatalogTable(
+ CatalogTableUtil.getCatalogTable("test", null, null, testName, rowType));
+ strategy.init(hadoopConf, "test-" + testName, "test-" + testName, 0);
+ strategy.beginTransaction(1L);
+ for (SeaTunnelRow row : rows) {
+ strategy.write(row);
+ }
+ strategy.finishAndCloseFile();
+ strategy.close();
+
+ ParquetReadStrategy readStrategy = new ParquetReadStrategy();
+ readStrategy.init(hadoopConf);
+ List<String> readFiles = readStrategy.getFileNamesByPath(tmpPath);
+ Assertions.assertFalse(readFiles.isEmpty(), "no Parquet files produced under " + tmpPath);
+
+ List<Object[]> readBack = new ArrayList<>();
+ Collector<SeaTunnelRow> collector =
+ new Collector<SeaTunnelRow>() {
+ @Override
+ public void collect(SeaTunnelRow record) {
+ Object[] fields = new Object[record.getArity()];
+ for (int i = 0; i < record.getArity(); i++) {
+ fields[i] = record.getField(i);
+ }
+ readBack.add(fields);
+ }
+
+ @Override
+ public Object getCheckpointLock() {
+ return null;
+ }
+ };
+ for (String file : readFiles) {
+ // Must call getSeaTunnelRowTypeInfo() before read() — it initializes per-file state.
+ readStrategy.getSeaTunnelRowTypeInfo(file);
+ readStrategy.read(file, testName, collector);
+ }
+ readStrategy.close();
+ return readBack;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyEvolutionTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyEvolutionTest.java
new file mode 100644
index 0000000000..67850bf2ef
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyEvolutionTest.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.file.writer;
+
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.source.Collector;
+import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
+import org.apache.seatunnel.api.table.catalog.PhysicalColumn;
+import org.apache.seatunnel.api.table.catalog.TableIdentifier;
+import org.apache.seatunnel.api.table.schema.event.AlterTableAddColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableDropColumnEvent;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat;
+import org.apache.seatunnel.connectors.seatunnel.file.sink.config.FileSinkConfig;
+import org.apache.seatunnel.connectors.seatunnel.file.sink.writer.ParquetWriteStrategy;
+import org.apache.seatunnel.connectors.seatunnel.file.source.reader.ParquetReadStrategy;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
+
+/**
+ * End-to-end Parquet schema-evolution test through {@link ParquetWriteStrategy}'s real I/O path (no
+ * mocks, no NoOp strategy). Covers the bugs that the {@code AlterTableSchemaEventHandler} swap
+ * (commit 01cc356f2) was supposed to fix and that would regress if the file-rotation contract or
+ * the cached Avro schema invalidation breaks.
+ */
+@Slf4j
+public class ParquetWriteStrategyEvolutionTest {
+
+ private static final String BASE_PATH = "file:///tmp/seatunnel/parquet/evolution";
+
+ private static final TableIdentifier TABLE_ID =
+ TableIdentifier.of("test", null, null, "evolution_table");
+
+ /**
+ * ADD_COLUMN flow: write 3 rows of [id, name], ALTER ADD email, write 3 rows of [id, name,
+ * email], close. The strategy must rotate the file on schema change so that the old rows live
+ * in a Parquet file with the pre-ALTER schema and the new rows live in a Parquet file with the
+ * post-ALTER schema. Read everything back and verify both halves.
+ */
+ @DisabledOnOs(OS.WINDOWS)
+ @Test
+ public void testAddColumnRotatesFileAndPreservesAllRows() throws Exception {
+ SeaTunnelRowType baseRowType =
+ new SeaTunnelRowType(
+ new String[] {"id", "name"},
+ new SeaTunnelDataType[] {BasicType.INT_TYPE, BasicType.STRING_TYPE});
+
+ Map<String, Object> writeConfig = new HashMap<>();
+ String tmpPath = BASE_PATH + "/add-column/tmp";
+ String finalPath = BASE_PATH + "/add-column/out";
+ writeConfig.put("tmp_path", tmpPath);
+ writeConfig.put("path", finalPath);
+ writeConfig.put("file_format_type", FileFormat.PARQUET.name());
+ writeConfig.put("schema_evolution_enabled", true);
+
+ FileSinkConfig sinkConfig =
+ new FileSinkConfig(
+ ReadonlyConfig.fromConfig(ConfigFactory.parseMap(writeConfig)),
+ baseRowType);
+ ParquetWriteStrategy strategy = new ParquetWriteStrategy(sinkConfig);
+ ParquetReadStrategyTest.LocalConf hadoopConf =
+ new ParquetReadStrategyTest.LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ strategy.setCatalogTable(
+ CatalogTableUtil.getCatalogTable(
+ "test", null, null, "evolution_table", baseRowType));
+ strategy.init(hadoopConf, "evolution-test", "evolution-test", 0);
+ strategy.beginTransaction(1L);
+
+ // Phase 1: write 3 rows with pre-ALTER schema [id, name]
+ strategy.write(new SeaTunnelRow(new Object[] {1, "alice"}));
+ strategy.write(new SeaTunnelRow(new Object[] {2, "bob"}));
+ strategy.write(new SeaTunnelRow(new Object[] {3, "carol"}));
+
+ // ALTER TABLE ADD COLUMN email STRING
+ PhysicalColumn emailCol =
+ PhysicalColumn.builder()
+ .name("email")
+ .dataType(BasicType.STRING_TYPE)
+ .nullable(true)
+ .build();
+ AlterTableAddColumnEvent addEvent = AlterTableAddColumnEvent.add(TABLE_ID, emailCol);
+ strategy.applySchemaChange(addEvent);
+ // Mirror what the SeaTunnel framework does after a schema change: snapshot the
+ // current state and begin a new transaction so the next file gets a distinct path.
+ // In production, this happens via snapshotState() between record batches.
+ strategy.beginTransaction(2L);
+
+ // Phase 2: write 3 rows with post-ALTER schema [id, name, email]
+ strategy.write(new SeaTunnelRow(new Object[] {4, "dave", "dave@x.com"}));
+ strategy.write(new SeaTunnelRow(new Object[] {5, "eve", "eve@x.com"}));
+ strategy.write(new SeaTunnelRow(new Object[] {6, "frank", "frank@x.com"}));
+
+ strategy.finishAndCloseFile();
+ strategy.close();
+
+ // Read every file under tmpPath. ParquetReadStrategy caches readColumns per
+ // instance — use a fresh reader per file to surface each file's true schema.
+ ParquetReadStrategy listStrategy = new ParquetReadStrategy();
+ listStrategy.init(hadoopConf);
+ List<String> files = listStrategy.getFileNamesByPath(tmpPath);
+ listStrategy.close();
+ Assertions.assertTrue(
+ files.size() >= 2,
+ "expected at least 2 Parquet files (file rotation on ALTER), got " + files.size());
+
+ int totalRows = 0;
+ boolean sawPreAlterFile = false;
+ boolean sawPostAlterFile = false;
+ for (String file : files) {
+ ParquetReadStrategy perFile = new ParquetReadStrategy();
+ perFile.init(hadoopConf);
+ SeaTunnelRowType fileSchema = perFile.getSeaTunnelRowTypeInfo(file);
+ List<String> fields = Arrays.asList(fileSchema.getFieldNames());
+ List<SeaTunnelRow> rows = readRows(perFile, file);
+ perFile.close();
+ totalRows += rows.size();
+ if (fields.contains("email")) {
+ sawPostAlterFile = true;
+ Assertions.assertEquals(3, rows.size(), "post-ALTER file should hold 3 rows");
+ for (SeaTunnelRow row : rows) {
+ Assertions.assertNotNull(
+ row.getField(2), "email must be populated in post-ALTER file");
+ }
+ } else {
+ sawPreAlterFile = true;
+ Assertions.assertEquals(3, rows.size(), "pre-ALTER file should hold 3 rows");
+ Assertions.assertEquals(2, fileSchema.getFieldNames().length);
+ }
+ }
+ Assertions.assertTrue(sawPreAlterFile, "should find a Parquet file with pre-ALTER schema");
+ Assertions.assertTrue(
+ sawPostAlterFile, "should find a Parquet file with post-ALTER schema");
+ Assertions.assertEquals(6, totalRows, "expected 6 rows total across all files");
+ }
+
+ /**
+ * DROP_COLUMN flow: ensures rotation also triggers on column drops and that the writer doesn't
+ * try to keep writing the old wider schema. Writes [id, name, age] then drops age then writes
+ * [id, name] — must produce a pre-DROP file with 3 columns and a post-DROP file with 2.
+ */
+ @DisabledOnOs(OS.WINDOWS)
+ @Test
+ public void testDropColumnRotatesFile() throws Exception {
+ SeaTunnelRowType baseRowType =
+ new SeaTunnelRowType(
+ new String[] {"id", "name", "age"},
+ new SeaTunnelDataType[] {
+ BasicType.INT_TYPE, BasicType.STRING_TYPE, BasicType.INT_TYPE
+ });
+
+ Map<String, Object> writeConfig = new HashMap<>();
+ String tmpPath = BASE_PATH + "/drop-column/tmp";
+ String finalPath = BASE_PATH + "/drop-column/out";
+ writeConfig.put("tmp_path", tmpPath);
+ writeConfig.put("path", finalPath);
+ writeConfig.put("file_format_type", FileFormat.PARQUET.name());
+ writeConfig.put("schema_evolution_enabled", true);
+
+ FileSinkConfig sinkConfig =
+ new FileSinkConfig(
+ ReadonlyConfig.fromConfig(ConfigFactory.parseMap(writeConfig)),
+ baseRowType);
+ ParquetWriteStrategy strategy = new ParquetWriteStrategy(sinkConfig);
+ ParquetReadStrategyTest.LocalConf hadoopConf =
+ new ParquetReadStrategyTest.LocalConf(FS_DEFAULT_NAME_DEFAULT);
+ strategy.setCatalogTable(
+ CatalogTableUtil.getCatalogTable(
+ "test", null, null, "evolution_table", baseRowType));
+ strategy.init(hadoopConf, "drop-test", "drop-test", 0);
+ strategy.beginTransaction(1L);
+
+ strategy.write(new SeaTunnelRow(new Object[] {1, "alice", 30}));
+ strategy.write(new SeaTunnelRow(new Object[] {2, "bob", 25}));
+
+ AlterTableDropColumnEvent dropEvent = new AlterTableDropColumnEvent(TABLE_ID, "age");
+ strategy.applySchemaChange(dropEvent);
+ strategy.beginTransaction(2L);
+
+ strategy.write(new SeaTunnelRow(new Object[] {3, "carol"}));
+ strategy.write(new SeaTunnelRow(new Object[] {4, "dave"}));
+
+ strategy.finishAndCloseFile();
+ strategy.close();
+
+ ParquetReadStrategy listStrategy = new ParquetReadStrategy();
+ listStrategy.init(hadoopConf);
+ List<String> files = listStrategy.getFileNamesByPath(tmpPath);
+ listStrategy.close();
+ Assertions.assertTrue(files.size() >= 2, "expected file rotation on DROP COLUMN");
+
+ int rowsWith3Cols = 0;
+ int rowsWith2Cols = 0;
+ for (String file : files) {
+ // Fresh reader per file because ParquetReadStrategy caches readColumns across calls.
+ ParquetReadStrategy perFile = new ParquetReadStrategy();
+ perFile.init(hadoopConf);
+ SeaTunnelRowType fileSchema = perFile.getSeaTunnelRowTypeInfo(file);
+ int cols = fileSchema.getFieldNames().length;
+ int rowCount = readRows(perFile, file).size();
+ perFile.close();
+ if (cols == 3) {
+ rowsWith3Cols += rowCount;
+ } else if (cols == 2) {
+ rowsWith2Cols += rowCount;
+ }
+ }
+ Assertions.assertEquals(2, rowsWith3Cols, "pre-DROP file should hold 2 rows with 3 cols");
+ Assertions.assertEquals(2, rowsWith2Cols, "post-DROP file should hold 2 rows with 2 cols");
+ }
+
+ // ── helpers ────────────────────────────────────────────────────────────────
+
+ private static List<SeaTunnelRow> readRows(ParquetReadStrategy readStrategy, String file)
+ throws Exception {
+ List<SeaTunnelRow> rows = new ArrayList<>();
+ Collector<SeaTunnelRow> collector =
+ new Collector<SeaTunnelRow>() {
+ @Override
+ public void collect(SeaTunnelRow record) {
+ rows.add(record);
+ }
+
+ @Override
+ public Object getCheckpointLock() {
+ return null;
+ }
+ };
+ readStrategy.read(file, "evolution_table", collector);
+ return rows;
+ }
+}
diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/flow/TransformFlowLifeCycle.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/flow/TransformFlowLifeCycle.java
index 3d83db6690..464effa634 100644
--- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/flow/TransformFlowLifeCycle.java
+++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/flow/TransformFlowLifeCycle.java
@@ -106,7 +106,16 @@ public class TransformFlowLifeCycle<T> extends ActionFlowLifeCycle
return;
}
SchemaChangeEvent event = (SchemaChangeEvent) record.getData();
- for (SeaTunnelTransform<T> t : transform) {
+ for (int i = 0; i < transform.size(); i++) {
+ SeaTunnelTransform<T> t = transform.get(i);
+ // Refresh this transform's input from upstream's post-event produced schema so
+ // its catalog matches the actual row layout it will receive. Without this, each
+ // transform applies ALTER to its own stale local catalog, diverging from the
+ // upstream's actual output positions and breaking name-based field access (SQL
+ // projections, FilterField excludes) after live ALTER ADD COLUMN.
+ if (i > 0) {
+ t.setInputCatalogTables(transform.get(i - 1).getProducedCatalogTables());
+ }
SchemaChangeEvent eventBefore = event;
event = t.mapSchemaChangeEvent(eventBefore);
if (event == null) {
diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractCatalogSupportMapTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractCatalogSupportMapTransform.java
index 6380d88539..af3ddf8a8b 100644
--- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractCatalogSupportMapTransform.java
+++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractCatalogSupportMapTransform.java
@@ -18,6 +18,10 @@
package org.apache.seatunnel.transform.common;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.TableSchema;
+import org.apache.seatunnel.api.table.schema.event.AlterTableEvent;
+import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent;
+import org.apache.seatunnel.api.table.schema.handler.AlterTableSchemaEventHandler;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.transform.SeaTunnelMapTransform;
@@ -41,4 +45,48 @@ public abstract class AbstractCatalogSupportMapTransform
public SeaTunnelRow map(SeaTunnelRow row) {
return transform(row);
}
+
+ /**
+ * Default schema-change handler for map transforms that cache field indices at init time.
+ *
+ * <p>Updates {@code inputCatalogTable} from the event, resets the output-schema cache, then
+ * re-runs {@link #transformTableSchema()} so that any field-index arrays (e.g. {@code
+ * inputValueIndexList} in {@code FilterFieldTransform}, {@code fieldsIndex} / {@code
+ * rowContainerGenerator} in {@code MultipleFieldOutputTransform}) are rebuilt against the new
+ * schema before the next data row arrives.
+ *
+ * <p>Subclasses that need special handling (e.g. {@code TableRenameTransform}, {@code
+ * FieldRenameTransform}) continue to override this method; their overrides take precedence via
+ * normal Java polymorphism and are not affected.
+ */
+ @Override
+ public SchemaChangeEvent mapSchemaChangeEvent(SchemaChangeEvent event) {
+ if (event instanceof AlterTableEvent) {
+ TableSchema newSchema =
+ new AlterTableSchemaEventHandler()
+ .reset(inputCatalogTable.getTableSchema())
+ .apply(event);
+ inputCatalogTable =
+ CatalogTable.of(
+ inputCatalogTable.getTableId(),
+ newSchema,
+ inputCatalogTable.getOptions(),
+ inputCatalogTable.getPartitionKeys(),
+ inputCatalogTable.getComment(),
+ inputCatalogTable.getTableId().getCatalogName(),
+ inputCatalogTable.getMetadataSchema());
+ outputCatalogTable = null;
+ // Rebuild all derived field-index state eagerly so the next transformRow() call
+ // sees correct indices without needing a lazy trigger.
+ transformTableSchema();
+ }
+ return event;
+ }
+
+ /** Eagerly rebuild any field-index/row-container caches that depend on inputCatalogTable. */
+ @Override
+ public void setInputCatalogTable(@NonNull CatalogTable inputCatalogTable) {
+ super.setInputCatalogTable(inputCatalogTable);
+ transformTableSchema();
+ }
}
diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractMultiCatalogTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractMultiCatalogTransform.java
index e432afb05e..0d3a5365c6 100644
--- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractMultiCatalogTransform.java
+++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractMultiCatalogTransform.java
@@ -19,6 +19,8 @@ package org.apache.seatunnel.transform.common;
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.schema.event.AlterTableEvent;
+import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.transform.SeaTunnelTransform;
@@ -102,6 +104,74 @@ public abstract class AbstractMultiCatalogTransform implements SeaTunnelTransfor
return outputCatalogTables.get(0);
}
+ /**
+ * Dispatches the schema-change event to the inner per-table transform that owns the affected
+ * table, then refreshes {@link #outputCatalogTables} so downstream sees the new schema.
+ *
+ * <p>Without this override, the wrapper would inherit the {@link SeaTunnelTransform} no-op
+ * default — the inner transforms in {@link #transformMap} would never see ALTER, their cached
+ * {@code inputCatalogTable} would stay at the original schema, and post-ALTER rows would have
+ * their new column values silently truncated by the inner transform's row container.
+ */
+ @Override
+ public SchemaChangeEvent mapSchemaChangeEvent(SchemaChangeEvent event) {
+ String targetTableId = event.tablePath().toString();
+ SeaTunnelTransform<SeaTunnelRow> targetTransform = transformMap.get(targetTableId);
+ if (targetTransform != null) {
+ targetTransform.mapSchemaChangeEvent(event);
+ refreshOutputCatalogTables();
+ // Propagate this transform's actual produced catalog through the chain via the event's
+ // changeAfter field (existing API designed for exactly this). Downstream transforms
+ // and the sink read changeAfter to adopt upstream's actual row layout instead of
+ // re-applying ALTER locally — which would diverge from the actual row order.
+ if (event instanceof AlterTableEvent) {
+ outputCatalogTables.stream()
+ .filter(t -> t.getTableId().toTablePath().toString().equals(targetTableId))
+ .findFirst()
+ .ifPresent(produced -> ((AlterTableEvent) event).setChangeAfter(produced));
+ }
+ }
+ return event;
+ }
+
+ /**
+ * Re-derives this wrapper (and its inner per-table transforms) from upstream's post-event
+ * produced schema. Called by the engine after an upstream transform has mapped a schema-change
+ * event. This ensures inner transforms see the same column order their actual data rows carry,
+ * instead of applying ALTER events to a stale local view.
+ */
+ @Override
+ public void setInputCatalogTables(List<CatalogTable> newInputCatalogTables) {
+ if (newInputCatalogTables == null || newInputCatalogTables.isEmpty()) {
+ return;
+ }
+ this.inputCatalogTables = newInputCatalogTables;
+ for (CatalogTable newInput : newInputCatalogTables) {
+ String tableId = newInput.getTableId().toTablePath().toString();
+ SeaTunnelTransform<SeaTunnelRow> inner = transformMap.get(tableId);
+ if (inner instanceof AbstractSeaTunnelTransform) {
+ ((AbstractSeaTunnelTransform<?, ?>) inner).setInputCatalogTable(newInput);
+ }
+ }
+ refreshOutputCatalogTables();
+ }
+
+ private void refreshOutputCatalogTables() {
+ this.outputCatalogTables =
+ inputCatalogTables.stream()
+ .map(
+ inputCatalogTable -> {
+ String tableName =
+ inputCatalogTable.getTableId().toTablePath().toString();
+ SeaTunnelTransform<SeaTunnelRow> inner =
+ transformMap.get(tableName);
+ return inner != null
+ ? inner.getProducedCatalogTable()
+ : inputCatalogTable;
+ })
+ .collect(Collectors.toList());
+ }
+
@Override
public void setTypeInfo(SeaTunnelDataType<SeaTunnelRow> inputDataType) {}
}
diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractSeaTunnelTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractSeaTunnelTransform.java
index edbba3ff08..569cdf8ad3 100644
--- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractSeaTunnelTransform.java
+++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractSeaTunnelTransform.java
@@ -59,6 +59,17 @@ public abstract class AbstractSeaTunnelTransform<T, R> implements SeaTunnelTrans
return outputCatalogTable;
}
+ /**
+ * Replaces this transform's input catalog table and invalidates derived state. Subclasses with
+ * extra cached state (e.g., row container generators, SQL engines) should override to clear
+ * those too. The default implementation is sufficient for transforms whose state is fully
+ * captured by {@link #inputCatalogTable} and {@link #outputCatalogTable}.
+ */
+ public void setInputCatalogTable(@NonNull CatalogTable inputCatalogTable) {
+ this.inputCatalogTable = inputCatalogTable;
+ this.outputCatalogTable = null;
+ }
+
@Override
public List<CatalogTable> getProducedCatalogTables() {
return Collections.singletonList(getProducedCatalogTable());
diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/SQLTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/SQLTransform.java
index bd6c4708e9..c17f2f88f6 100644
--- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/SQLTransform.java
+++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/SQLTransform.java
@@ -27,6 +27,9 @@ import org.apache.seatunnel.api.table.catalog.ConstraintKey;
import org.apache.seatunnel.api.table.catalog.PhysicalColumn;
import org.apache.seatunnel.api.table.catalog.TableIdentifier;
import org.apache.seatunnel.api.table.catalog.TableSchema;
+import org.apache.seatunnel.api.table.schema.event.AlterTableEvent;
+import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent;
+import org.apache.seatunnel.api.table.schema.handler.AlterTableSchemaEventHandler;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
@@ -181,8 +184,45 @@ public class SQLTransform extends AbstractCatalogSupportFlatMapTransform {
return inputCatalogTable.getTableId().copy();
}
+ @Override
+ public SchemaChangeEvent mapSchemaChangeEvent(SchemaChangeEvent event) {
+ if (event instanceof AlterTableEvent) {
+ TableSchema newSchema =
+ new AlterTableSchemaEventHandler()
+ .reset(inputCatalogTable.getTableSchema())
+ .apply(event);
+ inputCatalogTable =
+ CatalogTable.of(
+ inputCatalogTable.getTableId(),
+ newSchema,
+ inputCatalogTable.getOptions(),
+ inputCatalogTable.getPartitionKeys(),
+ inputCatalogTable.getComment(),
+ inputCatalogTable.getTableId().getCatalogName(),
+ inputCatalogTable.getMetadataSchema());
+ // Force re-initialization: sqlEngine caches allColumnsCount and outRowType based on
+ // the old inputRowType. After a schema change (e.g. ADD COLUMN), select * would
+ // produce an ArrayIndexOutOfBoundsException because the cached output size is stale.
+ sqlEngine = null;
+ outputCatalogTable = null;
+ }
+ return event;
+ }
+
+ /**
+ * Replace input schema from upstream and invalidate the cached SQL engine so the next {@code
+ * transformRow} re-initializes against the new schema.
+ */
+ @Override
+ public void setInputCatalogTable(@NonNull CatalogTable inputCatalogTable) {
+ super.setInputCatalogTable(inputCatalogTable);
+ this.sqlEngine = null;
+ }
+
@Override
public void close() {
- sqlEngine.close();
+ if (sqlEngine != null) {
+ sqlEngine.close();
+ }
}
}
diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/ChainTimestampPreservationTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/ChainTimestampPreservationTest.java
new file mode 100644
index 0000000000..c299966257
--- /dev/null
+++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/ChainTimestampPreservationTest.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.transform.metadata;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.Column;
+import org.apache.seatunnel.api.table.catalog.MetadataColumn;
+import org.apache.seatunnel.api.table.catalog.MetadataSchema;
+import org.apache.seatunnel.api.table.catalog.PhysicalColumn;
+import org.apache.seatunnel.api.table.catalog.TableIdentifier;
+import org.apache.seatunnel.api.table.catalog.TablePath;
+import org.apache.seatunnel.api.table.catalog.TableSchema;
+import org.apache.seatunnel.api.table.schema.event.AlterTableAddColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableColumnsEvent;
+import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.CommonOptions;
+import org.apache.seatunnel.api.table.type.LocalTimeType;
+import org.apache.seatunnel.api.table.type.MetadataUtil;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.transform.filter.FilterFieldMultiCatalogTransform;
+import org.apache.seatunnel.transform.filterrowkind.FieldRowKindMultiCatalogTransform;
+import org.apache.seatunnel.transform.rowkind.RowKindExtractorMultiCatalogTransform;
+import org.apache.seatunnel.transform.sql.SQLMultiCatalogFlatMapTransform;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Reproduces the production CCE: after ALTER ADD COLUMN, the row at the sink boundary has String at
+ * the position of a TIMESTAMP column instead of LocalDateTime. Verifies the chain preserves
+ * LocalDateTime through ALL transforms and emits it at the FINAL output.
+ *
+ * <p>This is the missing test for the v24 stage failure. If the row at the end of the chain has
+ * LocalDateTime, the bug is sink-side (order divergence already fixed). If it has String, the
+ * conversion happens inside the chain and we have a different fix to find.
+ */
+public class ChainTimestampPreservationTest {
+
+ private static final TablePath TBL = TablePath.of("ricky_test", "static_inventory");
+
+ /** Builds a base table mirroring the production pipeline: includes a TIMESTAMP column. */
+ private static CatalogTable buildBaseTable() {
+ List<Column> metadataCols = new ArrayList<>();
+ metadataCols.add(
+ MetadataColumn.of(
+ CommonOptions.EVENT_TIME.getName(),
+ BasicType.LONG_TYPE,
+ (Long) null,
+ true,
+ null,
+ null));
+ metadataCols.add(
+ MetadataColumn.of(
+ CommonOptions.DELAY.getName(),
+ BasicType.LONG_TYPE,
+ (Long) null,
+ true,
+ null,
+ null));
+ return CatalogTable.of(
+ TableIdentifier.of("catalog", TBL),
+ TableSchema.builder()
+ .column(
+ PhysicalColumn.of(
+ "id", BasicType.LONG_TYPE, (Long) null, false, null, null))
+ .column(
+ PhysicalColumn.of(
+ "name",
+ BasicType.STRING_TYPE,
+ (Long) null,
+ true,
+ null,
+ null))
+ .column(
+ PhysicalColumn.of(
+ "price",
+ BasicType.DOUBLE_TYPE,
+ (Long) null,
+ true,
+ null,
+ null))
+ .column(
+ PhysicalColumn.of(
+ "created_at",
+ LocalTimeType.LOCAL_DATE_TIME_TYPE,
+ (Long) null,
+ true,
+ null,
+ null))
+ .build(),
+ new HashMap<>(),
+ new ArrayList<>(),
+ "comment",
+ "test",
+ MetadataSchema.builder().columns(metadataCols).build());
+ }
+
+ @Test
+ void chainPreservesLocalDateTimeAtTimestampColumn() {
+ CatalogTable baseTable = buildBaseTable();
+
+ // Build the production wrapper chain
+ Map<String, Object> filterRowKindCfg = new HashMap<>();
+ filterRowKindCfg.put("include_kinds", Arrays.asList("INSERT", "UPDATE_AFTER", "DELETE"));
+ FieldRowKindMultiCatalogTransform filterRowKind =
+ new FieldRowKindMultiCatalogTransform(
+ Collections.singletonList(baseTable),
+ ReadonlyConfig.fromMap(filterRowKindCfg));
+
+ CatalogTable filterRowKindOut = filterRowKind.getProducedCatalogTables().get(0);
+ Map<String, String> metaMapping = new LinkedHashMap<>();
+ metaMapping.put("EventTime", "c_event_time");
+ metaMapping.put("Delay", "c_delay");
+ Map<String, Object> metaCfg = new HashMap<>();
+ metaCfg.put("metadata_fields", metaMapping);
+ MetadataMultiCatalogTransform metadata =
+ new MetadataMultiCatalogTransform(
+ Collections.singletonList(filterRowKindOut),
+ ReadonlyConfig.fromMap(metaCfg));
+
+ CatalogTable metadataOut = metadata.getProducedCatalogTables().get(0);
+ Map<String, Object> rkeCfg = new HashMap<>();
+ rkeCfg.put("custom_field_name", "c_operation_type");
+ rkeCfg.put("transform_type", "FULL");
+ RowKindExtractorMultiCatalogTransform rowKindExtractor =
+ new RowKindExtractorMultiCatalogTransform(
+ Collections.singletonList(metadataOut), ReadonlyConfig.fromMap(rkeCfg));
+
+ CatalogTable rkeOut = rowKindExtractor.getProducedCatalogTables().get(0);
+ Map<String, Object> sqlCfg = new HashMap<>();
+ sqlCfg.put("query", "select * from " + rkeOut.getTableId().getTableName());
+ SQLMultiCatalogFlatMapTransform sql =
+ new SQLMultiCatalogFlatMapTransform(
+ Collections.singletonList(rkeOut), ReadonlyConfig.fromMap(sqlCfg));
+
+ CatalogTable sqlOut = sql.getProducedCatalogTables().get(0);
+ Map<String, Object> filterFieldCfg = new HashMap<>();
+ filterFieldCfg.put("exclude_fields", Arrays.asList("c_delay"));
+ FilterFieldMultiCatalogTransform filterField =
+ new FilterFieldMultiCatalogTransform(
+ Collections.singletonList(sqlOut), ReadonlyConfig.fromMap(filterFieldCfg));
+
+ List<org.apache.seatunnel.api.transform.SeaTunnelTransform<SeaTunnelRow>> chain =
+ Arrays.asList(filterRowKind, metadata, rowKindExtractor, sql, filterField);
+
+ // Pre-ALTER: process a row with LocalDateTime at created_at, verify it survives
+ LocalDateTime sampleDateTime = LocalDateTime.of(2026, 4, 27, 5, 24, 27);
+ SeaTunnelRow preRow =
+ new SeaTunnelRow(new Object[] {1L, "Widget A", 19.99d, sampleDateTime});
+ preRow.setTableId(TBL.getFullName());
+ MetadataUtil.setEventTime(preRow, 1700000000000L);
+ MetadataUtil.setDelay(preRow, 50L);
+
+ SeaTunnelRow afterFilterPre = filterRowKind.map(preRow);
+ SeaTunnelRow afterMetaPre = metadata.map(afterFilterPre);
+ SeaTunnelRow afterRkePre = rowKindExtractor.map(afterMetaPre);
+ List<SeaTunnelRow> sqlOutPre = sql.flatMap(afterRkePre);
+ SeaTunnelRow afterFilterFieldPre = filterField.map(sqlOutPre.get(0));
+
+ // Find created_at in the post-Filter row by looking it up in the produced catalog.
+ // (Using catalog's column name lookup avoids hardcoding positions.)
+ CatalogTable filterFieldOut = filterField.getProducedCatalogTables().get(0);
+ int createdAtIdxPre =
+ filterFieldOut.getTableSchema().toPhysicalRowDataType().indexOf("created_at");
+ Assertions.assertTrue(
+ createdAtIdxPre >= 0, "Pre-ALTER: created_at must exist in final catalog");
+ Object createdAtValPre = afterFilterFieldPre.getField(createdAtIdxPre);
+ Assertions.assertTrue(
+ createdAtValPre instanceof LocalDateTime,
+ "Pre-ALTER: created_at value at position "
+ + createdAtIdxPre
+ + " must be LocalDateTime, was "
+ + (createdAtValPre == null ? "null" : createdAtValPre.getClass().getName())
+ + " = "
+ + createdAtValPre);
+
+ // ============ Live ALTER ADD COLUMN supplier AFTER price ============
+ TableIdentifier tid = baseTable.getTableId();
+ SchemaChangeEvent alter =
+ new AlterTableColumnsEvent(tid)
+ .addEvent(
+ AlterTableAddColumnEvent.addAfter(
+ tid,
+ PhysicalColumn.of(
+ "supplier",
+ BasicType.STRING_TYPE,
+ (Long) null,
+ true,
+ null,
+ null),
+ "price"));
+
+ // Mimic engine: setInputCatalogTables on each downstream, then mapSchemaChangeEvent
+ SchemaChangeEvent ev = alter;
+ for (int i = 0; i < chain.size(); i++) {
+ if (i > 0) {
+ chain.get(i).setInputCatalogTables(chain.get(i - 1).getProducedCatalogTables());
+ }
+ ev = chain.get(i).mapSchemaChangeEvent(ev);
+ }
+
+ // Post-ALTER: row now has 5 base cols (id, name, price, supplier, created_at)
+ SeaTunnelRow postRow =
+ new SeaTunnelRow(
+ new Object[] {2L, "Premium A", 59.99d, "OfficeWorld", sampleDateTime});
+ postRow.setTableId(TBL.getFullName());
+ MetadataUtil.setEventTime(postRow, 1700000010000L);
+ MetadataUtil.setDelay(postRow, 60L);
+
+ SeaTunnelRow afterFilterPost = filterRowKind.map(postRow);
+ SeaTunnelRow afterMetaPost = metadata.map(afterFilterPost);
+ SeaTunnelRow afterRkePost = rowKindExtractor.map(afterMetaPost);
+ List<SeaTunnelRow> sqlOutPost = sql.flatMap(afterRkePost);
+ SeaTunnelRow afterFilterFieldPost = filterField.map(sqlOutPost.get(0));
+
+ // Find created_at again in the new produced catalog
+ CatalogTable filterFieldOutPost = filterField.getProducedCatalogTables().get(0);
+ int createdAtIdxPost =
+ filterFieldOutPost.getTableSchema().toPhysicalRowDataType().indexOf("created_at");
+ Assertions.assertTrue(
+ createdAtIdxPost >= 0, "Post-ALTER: created_at must exist in final catalog");
+
+ Object createdAtValPost = afterFilterFieldPost.getField(createdAtIdxPost);
+ Assertions.assertTrue(
+ createdAtValPost instanceof LocalDateTime,
+ "Post-ALTER: created_at value at position "
+ + createdAtIdxPost
+ + " must be LocalDateTime, was "
+ + (createdAtValPost == null
+ ? "null"
+ : createdAtValPost.getClass().getName())
+ + " = "
+ + createdAtValPost
+ + ". This reproduces the production CCE — chain converted "
+ + "LocalDateTime to String somewhere.");
+
+ Assertions.assertEquals(
+ sampleDateTime,
+ createdAtValPost,
+ "Post-ALTER: created_at value should be unchanged through chain");
+ }
+}
diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/MetadataMultiCatalogSchemaChangeTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/MetadataMultiCatalogSchemaChangeTest.java
new file mode 100644
index 0000000000..c4833a603e
--- /dev/null
+++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/MetadataMultiCatalogSchemaChangeTest.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.transform.metadata;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.Column;
+import org.apache.seatunnel.api.table.catalog.MetadataColumn;
+import org.apache.seatunnel.api.table.catalog.MetadataSchema;
+import org.apache.seatunnel.api.table.catalog.PhysicalColumn;
+import org.apache.seatunnel.api.table.catalog.TableIdentifier;
+import org.apache.seatunnel.api.table.catalog.TablePath;
+import org.apache.seatunnel.api.table.catalog.TableSchema;
+import org.apache.seatunnel.api.table.schema.event.AlterTableAddColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableColumnsEvent;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.CommonOptions;
+import org.apache.seatunnel.api.table.type.MetadataUtil;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Reproduces upstream Bug 1: schema-change events are silently dropped by {@code
+ * AbstractMultiCatalogTransform} subclasses, leaving the inner per-table transforms with stale
+ * {@code inputCatalogTable} after ALTER. Result: post-ALTER rows lose new column values.
+ *
+ * <p>This test calls the OUTER wrapper ({@code MetadataMultiCatalogTransform}) — the same instance
+ * the SeaTunnel engine constructs via the factory and feeds via {@code TransformFlowLifeCycle}.
+ * Earlier {@code TransformChainLiveAlterTest} called the inner transform directly, missing the bug.
+ *
+ * <p>Without the fix in {@code AbstractMultiCatalogTransform.mapSchemaChangeEvent}, the wrapper's
+ * default no-op returns the event without dispatching to inner transforms, so this test FAILS at
+ * the post-ALTER arity assertion.
+ */
+public class MetadataMultiCatalogSchemaChangeTest {
+
+ private static final TablePath TBL = TablePath.of("ricky_test", "static_inventory");
+
+ private static CatalogTable buildBaseTable() {
+ List<Column> metadata = new ArrayList<>();
+ metadata.add(
+ MetadataColumn.of(
+ CommonOptions.EVENT_TIME.getName(),
+ BasicType.LONG_TYPE,
+ (Long) null,
+ true,
+ null,
+ null));
+ metadata.add(
+ MetadataColumn.of(
+ CommonOptions.DELAY.getName(),
+ BasicType.LONG_TYPE,
+ (Long) null,
+ true,
+ null,
+ null));
+ return CatalogTable.of(
+ TableIdentifier.of("catalog", TBL),
+ TableSchema.builder()
+ .column(
+ PhysicalColumn.of(
+ "id", BasicType.LONG_TYPE, (Long) null, false, null, null))
+ .column(
+ PhysicalColumn.of(
+ "name",
+ BasicType.STRING_TYPE,
+ (Long) null,
+ true,
+ null,
+ null))
+ .build(),
+ new HashMap<>(),
+ new ArrayList<>(),
+ "comment",
+ "test",
+ MetadataSchema.builder().columns(metadata).build());
+ }
+
+ @Test
+ void multiCatalogWrapperPropagatesSchemaChangeToInnerTransforms() {
+ CatalogTable baseTable = buildBaseTable();
+
+ Map<String, String> metaMapping = new LinkedHashMap<>();
+ metaMapping.put("EventTime", "c_event_time");
+ metaMapping.put("Delay", "c_delay");
+ Map<String, Object> cfg = new HashMap<>();
+ cfg.put("metadata_fields", metaMapping);
+ // table_match_regex defaults to ".*" which matches all tables; this lets the wrapper apply
+ // its config to the only inner table without per-table overrides.
+ ReadonlyConfig config = ReadonlyConfig.fromMap(cfg);
+
+ MetadataMultiCatalogTransform wrapper =
+ new MetadataMultiCatalogTransform(Collections.singletonList(baseTable), config);
+
+ // Pre-ALTER row: 2 base cols
+ SeaTunnelRow preRow = new SeaTunnelRow(new Object[] {1L, "Widget A"});
+ preRow.setTableId(TBL.getFullName());
+ MetadataUtil.setEventTime(preRow, 1700000000000L);
+ MetadataUtil.setDelay(preRow, 50L);
+
+ SeaTunnelRow preOut = wrapper.map(preRow);
+ Assertions.assertEquals(4, preOut.getArity(), "pre-ALTER: 2 base + 2 metadata = 4");
+
+ // Live ALTER ADD COLUMN discount_pct, is_featured
+ TableIdentifier tid = baseTable.getTableId();
+ AlterTableColumnsEvent alter =
+ new AlterTableColumnsEvent(tid)
+ .addEvent(
+ AlterTableAddColumnEvent.add(
+ tid,
+ PhysicalColumn.of(
+ "discount_pct",
+ BasicType.DOUBLE_TYPE,
+ (Long) null,
+ true,
+ null,
+ null)))
+ .addEvent(
+ AlterTableAddColumnEvent.add(
+ tid,
+ PhysicalColumn.of(
+ "is_featured",
+ BasicType.BOOLEAN_TYPE,
+ (Long) null,
+ true,
+ null,
+ null)));
+
+ // This is the exact call TransformFlowLifeCycle.received makes on the outer wrapper.
+ // Without the fix: wrapper's default no-op returns event unchanged; inner transformMap
+ // entries never see ALTER; their inputCatalogTable stays at 2 cols.
+ wrapper.mapSchemaChangeEvent(alter);
+
+ // Post-ALTER row: 4 base cols (id, name, discount_pct, is_featured)
+ SeaTunnelRow postRow =
+ new SeaTunnelRow(new Object[] {2L, "Premium A", 10.00d, Boolean.TRUE});
+ postRow.setTableId(TBL.getFullName());
+ MetadataUtil.setEventTime(postRow, 1700000010000L);
+ MetadataUtil.setDelay(postRow, 60L);
+
+ SeaTunnelRow postOut = wrapper.map(postRow);
+
+ Assertions.assertEquals(
+ 6,
+ postOut.getArity(),
+ "post-ALTER MUST be arity 6 (4 base + 2 metadata). If 4, the wrapper "
+ + "swallowed the schema change without notifying inner MetadataTransform — "
+ + "exactly Bug 1.");
+ Assertions.assertEquals(2L, postOut.getField(0));
+ Assertions.assertEquals("Premium A", postOut.getField(1));
+ Assertions.assertEquals(
+ 10.00d,
+ postOut.getField(2),
+ "discount_pct must survive the wrapper after live ALTER");
+ Assertions.assertEquals(
+ Boolean.TRUE,
+ postOut.getField(3),
+ "is_featured must survive the wrapper after live ALTER");
+ }
+}
diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/ProductionPipelineSchemaChangeTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/ProductionPipelineSchemaChangeTest.java
new file mode 100644
index 0000000000..99c43affbb
--- /dev/null
+++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/ProductionPipelineSchemaChangeTest.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.transform.metadata;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.Column;
+import org.apache.seatunnel.api.table.catalog.MetadataColumn;
+import org.apache.seatunnel.api.table.catalog.MetadataSchema;
+import org.apache.seatunnel.api.table.catalog.PhysicalColumn;
+import org.apache.seatunnel.api.table.catalog.TableIdentifier;
+import org.apache.seatunnel.api.table.catalog.TablePath;
+import org.apache.seatunnel.api.table.catalog.TableSchema;
+import org.apache.seatunnel.api.table.schema.event.AlterTableAddColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableColumnsEvent;
+import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.CommonOptions;
+import org.apache.seatunnel.api.table.type.MetadataUtil;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.transform.filter.FilterFieldMultiCatalogTransform;
+import org.apache.seatunnel.transform.filterrowkind.FieldRowKindMultiCatalogTransform;
+import org.apache.seatunnel.transform.rowkind.RowKindExtractorMultiCatalogTransform;
+import org.apache.seatunnel.transform.sql.SQLMultiCatalogFlatMapTransform;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Full-pipeline regression test that mirrors a production CDC + transforms + sink pipeline using
+ * the actual MultiCatalog wrapper classes (the same instances the engine constructs via factories
+ * and feeds via {@link org.apache.seatunnel.engine.server.task.flow.TransformFlowLifeCycle
+ * TransformFlowLifeCycle}).
+ *
+ * <p>Pipeline: FieldRowKind → Metadata → RowKindExtractor → SQL → FieldField. This is the exact
+ * shape of the user's production pipeline that exposed Bug 1.
+ *
+ * <p>Pre-fix: post-ALTER inserts had their new column values silently dropped. Post-fix (override
+ * of {@code mapSchemaChangeEvent} in {@link
+ * org.apache.seatunnel.transform.common.AbstractMultiCatalogTransform
+ * AbstractMultiCatalogTransform}): post-ALTER rows preserve all values through the chain.
+ */
+public class ProductionPipelineSchemaChangeTest {
+
+ private static final TablePath TBL = TablePath.of("ricky_test", "static_inventory");
+
+ private static CatalogTable buildBaseTable() {
+ List<Column> metadata = new ArrayList<>();
+ metadata.add(
+ MetadataColumn.of(
+ CommonOptions.EVENT_TIME.getName(),
+ BasicType.LONG_TYPE,
+ (Long) null,
+ true,
+ null,
+ null));
+ metadata.add(
+ MetadataColumn.of(
+ CommonOptions.DELAY.getName(),
+ BasicType.LONG_TYPE,
+ (Long) null,
+ true,
+ null,
+ null));
+ return CatalogTable.of(
+ TableIdentifier.of("catalog", TBL),
+ TableSchema.builder()
+ .column(
+ PhysicalColumn.of(
+ "id", BasicType.LONG_TYPE, (Long) null, false, null, null))
+ .column(
+ PhysicalColumn.of(
+ "name",
+ BasicType.STRING_TYPE,
+ (Long) null,
+ true,
+ null,
+ null))
+ .build(),
+ new HashMap<>(),
+ new ArrayList<>(),
+ "comment",
+ "test",
+ MetadataSchema.builder().columns(metadata).build());
+ }
+
+ @Test
+ void productionWrapperPipelinePreservesPostAlterValues() {
+ CatalogTable baseTable = buildBaseTable();
+
+ // 1. FieldRowKindMultiCatalogTransform — filter by INSERT/UPDATE_AFTER/DELETE
+ Map<String, Object> filterRowKindCfg = new HashMap<>();
+ filterRowKindCfg.put("include_kinds", Arrays.asList("INSERT", "UPDATE_AFTER", "DELETE"));
+ FieldRowKindMultiCatalogTransform filterRowKind =
+ new FieldRowKindMultiCatalogTransform(
+ Collections.singletonList(baseTable),
+ ReadonlyConfig.fromMap(filterRowKindCfg));
+
+ // 2. MetadataMultiCatalogTransform — append c_event_time, c_delay
+ CatalogTable filterRowKindOut = filterRowKind.getProducedCatalogTables().get(0);
+ Map<String, String> metaMapping = new LinkedHashMap<>();
+ metaMapping.put("EventTime", "c_event_time");
+ metaMapping.put("Delay", "c_delay");
+ Map<String, Object> metaCfg = new HashMap<>();
+ metaCfg.put("metadata_fields", metaMapping);
+ MetadataMultiCatalogTransform metadata =
+ new MetadataMultiCatalogTransform(
+ Collections.singletonList(filterRowKindOut),
+ ReadonlyConfig.fromMap(metaCfg));
+
+ // 3. RowKindExtractorMultiCatalogTransform — append c_operation_type
+ CatalogTable metadataOut = metadata.getProducedCatalogTables().get(0);
+ Map<String, Object> rkeCfg = new HashMap<>();
+ rkeCfg.put("custom_field_name", "c_operation_type");
+ rkeCfg.put("transform_type", "FULL");
+ RowKindExtractorMultiCatalogTransform rowKindExtractor =
+ new RowKindExtractorMultiCatalogTransform(
+ Collections.singletonList(metadataOut), ReadonlyConfig.fromMap(rkeCfg));
+
+ // 4. SQLMultiCatalogFlatMapTransform — select *
+ CatalogTable rkeOut = rowKindExtractor.getProducedCatalogTables().get(0);
+ Map<String, Object> sqlCfg = new HashMap<>();
+ sqlCfg.put("query", "select * from " + rkeOut.getTableId().getTableName());
+ SQLMultiCatalogFlatMapTransform sql =
+ new SQLMultiCatalogFlatMapTransform(
+ Collections.singletonList(rkeOut), ReadonlyConfig.fromMap(sqlCfg));
+
+ // 5. FilterFieldMultiCatalogTransform — exclude c_delay
+ CatalogTable sqlOut = sql.getProducedCatalogTables().get(0);
+ Map<String, Object> filterFieldCfg = new HashMap<>();
+ filterFieldCfg.put("exclude_fields", Arrays.asList("c_delay"));
+ FilterFieldMultiCatalogTransform filterField =
+ new FilterFieldMultiCatalogTransform(
+ Collections.singletonList(sqlOut), ReadonlyConfig.fromMap(filterFieldCfg));
+
+ // List of all wrappers in chain order — engine iterates in this order during ALTER.
+ List<org.apache.seatunnel.api.transform.SeaTunnelTransform<SeaTunnelRow>> chain =
+ Arrays.asList(filterRowKind, metadata, rowKindExtractor, sql, filterField);
+
+ // ============ Pre-ALTER ============
+ SeaTunnelRow preRow = new SeaTunnelRow(new Object[] {1L, "Widget A"});
+ preRow.setTableId(TBL.getFullName());
+ MetadataUtil.setEventTime(preRow, 1700000000000L);
+ MetadataUtil.setDelay(preRow, 50L);
+
+ SeaTunnelRow afterFilter = filterRowKind.map(preRow);
+ Assertions.assertNotNull(afterFilter);
+ SeaTunnelRow afterMeta = metadata.map(afterFilter);
+ Assertions.assertEquals(4, afterMeta.getArity(), "pre-ALTER Metadata: 2+2=4");
+ SeaTunnelRow afterRke = rowKindExtractor.map(afterMeta);
+ Assertions.assertEquals(5, afterRke.getArity(), "pre-ALTER RowKind: 4+1=5");
+ List<SeaTunnelRow> sqlOutPre = sql.flatMap(afterRke);
+ Assertions.assertEquals(1, sqlOutPre.size());
+ Assertions.assertEquals(5, sqlOutPre.get(0).getArity(), "pre-ALTER SQL select *: 5");
+ SeaTunnelRow afterFilterField = filterField.map(sqlOutPre.get(0));
+ Assertions.assertEquals(
+ 4,
+ afterFilterField.getArity(),
+ "pre-ALTER FilterField: 5 - 1 (excluded c_delay) = 4");
+
+ // ============ Live ALTER ADD COLUMN discount_pct, is_featured ============
+ TableIdentifier tid = baseTable.getTableId();
+ SchemaChangeEvent alter =
+ new AlterTableColumnsEvent(tid)
+ .addEvent(
+ AlterTableAddColumnEvent.add(
+ tid,
+ PhysicalColumn.of(
+ "discount_pct",
+ BasicType.DOUBLE_TYPE,
+ (Long) null,
+ true,
+ null,
+ null)))
+ .addEvent(
+ AlterTableAddColumnEvent.add(
+ tid,
+ PhysicalColumn.of(
+ "is_featured",
+ BasicType.BOOLEAN_TYPE,
+ (Long) null,
+ true,
+ null,
+ null)));
+
+ // Mimic TransformFlowLifeCycle.received's schema-change branch: each downstream wrapper
+ // is refreshed from the previous wrapper's post-event produced schema before its own
+ // mapSchemaChangeEvent runs.
+ SchemaChangeEvent ev = alter;
+ for (int i = 0; i < chain.size(); i++) {
+ if (i > 0) {
+ chain.get(i).setInputCatalogTables(chain.get(i - 1).getProducedCatalogTables());
+ }
+ ev = chain.get(i).mapSchemaChangeEvent(ev);
+ }
+ Assertions.assertNotNull(ev, "schema-change event should propagate downstream");
+
+ // ============ Post-ALTER ============
+ SeaTunnelRow postRow =
+ new SeaTunnelRow(new Object[] {2L, "Premium A", 10.00d, Boolean.TRUE});
+ postRow.setTableId(TBL.getFullName());
+ MetadataUtil.setEventTime(postRow, 1700000010000L);
+ MetadataUtil.setDelay(postRow, 60L);
+
+ SeaTunnelRow postFilter = filterRowKind.map(postRow);
+ Assertions.assertEquals(
+ 4,
+ postFilter.getArity(),
+ "FilterRowKind passes 4 base cols through unchanged post-ALTER");
+ Assertions.assertEquals(10.00d, postFilter.getField(2));
+ Assertions.assertEquals(Boolean.TRUE, postFilter.getField(3));
+
+ SeaTunnelRow postMeta = metadata.map(postFilter);
+ Assertions.assertEquals(
+ 6, postMeta.getArity(), "Metadata post-ALTER: 4 base + 2 metadata = 6");
+ Assertions.assertEquals(
+ 10.00d, postMeta.getField(2), "discount_pct survives Metadata wrapper");
+ Assertions.assertEquals(
+ Boolean.TRUE, postMeta.getField(3), "is_featured survives Metadata wrapper");
+
+ SeaTunnelRow postRke = rowKindExtractor.map(postMeta);
+ Assertions.assertEquals(7, postRke.getArity(), "RowKindExtractor post-ALTER: 6 + 1 = 7");
+ Assertions.assertEquals(
+ 10.00d, postRke.getField(2), "discount_pct survives RowKindExtractor wrapper");
+ Assertions.assertEquals(
+ Boolean.TRUE, postRke.getField(3), "is_featured survives RowKindExtractor wrapper");
+
+ List<SeaTunnelRow> sqlOutPost = sql.flatMap(postRke);
+ Assertions.assertEquals(1, sqlOutPost.size());
+ SeaTunnelRow postSql = sqlOutPost.get(0);
+ Assertions.assertEquals(
+ 7,
+ postSql.getArity(),
+ "SQL select * post-ALTER MUST be 7 cols (4 base + 2 metadata + 1 rowkind)");
+ Assertions.assertEquals(10.00d, postSql.getField(2), "discount_pct survives SQL wrapper");
+ Assertions.assertEquals(
+ Boolean.TRUE, postSql.getField(3), "is_featured survives SQL wrapper");
+
+ SeaTunnelRow postFinal = filterField.map(postSql);
+ Assertions.assertEquals(
+ 6,
+ postFinal.getArity(),
+ "FilterField post-ALTER: 7 - 1 (c_delay excluded) = 6 — values preserved");
+ Assertions.assertEquals(
+ 10.00d,
+ postFinal.getField(2),
+ "discount_pct preserved at sink boundary — proves order-divergence fix works");
+ Assertions.assertEquals(
+ Boolean.TRUE,
+ postFinal.getField(3),
+ "is_featured preserved at sink boundary — proves order-divergence fix works");
+ }
+}
diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/TransformChainLiveAlterTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/TransformChainLiveAlterTest.java
new file mode 100644
index 0000000000..558cbe19c6
--- /dev/null
+++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/metadata/TransformChainLiveAlterTest.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.transform.metadata;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.Column;
+import org.apache.seatunnel.api.table.catalog.MetadataColumn;
+import org.apache.seatunnel.api.table.catalog.MetadataSchema;
+import org.apache.seatunnel.api.table.catalog.PhysicalColumn;
+import org.apache.seatunnel.api.table.catalog.TableIdentifier;
+import org.apache.seatunnel.api.table.catalog.TablePath;
+import org.apache.seatunnel.api.table.catalog.TableSchema;
+import org.apache.seatunnel.api.table.schema.event.AlterTableAddColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableColumnsEvent;
+import org.apache.seatunnel.api.table.schema.event.SchemaChangeEvent;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.CommonOptions;
+import org.apache.seatunnel.api.table.type.MetadataUtil;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.transform.filter.FilterFieldTransform;
+import org.apache.seatunnel.transform.filterrowkind.FilterRowKindTransform;
+import org.apache.seatunnel.transform.rowkind.RowKindExtractorTransform;
+import org.apache.seatunnel.transform.sql.SQLTransform;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Reproduces Bug 1 (live-ALTER column values dropped) by replaying the production transform chain
+ * in unit form: FilterRowKind → Metadata → RowKindExtractor.
+ *
+ * <p>The empirical stage A/B (no-transforms config = correct values; with-transforms config = null
+ * values) proves the drop happens inside the transform chain. Single-transform tests pass. This
+ * test exercises the chain to show whether the chain itself reproduces the bug, and serves as the
+ * green-light indicator that any fix actually closes Bug 1.
+ */
+public class TransformChainLiveAlterTest {
+
+ private static final TablePath TBL = TablePath.of("ricky_test", "static_inventory");
+
+ /** Builds a 2-col base table: id BIGINT, name STRING. */
+ private static CatalogTable buildBaseTable() {
+ List<Column> metadata = new ArrayList<>();
+ metadata.add(
+ MetadataColumn.of(
+ CommonOptions.EVENT_TIME.getName(),
+ BasicType.LONG_TYPE,
+ (Long) null,
+ true,
+ null,
+ null));
+ metadata.add(
+ MetadataColumn.of(
+ CommonOptions.DELAY.getName(),
+ BasicType.LONG_TYPE,
+ (Long) null,
+ true,
+ null,
+ null));
+ return CatalogTable.of(
+ TableIdentifier.of("catalog", TBL),
+ TableSchema.builder()
+ .column(
+ PhysicalColumn.of(
+ "id", BasicType.LONG_TYPE, (Long) null, false, null, null))
+ .column(
+ PhysicalColumn.of(
+ "name",
+ BasicType.STRING_TYPE,
+ (Long) null,
+ true,
+ null,
+ null))
+ .build(),
+ new HashMap<>(),
+ new ArrayList<>(),
+ "comment",
+ "test",
+ MetadataSchema.builder().columns(metadata).build());
+ }
+
+ private static AlterTableColumnsEvent buildAddTwoColsEvent(CatalogTable table) {
+ TableIdentifier id = table.getTableId();
+ return new AlterTableColumnsEvent(id)
+ .addEvent(
+ AlterTableAddColumnEvent.add(
+ id,
+ PhysicalColumn.of(
+ "discount_pct",
+ BasicType.DOUBLE_TYPE,
+ (Long) null,
+ true,
+ null,
+ null)))
+ .addEvent(
+ AlterTableAddColumnEvent.add(
+ id,
+ PhysicalColumn.of(
+ "is_featured",
+ BasicType.BOOLEAN_TYPE,
+ (Long) null,
+ true,
+ null,
+ null)));
+ }
+
+ /**
+ * Full production chain: FilterRowKind → Metadata → RowKindExtractor.
+ *
+ * <p>Pre-ALTER: 2 base cols → FilterRowKind passes through (arity 2) → Metadata adds 2 metadata
+ * cols (arity 4) → RowKindExtractor adds 1 (arity 5).
+ *
+ * <p>Post-ALTER: 4 base cols → FilterRowKind passes through (arity 4) → Metadata adds 2 (arity
+ * 6) → RowKindExtractor adds 1 (arity 7).
+ *
+ * <p>Bug 1: production shows post-ALTER input arity to SQL = pre-ALTER value, meaning Metadata
+ * silently truncated. Reproducing here proves the chain has the bug; assertions failing means
+ * we've nailed the layer.
+ */
+ @Test
+ void chainPreservesPostAlterColumnValues() {
+ CatalogTable baseTable = buildBaseTable();
+
+ // Build chain mirroring user's prod pipeline order.
+ Map<String, Object> filterCfg = new HashMap<>();
+ filterCfg.put("include_kinds", Arrays.asList("INSERT", "UPDATE_AFTER", "DELETE"));
+ FilterRowKindTransform filterRowKind =
+ new FilterRowKindTransform(ReadonlyConfig.fromMap(filterCfg), baseTable);
+
+ // FilterRowKind doesn't change schema; Metadata's input is FilterRowKind's output schema =
+ // baseTable.
+ Map<String, String> metaMapping = new LinkedHashMap<>();
+ metaMapping.put("EventTime", "c_event_time");
+ metaMapping.put("Delay", "c_delay");
+ Map<String, Object> metaCfg = new HashMap<>();
+ metaCfg.put("metadata_fields", metaMapping);
+ MetadataTransform metadata =
+ new MetadataTransform(ReadonlyConfig.fromMap(metaCfg), baseTable);
+ metadata.initRowContainerGenerator();
+
+ // RowKindExtractor's input is Metadata's output schema. Construct from Metadata's produced
+ // table so column-name conflict checks see the metadata cols already present.
+ CatalogTable metadataOutput = metadata.getProducedCatalogTable();
+ Map<String, Object> rkeCfg = new HashMap<>();
+ rkeCfg.put("custom_field_name", "c_operation_type");
+ rkeCfg.put("transform_type", "FULL");
+ RowKindExtractorTransform rowKindExtractor =
+ new RowKindExtractorTransform(ReadonlyConfig.fromMap(rkeCfg), metadataOutput);
+ rowKindExtractor.initRowContainerGenerator();
+
+ // ============ Pre-ALTER ============
+ SeaTunnelRow preRow = new SeaTunnelRow(new Object[] {1L, "Widget A"});
+ preRow.setTableId(TBL.getFullName());
+ MetadataUtil.setEventTime(preRow, 1700000000000L);
+ MetadataUtil.setDelay(preRow, 50L);
+
+ SeaTunnelRow afterFilter = filterRowKind.map(preRow);
+ Assertions.assertNotNull(afterFilter, "FilterRowKind should pass INSERT through");
+ Assertions.assertEquals(2, afterFilter.getArity(), "FilterRowKind doesn't change arity");
+
+ SeaTunnelRow afterMeta = metadata.map(afterFilter);
+ Assertions.assertEquals(
+ 4, afterMeta.getArity(), "Metadata: 2 base + 2 metadata cols pre-ALTER");
+ Assertions.assertEquals(1L, afterMeta.getField(0));
+ Assertions.assertEquals("Widget A", afterMeta.getField(1));
+
+ SeaTunnelRow afterRke = rowKindExtractor.map(afterMeta);
+ Assertions.assertEquals(
+ 5, afterRke.getArity(), "RowKindExtractor: 4 + 1 rowkind col pre-ALTER");
+
+ // ============ Live ALTER ADD COLUMN discount_pct, is_featured ============
+ SchemaChangeEvent alter = buildAddTwoColsEvent(baseTable);
+ // TransformFlowLifeCycle iterates in chain order, calling each transform's
+ // mapSchemaChangeEvent with the previous return value. Pass the same event through.
+ SchemaChangeEvent eventThroughChain = filterRowKind.mapSchemaChangeEvent(alter);
+ eventThroughChain = metadata.mapSchemaChangeEvent(eventThroughChain);
+ eventThroughChain = rowKindExtractor.mapSchemaChangeEvent(eventThroughChain);
+ Assertions.assertNotNull(eventThroughChain, "schema-change event should propagate");
+
+ // ============ Post-ALTER row (4 base cols: id, name, discount_pct, is_featured)
+ // ============
+ SeaTunnelRow postRow =
+ new SeaTunnelRow(new Object[] {2L, "Premium A", 10.00d, Boolean.TRUE});
+ postRow.setTableId(TBL.getFullName());
+ MetadataUtil.setEventTime(postRow, 1700000010000L);
+ MetadataUtil.setDelay(postRow, 60L);
+
+ SeaTunnelRow postFilter = filterRowKind.map(postRow);
+ Assertions.assertEquals(
+ 4,
+ postFilter.getArity(),
+ "FilterRowKind must pass through full post-ALTER arity (4 base cols)");
+ Assertions.assertEquals(10.00d, postFilter.getField(2));
+ Assertions.assertEquals(Boolean.TRUE, postFilter.getField(3));
+
+ SeaTunnelRow postMeta = metadata.map(postFilter);
+ // The smoking-gun assertion: must be 4 base + 2 metadata = 6, NOT 2 base + 2 metadata = 4.
+ Assertions.assertEquals(
+ 6,
+ postMeta.getArity(),
+ "Metadata MUST produce arity 6 post-ALTER (4 base + 2 metadata). "
+ + "If this fails with arity 4, that's Bug 1 — silent column drop in "
+ + "MultipleFieldOutputTransform's cached System.arraycopy.");
+ Assertions.assertEquals(2L, postMeta.getField(0));
+ Assertions.assertEquals("Premium A", postMeta.getField(1));
+ Assertions.assertEquals(
+ 10.00d,
+ postMeta.getField(2),
+ "discount_pct must survive Metadata transform after live ALTER");
+ Assertions.assertEquals(
+ Boolean.TRUE,
+ postMeta.getField(3),
+ "is_featured must survive Metadata transform after live ALTER");
+
+ SeaTunnelRow postRke = rowKindExtractor.map(postMeta);
+ Assertions.assertEquals(
+ 7,
+ postRke.getArity(),
+ "RowKindExtractor MUST produce arity 7 post-ALTER (4 base + 2 metadata + 1 "
+ + "rowkind). If arity is less, RowKindExtractor's cache is also stale.");
+ Assertions.assertEquals(10.00d, postRke.getField(2));
+ Assertions.assertEquals(Boolean.TRUE, postRke.getField(3));
+ }
+
+ /**
+ * Full production chain INCLUDING SQL transform: FilterRowKind → Metadata → RowKindExtractor →
+ * SQL → FilterFieldTransform.
+ *
+ * <p>SQLTransform.mapSchemaChangeEvent does NOT call transformTableSchema() — it only nullifies
+ * sqlEngine and outputCatalogTable. So outRowType (set inside transformTableSchema) stays at
+ * the pre-ALTER value forever post-ALTER. transformBySQL(inputRow, outRowType) constructs
+ * output rows shaped to outRowType. Stale outRowType ⇒ stale output shape ⇒ Bug 1.
+ */
+ @Test
+ void chainWithSqlAndFilterPreservesPostAlterColumnValues() {
+ CatalogTable baseTable = buildBaseTable();
+
+ Map<String, Object> filterCfg = new HashMap<>();
+ filterCfg.put("include_kinds", Arrays.asList("INSERT", "UPDATE_AFTER", "DELETE"));
+ FilterRowKindTransform filterRowKind =
+ new FilterRowKindTransform(ReadonlyConfig.fromMap(filterCfg), baseTable);
+
+ Map<String, String> metaMapping = new LinkedHashMap<>();
+ metaMapping.put("EventTime", "c_event_time");
+ metaMapping.put("Delay", "c_delay");
+ Map<String, Object> metaCfg = new HashMap<>();
+ metaCfg.put("metadata_fields", metaMapping);
+ MetadataTransform metadata =
+ new MetadataTransform(ReadonlyConfig.fromMap(metaCfg), baseTable);
+ metadata.initRowContainerGenerator();
+
+ CatalogTable metadataOutput = metadata.getProducedCatalogTable();
+ Map<String, Object> rkeCfg = new HashMap<>();
+ rkeCfg.put("custom_field_name", "c_operation_type");
+ rkeCfg.put("transform_type", "FULL");
+ RowKindExtractorTransform rowKindExtractor =
+ new RowKindExtractorTransform(ReadonlyConfig.fromMap(rkeCfg), metadataOutput);
+ rowKindExtractor.initRowContainerGenerator();
+
+ // SQL transform with user's actual production query shape: `select *, computed_a,
+ // computed_b from rkeOutput`. This adds projected columns whose positions are CACHED in
+ // SQLTransform.outRowType. SQLTransform.mapSchemaChangeEvent does NOT call
+ // transformTableSchema, so outRowType stays at pre-ALTER positions ⇒ Bug 1.
+ CatalogTable rkeOutput = rowKindExtractor.getProducedCatalogTable();
+ Map<String, Object> sqlCfg = new HashMap<>();
+ sqlCfg.put(
+ "query",
+ "select *, c_event_time AS c_event_time_copy, c_operation_type AS c_op_copy from "
+ + rkeOutput.getTableId().getTableName());
+ SQLTransform sqlTransform = new SQLTransform(ReadonlyConfig.fromMap(sqlCfg), rkeOutput);
+
+ // FilterFieldTransform: exclude one of the metadata cols (mimic user's exclude_fields)
+ CatalogTable sqlOutput = sqlTransform.getProducedCatalogTable();
+ Map<String, Object> filterFieldCfg = new HashMap<>();
+ filterFieldCfg.put("exclude_fields", Arrays.asList("c_delay"));
+ FilterFieldTransform filterField =
+ new FilterFieldTransform(ReadonlyConfig.fromMap(filterFieldCfg), sqlOutput);
+ // Trigger lazy init of inputValueIndexList by computing the produced catalog table.
+ filterField.getProducedCatalogTable();
+
+ // ============ Pre-ALTER ============
+ SeaTunnelRow preRow = new SeaTunnelRow(new Object[] {1L, "Widget A"});
+ preRow.setTableId(TBL.getFullName());
+ MetadataUtil.setEventTime(preRow, 1700000000000L);
+ MetadataUtil.setDelay(preRow, 50L);
+
+ SeaTunnelRow afterFilter = filterRowKind.map(preRow);
+ SeaTunnelRow afterMeta = metadata.map(afterFilter);
+ SeaTunnelRow afterRke = rowKindExtractor.map(afterMeta);
+ Assertions.assertEquals(5, afterRke.getArity(), "pre-ALTER through RowKindExtractor: 5");
+
+ List<SeaTunnelRow> sqlOut = sqlTransform.flatMap(afterRke);
+ Assertions.assertEquals(1, sqlOut.size());
+ SeaTunnelRow afterSql = sqlOut.get(0);
+ // 5 input + 2 computed = 7
+ Assertions.assertEquals(
+ 7,
+ afterSql.getArity(),
+ "pre-ALTER SQL select *, computed_a, computed_b: 5 + 2 = 7");
+
+ SeaTunnelRow afterFilterField = filterField.map(afterSql);
+ Assertions.assertEquals(
+ 6,
+ afterFilterField.getArity(),
+ "pre-ALTER FilterField excludes c_delay: 7 - 1 = 6");
+
+ // ============ Live ALTER ADD COLUMN discount_pct, is_featured ============
+ SchemaChangeEvent alter = buildAddTwoColsEvent(baseTable);
+ SchemaChangeEvent ev = filterRowKind.mapSchemaChangeEvent(alter);
+ ev = metadata.mapSchemaChangeEvent(ev);
+ ev = rowKindExtractor.mapSchemaChangeEvent(ev);
+ ev = sqlTransform.mapSchemaChangeEvent(ev);
+ ev = filterField.mapSchemaChangeEvent(ev);
+ Assertions.assertNotNull(ev);
+
+ // ============ Post-ALTER ============
+ SeaTunnelRow postRow =
+ new SeaTunnelRow(new Object[] {2L, "Premium A", 10.00d, Boolean.TRUE});
+ postRow.setTableId(TBL.getFullName());
+ MetadataUtil.setEventTime(postRow, 1700000010000L);
+ MetadataUtil.setDelay(postRow, 60L);
+
+ SeaTunnelRow postFilter = filterRowKind.map(postRow);
+ Assertions.assertEquals(4, postFilter.getArity());
+
+ SeaTunnelRow postMeta = metadata.map(postFilter);
+ Assertions.assertEquals(
+ 6, postMeta.getArity(), "Metadata post-ALTER: 4 base + 2 metadata = 6");
+ Assertions.assertEquals(10.00d, postMeta.getField(2));
+ Assertions.assertEquals(Boolean.TRUE, postMeta.getField(3));
+
+ SeaTunnelRow postRke = rowKindExtractor.map(postMeta);
+ Assertions.assertEquals(7, postRke.getArity(), "RowKindExtractor post-ALTER: 6 + 1 = 7");
+ Assertions.assertEquals(10.00d, postRke.getField(2));
+ Assertions.assertEquals(Boolean.TRUE, postRke.getField(3));
+
+ // ============ THE SMOKING GUN: SQL transform's transformBySQL ============
+ // Pre-ALTER: 5 input + 2 computed = 7. Post-ALTER expected: 7 input + 2 computed = 9.
+ List<SeaTunnelRow> postSqlOut = sqlTransform.flatMap(postRke);
+ Assertions.assertEquals(1, postSqlOut.size());
+ SeaTunnelRow postSql = postSqlOut.get(0);
+ Assertions.assertEquals(
+ 9,
+ postSql.getArity(),
+ "SQL post-ALTER MUST produce arity 9 (7 input + 2 computed). If 7, "
+ + "SQLTransform.outRowType is stale at pre-ALTER size and the engine "
+ + "projects with old positions ⇒ Bug 1 reproduced.");
+ Assertions.assertEquals(
+ 10.00d, postSql.getField(2), "discount_pct must survive SQL after live ALTER");
+ Assertions.assertEquals(
+ Boolean.TRUE, postSql.getField(3), "is_featured must survive SQL after live ALTER");
+ }
+}
diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLMultiCatalogSchemaChangeTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLMultiCatalogSchemaChangeTest.java
new file mode 100644
index 0000000000..d965059b35
--- /dev/null
+++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLMultiCatalogSchemaChangeTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.transform.sql;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.PhysicalColumn;
+import org.apache.seatunnel.api.table.catalog.TableIdentifier;
+import org.apache.seatunnel.api.table.catalog.TablePath;
+import org.apache.seatunnel.api.table.catalog.TableSchema;
+import org.apache.seatunnel.api.table.schema.event.AlterTableAddColumnEvent;
+import org.apache.seatunnel.api.table.schema.event.AlterTableColumnsEvent;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Wrapper-level reproduction for {@link SQLMultiCatalogFlatMapTransform}: confirms the fix in
+ * {@code AbstractMultiCatalogTransform.mapSchemaChangeEvent} dispatches schema-change events into
+ * the inner {@link SQLTransform}, so post-ALTER rows preserve their new column values through a
+ * {@code select *}-style query.
+ */
+public class SQLMultiCatalogSchemaChangeTest {
+
+ private static final TablePath TBL = TablePath.of("ricky_test", "static_inventory");
+
+ private static CatalogTable buildBaseTable() {
+ return CatalogTable.of(
+ TableIdentifier.of("catalog", TBL),
+ TableSchema.builder()
+ .column(
+ PhysicalColumn.of(
+ "id", BasicType.LONG_TYPE, (Long) null, false, null, null))
+ .column(
+ PhysicalColumn.of(
+ "name",
+ BasicType.STRING_TYPE,
+ (Long) null,
+ true,
+ null,
+ null))
+ .build(),
+ new HashMap<>(),
+ new ArrayList<>(),
+ "test");
+ }
+
+ @Test
+ void sqlMultiCatalogWrapperPropagatesSchemaChange() {
+ CatalogTable baseTable = buildBaseTable();
+
+ Map<String, Object> cfg = new HashMap<>();
+ cfg.put("query", "select * from " + TBL.getTableName());
+ ReadonlyConfig config = ReadonlyConfig.fromMap(cfg);
+
+ SQLMultiCatalogFlatMapTransform wrapper =
+ new SQLMultiCatalogFlatMapTransform(Collections.singletonList(baseTable), config);
+
+ // Pre-ALTER row through wrapper: 2-col select * → 2 cols out
+ SeaTunnelRow preRow = new SeaTunnelRow(new Object[] {1L, "Widget A"});
+ preRow.setTableId(TBL.getFullName());
+ List<SeaTunnelRow> preOut = wrapper.flatMap(preRow);
+ Assertions.assertEquals(1, preOut.size());
+ Assertions.assertEquals(2, preOut.get(0).getArity(), "pre-ALTER select *: 2 cols");
+
+ // Live ALTER ADD discount_pct, is_featured
+ TableIdentifier tid = baseTable.getTableId();
+ AlterTableColumnsEvent alter =
+ new AlterTableColumnsEvent(tid)
+ .addEvent(
+ AlterTableAddColumnEvent.add(
+ tid,
+ PhysicalColumn.of(
+ "discount_pct",
+ BasicType.DOUBLE_TYPE,
+ (Long) null,
+ true,
+ null,
+ null)))
+ .addEvent(
+ AlterTableAddColumnEvent.add(
+ tid,
+ PhysicalColumn.of(
+ "is_featured",
+ BasicType.BOOLEAN_TYPE,
+ (Long) null,
+ true,
+ null,
+ null)));
+
+ wrapper.mapSchemaChangeEvent(alter);
+
+ // Post-ALTER row through wrapper: 4-col select * → 4 cols out (all values preserved)
+ SeaTunnelRow postRow =
+ new SeaTunnelRow(new Object[] {2L, "Premium A", 10.00d, Boolean.TRUE});
+ postRow.setTableId(TBL.getFullName());
+ List<SeaTunnelRow> postOut = wrapper.flatMap(postRow);
+
+ Assertions.assertEquals(1, postOut.size());
+ SeaTunnelRow postSql = postOut.get(0);
+ Assertions.assertEquals(
+ 4,
+ postSql.getArity(),
+ "post-ALTER select * MUST be 4 cols (id, name, discount_pct, is_featured). "
+ + "If 2, the wrapper swallowed ALTER and inner SQLTransform's sqlEngine "
+ + "still has the old 2-col schema.");
+ Assertions.assertEquals(2L, postSql.getField(0));
+ Assertions.assertEquals("Premium A", postSql.getField(1));
+ Assertions.assertEquals(10.00d, postSql.getField(2), "discount_pct preserved");
+ Assertions.assertEquals(Boolean.TRUE, postSql.getField(3), "is_featured preserved");
+ }
+}