You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ni...@apache.org on 2026/08/04 04:34:47 UTC
(seatunnel) branch codex/docs-localfile-csv-header-9948 updated (7e2007c70d -> 685dc0f173)
This is an automated email from the ASF dual-hosted git repository.
nielifeng pushed a change to branch codex/docs-localfile-csv-header-9948
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
from 7e2007c70d [Docs][Connector-V2] Clarify LocalFile CSV header semantics
add c299232132 [Docs][CLI] Add "Running the Benchmark Yourself" section to the AI CLI benchmark pages (#11605)
add 82cdefb550 docs: add Windows local deployment guidance (#11603)
add 4c7f41373b [Fix][Connector-V2] Prevent TiDB CDC from advancing past slow regions (#11407)
add 91e3da1180 [Fix][Core] Exclude Scala classes from Spark starter (#11393)
add ac65b8f13e [Fix][Connector-V2] Fix Paimon stream enumerator checkpoint race (#11392)
add e907b63ca5 [Fix][Connector-V2][JDBC] Fix duplicate column error when fetching SQL Server metadata with multiple extended properties (#11374)
add 7fdab176bd [Docs][Connector-V2] Improve S3File SftpFile and StarRocks docs (#11326)
add 1610141868 [CI][Core] Exclude build outputs from import class check (#11621)
add 458cefdc51 [Test][API] Avoid ImportClassCheckTest OOM on generated sources (#11623)
add 01c0b21e99 [Docs][Jdbc] Clarify MariaDB driver configuration (#11604)
add bc2708af44 [Fix][Connector-V2] fix fluss-connector‘s sink writer issues (#11116)
add 2b51d36140 [Feature][Zeta] Support restoring jobs from the latest completed checkpoint (#11421)
add 81d69d83bc [CI][E2E] Stabilize hosted connector IT failures (#11624)
add ac8306f332 [Fix][Connector-V2][File] Close the GZ input stream in AbstractReadStrategy (#10532)
add 6bf786bee7 Fix schema change routing for shared multi-table sinks (#11015)
add 685dc0f173 Merge branch 'dev' into codex/docs-localfile-csv-header-9948
No new revisions were added by this update.
Summary of changes:
.github/workflows/backend.yml | 3 +
docs/en/ai-cli/benchmark.md | 35 ++
docs/en/connectors/sink/Jdbc.md | 13 +
docs/en/connectors/sink/S3File.md | 13 +-
docs/en/connectors/sink/SftpFile.md | 18 +-
docs/en/connectors/source/Jdbc.md | 13 +
docs/en/connectors/source/StarRocks.md | 51 +-
docs/en/engines/zeta/rest-api-job-lifecycle.md | 65 +-
docs/en/engines/zeta/state-storage-and-recovery.md | 14 +-
docs/en/engines/zeta/user-command.md | 3 +-
docs/en/getting-started/locally/deployment.md | 24 +
.../locally/quick-start-seatunnel-engine.md | 13 +
.../introduction/configuration/schema-evolution.md | 96 +++
docs/zh/ai-cli/benchmark.md | 32 +
docs/zh/connectors/sink/Jdbc.md | 13 +
docs/zh/connectors/sink/S3File.md | 17 +-
docs/zh/connectors/sink/SftpFile.md | 19 +-
docs/zh/connectors/sink/StarRocks.md | 48 +-
docs/zh/connectors/source/Jdbc.md | 13 +
docs/zh/connectors/source/StarRocks.md | 53 +-
docs/zh/engines/zeta/rest-api-job-lifecycle.md | 57 +-
docs/zh/engines/zeta/state-storage-and-recovery.md | 13 +-
docs/zh/engines/zeta/user-command.md | 3 +-
.../introduction/configuration/schema-evolution.md | 85 +++
.../seatunnel/api/options/EnvCommonOptions.java | 8 +
.../seatunnel/api/options/EnvOptionRule.java | 1 +
.../api/sink/SupportSchemaEvolutionSinkWriter.java | 16 +
.../sink/multitablesink/MultiTableSinkWriter.java | 385 +++++++++---
.../multitablesink/MultiTableWriterRunnable.java | 234 +++++--
.../sink/multitablesink/SchemaChangeBarrier.java | 128 ++++
.../multitablesink/SchemaChangeDispatchTarget.java | 54 ++
...tiTableSinkWriterSchemaChangeBroadcastTest.java | 698 +++++++++++++++++++++
.../multitablesink/MultiTableSinkWriterTest.java | 25 +-
.../apache/seatunnel/api/ImportClassCheckTest.java | 87 ++-
.../base/utils/MessageDelayedEventLimiterTest.java | 3 +-
.../mysql/source/MySqlSchemaChangeResolver.java | 3 +-
.../source/parser/CustomMySqlAntlrDdlParser.java | 23 +-
.../parser/CustomMySqlAntlrDdlParserTest.java | 72 +++
.../cdc/mysql/testutils/MySqlContainer.java | 20 +-
.../cdc/tidb/source/reader/TiDBSourceReader.java | 3 +-
.../tidb/source/reader/TiDBSourceReaderTest.java | 41 ++
.../file/source/reader/AbstractReadStrategy.java | 31 +-
.../reader/AbstractReadStrategyGzCloseTest.java | 180 ++++++
.../seatunnel/fluss/sink/FlussSinkWriter.java | 83 ++-
.../seatunnel/fluss/sink/FlussSinkWriterTest.java | 188 ++++++
.../jdbc/catalog/sqlserver/SqlServerCatalog.java | 4 +-
.../jdbc/sink/AbstractJdbcSinkWriter.java | 10 +
...kWriterSharedPhysicalTableSchemaChangeTest.java | 247 ++++++++
.../seatunnel/paimon/source/PaimonSourceState.java | 5 +-
.../source/enumerator/AbstractSplitEnumerator.java | 80 +--
.../PaimonStreamSourceSplitEnumerator.java | 10 +-
.../PaimonStreamSourceSplitEnumeratorTest.java | 94 +++
.../starrocks/sink/StarRocksSinkWriter.java | 9 +
seatunnel-core/seatunnel-spark-starter/pom.xml | 14 +
.../spark/TestSparkStarterJarContentIT.java | 140 +++++
.../spark/TestSparkStarterJarContentIT.java | 141 +++++
.../starter/seatunnel/args/ClientCommandArgs.java | 41 ++
.../seatunnel/command/ClientExecuteCommand.java | 12 +
.../seatunnel/args/ClientCommandArgsTest.java | 91 +++
...eteIT.java => MysqlCDCCheckpointRestoreIT.java} | 349 ++++++-----
.../cdc/mysql/MysqlCDCWithSchemaChangeIT.java | 291 ++++++++-
.../ddl/multi_db_same_name_add_columns.sql | 48 ++
.../ddl/multi_db_same_name_schema_change.sql | 78 +++
...mysqlcdc_to_mysql_with_checkpoint_restore.conf} | 13 +-
...sql_with_multi_db_same_name_schema_change.conf} | 24 +-
.../e2e/connector/databend/DatabendCDCSinkIT.java | 7 +
.../common/container/AbstractTestContainer.java | 51 +-
.../e2e/common/container/TestContainer.java | 20 +
.../container/seatunnel/SeaTunnelContainer.java | 119 +++-
.../CheckpointableSequenceSource.java | 139 ++++
.../CheckpointableSequenceSourceFactory.java | 38 +-
.../CheckpointableSequenceSourceReader.java | 97 +++
.../CheckpointableSequenceSplit.java | 42 +-
.../CheckpointableSequenceSplitEnumerator.java | 100 +++
.../CheckpointableSequenceState.java} | 24 +-
.../engine/e2e/CheckpointRestoreWithStopIT.java | 281 +++++++++
.../seatunnel/engine/e2e/SavepointRestoreIT.java | 279 ++++++++
.../plugin-mapping.properties | 10 +-
...tream_checkpointable_sequence_to_localfile.conf | 29 +-
...tream_checkpointable_sequence_to_localfile.conf | 29 +-
.../seatunnel/engine/client/SeaTunnelClient.java | 33 +-
.../engine/client/SeaTunnelClientInstance.java | 8 +
.../client/job/ClientJobExecutionEnvironment.java | 48 +-
.../seatunnel/engine/client/job/JobClient.java | 8 +-
.../config/YamlSeaTunnelDomConfigProcessor.java | 5 +
.../common/config/server/CheckpointConfig.java | 8 +
.../common/config/server/ServerConfigOptions.java | 9 +
.../engine/core/job/AbstractJobEnvironment.java | 11 +-
.../engine/core/job/JobImmutableInformation.java | 81 ++-
.../seatunnel/engine/core/job/RestoreMode.java | 66 ++
.../codec/SeaTunnelGetJobCheckpointCodec.java | 40 +-
.../JobImmutableInformationCompatibilityTest.java | 156 +++++
.../seatunnel/engine/core/job/RestoreModeTest.java | 46 ++
.../codec/SeaTunnelGetJobCheckpointCodecTest.java | 65 ++
.../engine/server/CheckpointRestoreValidator.java | 62 ++
.../seatunnel/engine/server/CheckpointService.java | 81 ++-
.../engine/server/CoordinatorService.java | 61 +-
.../server/checkpoint/CheckpointManager.java | 63 +-
.../server/dag/physical/PhysicalPlanGenerator.java | 2 +-
.../seatunnel/engine/server/master/JobMaster.java | 14 +-
.../operation/GetJobCheckpointOperation.java | 29 +-
.../server/protocol/task/GetJobCheckpointTask.java | 8 +-
.../seatunnel/engine/server/rest/RestConstant.java | 4 +
.../server/rest/RestJobExecutionEnvironment.java | 95 ++-
.../engine/server/rest/service/BaseService.java | 36 +-
.../engine/server/rest/service/JobInfoService.java | 15 +
.../server/utils/CheckpointRestoreUtils.java | 45 ++
.../server/CheckpointRestoreValidatorTest.java | 76 +++
.../engine/server/CheckpointServiceTest.java | 172 +++++
.../server/CoordinatorServiceJobCleanupTest.java | 65 ++
.../checkpoint/CheckpointCoordinatorTest.java | 11 +
.../server/checkpoint/CheckpointManagerTest.java | 136 +++-
.../engine/server/master/JobMasterTest.java | 40 ++
.../RestApiSubmitJobStartWithSavePointTest.java | 103 ++-
.../rest/service/JobInfoServiceNullSafetyTest.java | 80 +++
115 files changed, 6959 insertions(+), 786 deletions(-)
create mode 100644 seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/SchemaChangeBarrier.java
create mode 100644 seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/SchemaChangeDispatchTarget.java
create mode 100644 seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterSchemaChangeBroadcastTest.java
create mode 100644 seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/parser/CustomMySqlAntlrDdlParserTest.java
create mode 100644 seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategyGzCloseTest.java
create mode 100644 seatunnel-connectors-v2/connector-fluss/src/test/java/org/apache/seatunnel/connectors/seatunnel/fluss/sink/FlussSinkWriterTest.java
create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkWriterSharedPhysicalTableSchemaChangeTest.java
create mode 100644 seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/test/java/org/apache/seatunnel/core/starter/spark/TestSparkStarterJarContentIT.java
create mode 100644 seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/TestSparkStarterJarContentIT.java
copy seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/{MysqlCDCWithBinlogDeleteIT.java => MysqlCDCCheckpointRestoreIT.java} (52%)
create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/multi_db_same_name_add_columns.sql
create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/ddl/multi_db_same_name_schema_change.sql
copy seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/{mysqlcdc_to_mysql_with_no_primary_key.conf => mysqlcdc_to_mysql_with_checkpoint_restore.conf} (84%)
copy seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/{mysqlcdc_to_mysql_with_schema_change.conf => mysqlcdc_to_mysql_with_multi_db_same_name_schema_change.conf} (68%)
create mode 100644 seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/source/checkpointable/CheckpointableSequenceSource.java
copy seatunnel-connectors-v2/connector-tablestore/src/main/java/org/apache/seatunnel/connectors/seatunnel/tablestore/source/TableStoreSourceFactory.java => seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/source/checkpointable/CheckpointableSequenceSourceFactory.java (59%)
create mode 100644 seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/source/checkpointable/CheckpointableSequenceSourceReader.java
copy seatunnel-connectors-v2/connector-iotdb-v2/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdbv2/source/IoTDBv2SourceSplit.java => seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/source/checkpointable/CheckpointableSequenceSplit.java (54%)
create mode 100644 seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/source/checkpointable/CheckpointableSequenceSplitEnumerator.java
copy seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/{sink/inmemory/InMemoryMultiTableResourceManager.java => source/checkpointable/CheckpointableSequenceState.java} (62%)
create mode 100644 seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/CheckpointRestoreWithStopIT.java
create mode 100644 seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SavepointRestoreIT.java
copy seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test_write_hive.conf => seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/checkpoint-restore-with-stop/plugin-mapping.properties (79%)
copy tools/upgrade_compatibility/scenarios/generic-fake-localfile/job.conf => seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/checkpoint-restore-with-stop/stream_checkpointable_sequence_to_localfile.conf (73%)
copy tools/upgrade_compatibility/scenarios/generic-fake-localfile/job.conf => seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/savepoint-restore/stream_checkpointable_sequence_to_localfile.conf (73%)
create mode 100644 seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/RestoreMode.java
create mode 100644 seatunnel-engine/seatunnel-engine-core/src/test/java/org/apache/seatunnel/engine/core/job/JobImmutableInformationCompatibilityTest.java
create mode 100644 seatunnel-engine/seatunnel-engine-core/src/test/java/org/apache/seatunnel/engine/core/job/RestoreModeTest.java
create mode 100644 seatunnel-engine/seatunnel-engine-core/src/test/java/org/apache/seatunnel/engine/core/protocol/codec/SeaTunnelGetJobCheckpointCodecTest.java
create mode 100644 seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/CheckpointRestoreValidator.java
create mode 100644 seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/utils/CheckpointRestoreUtils.java
create mode 100644 seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CheckpointRestoreValidatorTest.java
create mode 100644 seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/CheckpointServiceTest.java