You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by zo...@apache.org on 2023/03/04 07:31:50 UTC
[incubator-seatunnel] 01/01: [Feature] add cdc multiple table support & fix zeta bug
This is an automated email from the ASF dual-hosted git repository.
zongwen pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
commit 533ff2c2fa9262defb86f37235ee10bf16135595
Merge: 36a8774d9 973a2fae3
Author: Zongwen Li <zo...@apache.org>
AuthorDate: Sat Mar 4 15:31:41 2023 +0800
[Feature] add cdc multiple table support & fix zeta bug
Merge pull request #4256 from apache/cdc-multiple-table
config/hazelcast.yaml | 1 +
config/jvm_options | 5 +-
config/log4j2_client.properties | 48 +--
config/seatunnel.yaml | 2 +-
docs/en/connector-v2/sink/HdfsFile.md | 10 +-
docs/en/connector-v2/sink/Jdbc.md | 10 +-
docs/en/connector-v2/sink/OssFile.md | 12 +-
docs/en/connector-v2/sink/OssJindoFile.md | 12 +-
docs/en/connector-v2/sink/S3-Redshift.md | 12 +-
docs/en/connector-v2/sink/S3File.md | 12 +-
docs/en/connector-v2/sink/SftpFile.md | 8 +-
docs/en/connector-v2/sink/StarRocks.md | 37 +-
docs/en/connector-v2/source/FtpFile.md | 8 +-
docs/en/connector-v2/source/HdfsFile.md | 8 +-
docs/en/connector-v2/source/Iceberg.md | 13 +-
docs/en/connector-v2/source/LocalFile.md | 10 +-
docs/en/connector-v2/source/MySQL-CDC.md | 30 +-
docs/en/connector-v2/source/OssFile.md | 10 +-
docs/en/connector-v2/source/OssJindoFile.md | 10 +-
docs/en/connector-v2/source/S3File.md | 10 +-
docs/en/connector-v2/source/SftpFile.md | 8 +-
docs/en/seatunnel-engine/checkpoint-storage.md | 16 +-
pom.xml | 5 +-
release-note.md | 1 +
.../CommonOptions.java} | 24 +-
.../api/common/SeaTunnelAPIErrorCode.java | 2 +
.../api/configuration/util/ConfigUtil.java | 15 +-
.../util/OptionValidationException.java | 9 +
.../apache/seatunnel/api/env/EnvCommonOptions.java | 20 +-
.../apache/seatunnel/api/env/EnvOptionRule.java | 3 +-
.../org/apache/seatunnel/api/env/ParsingMode.java | 58 +++
.../seatunnel/api/sink/SaveModeConstants.java | 15 +-
.../seatunnel/api/sink/SinkCommonOptions.java | 20 -
.../seatunnel/api/source/SourceCommonOptions.java | 46 --
.../seatunnel/api/table/catalog/Catalog.java | 39 ++
.../api/table/catalog/CatalogOptions.java | 66 +++
.../seatunnel/api/table/catalog/CatalogTable.java | 17 +
.../api/table/catalog/CatalogTableUtil.java | 335 +++++++++++++++
.../apache/seatunnel/api/table/catalog/Column.java | 177 ++------
.../seatunnel/api/table/catalog/ConstraintKey.java | 77 ++++
.../table/catalog/DataTypeConvertException.java | 61 +++
.../api/table/catalog/DataTypeConvertor.java | 62 +++
.../api/table/catalog/MetadataColumn.java | 67 +++
.../api/table/catalog/PhysicalColumn.java | 60 +++
.../seatunnel/api/table/catalog/PrimaryKey.java | 30 +-
.../api/table/catalog/TableIdentifier.java | 2 +-
.../seatunnel/api/table/catalog/TablePath.java | 4 +
.../seatunnel/api/table/catalog/TableSchema.java | 80 +---
.../exception/DatabaseAlreadyExistException.java | 37 ++
.../exception/TableAlreadyExistException.java} | 22 +-
.../seatunnel/api/table/connector/TableSource.java | 1 +
.../table/factory/DataTypeConvertorFactory.java | 54 +++
.../seatunnel/api/table/factory/FactoryUtil.java | 145 +++++--
.../api/table/factory/TableFactoryContext.java | 5 +
.../api/table/factory/TableSourceFactory.java | 4 +
.../seatunnel/api/table/type/MultipleRowType.java | 63 +++
.../seatunnel/api/table/type/SeaTunnelRow.java | 6 +-
.../apache/seatunnel/api/table/type/SqlType.java | 3 +-
.../api/configuration/util/ConfigUtilTest.java | 49 +++
.../api/table/catalog/CatalogTableUtilTest.java | 26 +-
.../src/test/resources/conf}/complex.schema.conf | 0
.../src/test/resources/conf}/simple.schema.conf | 0
.../org/apache/seatunnel/common/config/Common.java | 7 +-
.../apache/seatunnel/common/utils/FileUtils.java | 4 +-
.../apache/seatunnel/common/utils/JdbcUrlUtil.java | 93 ++++
.../apache/seatunnel/common/utils/RetryUtils.java | 9 +-
.../seatunnel/common/utils/JdbcUrlUtilTest.java | 56 +++
seatunnel-connectors-v2/README.md | 2 +-
seatunnel-connectors-v2/README.zh.md | 2 +-
.../config/AmazonDynamoDBSourceOptions.java | 6 +-
.../source/AmazonDynamoDBSource.java | 8 +-
.../source/AmazonDynamoDBSourceFactory.java | 4 +-
.../cdc/base/config/JdbcSourceConfig.java | 7 +
.../cdc/base/config/JdbcSourceConfigFactory.java | 17 +-
.../cdc/base/option/JdbcSourceOptions.java | 14 +-
.../connection/JdbcConnectionPoolFactory.java | 2 +-
.../relational/connection/JdbcConnectionPools.java | 2 +-
.../cdc/base/source/IncrementalSource.java | 20 +-
.../enumerator/IncrementalSourceEnumerator.java | 4 +-
.../external/IncrementalSourceScanFetcher.java | 4 +-
.../row/SeaTunnelRowDebeziumDeserializeSchema.java | 69 ++-
.../cdc/mysql/config/MySqlSourceConfig.java | 2 +
.../cdc/mysql/config/MySqlSourceConfigFactory.java | 1 +
.../seatunnel/cdc/mysql/source/MySqlDialect.java | 16 +-
.../cdc/mysql/source/MySqlIncrementalSource.java | 48 ++-
.../source/MySqlIncrementalSourceFactory.java | 49 ++-
.../reader/fetch/MySqlSourceFetchTaskContext.java | 39 +-
.../reader/fetch/scan/MySqlSnapshotFetchTask.java | 9 +-
.../fetch/scan/MySqlSnapshotSplitReadTask.java | 15 +-
.../seatunnel/cdc/mysql/utils/MySqlUtils.java | 9 +-
.../cdc/mysql/utils/TableDiscoveryUtils.java | 4 +-
.../source/config/SqlServerSourceConfig.java | 2 +
.../config/SqlServerSourceConfigFactory.java | 1 +
.../sqlserver/source/source/SqlServerDialect.java | 17 +-
.../source/SqlServerIncrementalSourceFactory.java | 5 +-
.../fetch/SqlServerSourceFetchTaskContext.java | 39 +-
.../source/utils/TableDiscoveryUtils.java | 2 +-
.../seatunnel/common/schema/SeaTunnelSchema.java | 287 -------------
.../common/source/reader/SourceReaderBase.java | 12 +-
.../seatunnel/console/sink/ConsoleSink.java | 12 +-
.../seatunnel/console/sink/ConsoleSinkFactory.java | 8 +
.../seatunnel/console/sink/ConsoleSinkWriter.java | 4 +-
.../catalog/ElasticSearchCatalog.java | 221 ++++++++++
.../catalog/ElasticSearchCatalogFactory.java | 37 +-
.../catalog/ElasticSearchDataTypeConvertor.java | 123 ++++++
.../elasticsearch/client/EsRestClient.java | 132 +++++-
.../exception/ElasticsearchConnectorErrorCode.java | 6 +-
.../elasticsearch/source/ElasticsearchSource.java | 14 +-
.../source/ElasticsearchSourceFactory.java | 4 +-
.../seatunnel/fake/source/FakeDataGenerator.java | 14 +-
.../seatunnel/fake/source/FakeSource.java | 14 +-
.../seatunnel/fake/source/FakeSourceFactory.java | 4 +-
.../seatunnel/fake/source/FakeSourceReader.java | 15 +-
.../fake/source/FakeSourceSplitEnumerator.java | 31 +-
.../fake/source/FakeDataGeneratorTest.java | 15 +-
.../file/hdfs/source/BaseHdfsFileSource.java | 16 +-
.../seatunnel/file/config/BaseFileSinkConfig.java | 6 +-
.../seatunnel/file/config/BaseSinkConfig.java | 6 +-
.../seatunnel/file/config/BaseSourceConfig.java | 7 +-
.../file/source/BaseFileSourceReader.java | 47 ++-
.../file/source/reader/TextReadStrategy.java | 8 +-
.../file/ftp/sink/FtpFileSinkFactory.java | 18 +-
.../seatunnel/file/ftp/source/FtpFileSource.java | 15 +-
.../file/ftp/source/FtpFileSourceFactory.java | 12 +-
.../file/hdfs/sink/HdfsFileSinkFactory.java | 18 +-
.../file/hdfs/source/HdfsFileSourceFactory.java | 12 +-
.../file/local/sink/LocalFileSinkFactory.java | 18 +-
.../file/local/source/LocalFileSource.java | 16 +-
.../file/local/source/LocalFileSourceFactory.java | 12 +-
.../file/oss/sink/OssFileSinkFactory.java | 18 +-
.../seatunnel/file/oss/source/OssFileSource.java | 15 +-
.../file/oss/source/OssFileSourceFactory.java | 12 +-
.../file/oss/sink/OssFileSinkFactory.java | 18 +-
.../seatunnel/file/oss/source/OssFileSource.java | 15 +-
.../file/oss/source/OssFileSourceFactory.java | 12 +-
.../connector-file/connector-file-s3/pom.xml | 16 +
.../seatunnel/file/s3/catalog/S3Catalog.java | 200 +++++++++
.../file/s3/catalog/S3CatalogFactory.java | 48 +++
.../file/s3/catalog/S3DataTypeConvertor.java | 63 +++
.../seatunnel/file/s3/sink/S3FileSinkFactory.java | 18 +-
.../seatunnel/file/s3/source/S3FileSource.java | 15 +-
.../file/s3/source/S3FileSourceFactory.java | 12 +-
.../file/sftp/sink/SftpFileSinkFactory.java | 18 +-
.../seatunnel/file/sftp/source/SftpFileSource.java | 14 +-
.../file/sftp/source/SftpFileSourceFactory.java | 12 +-
.../google/sheets/source/SheetsSource.java | 12 +-
.../google/sheets/source/SheetsSourceFactory.java | 4 +-
.../connectors/seatunnel/hive/sink/HiveSink.java | 12 +-
.../seatunnel/hive/source/HiveSource.java | 14 +-
.../seatunnel/http/source/HttpSource.java | 9 +-
.../seatunnel/http/source/HttpSourceFactory.java | 4 +-
.../seatunnel/iceberg/source/IcebergSource.java | 15 +-
.../iceberg/source/IcebergSourceFactory.java | 4 +-
.../seatunnel/influxdb/source/InfluxDBSource.java | 7 +-
.../influxdb/source/InfluxDBSourceFactory.java | 4 +-
.../seatunnel/iotdb/source/IoTDBSource.java | 7 +-
.../seatunnel/iotdb/source/IoTDBSourceFactory.java | 2 +-
.../jdbc/catalog/AbstractJdbcCatalog.java | 212 ++++++----
.../seatunnel/jdbc/catalog/JdbcCatalogOptions.java | 25 +-
.../seatunnel/jdbc/catalog/MySqlCatalog.java | 251 -----------
.../seatunnel/jdbc/catalog/mysql/MySqlCatalog.java | 275 ++++++++++++
.../jdbc/catalog/mysql/MySqlCatalogFactory.java | 60 +++
.../catalog/mysql/MysqlCreateTableSqlBuilder.java | 222 ++++++++++
.../jdbc/catalog/mysql/MysqlDataTypeConvertor.java | 205 +++++++++
.../redshift/RedshiftDataTypeConvertor.java | 206 +++++++++
.../jdbc/config/JdbcConnectionConfig.java | 220 ++++++++++
.../config/{JdbcConfig.java => JdbcOptions.java} | 124 ++----
.../seatunnel/jdbc/config/JdbcSinkConfig.java | 61 +++
.../seatunnel/jdbc/config/JdbcSinkOptions.java | 65 ---
...dbcSourceOptions.java => JdbcSourceConfig.java} | 52 +--
.../seatunnel/jdbc/internal/JdbcOutputFormat.java | 24 +-
.../jdbc/internal/JdbcOutputFormatBuilder.java | 73 +++-
.../jdbc/internal/connection/DataSourceUtils.java | 20 +-
.../connection/SimpleJdbcConnectionProvider.java | 26 +-
.../jdbc/internal/dialect/JdbcDialect.java | 31 +-
.../jdbc/internal/dialect/db2/DB2Dialect.java | 2 +-
.../jdbc/internal/dialect/dm/DmdbDialect.java | 2 +-
.../internal/dialect/gbase8a/Gbase8aDialect.java | 2 +-
.../jdbc/internal/dialect/mysql/MysqlDialect.java | 4 +-
.../internal/dialect/oracle/OracleDialect.java | 5 +-
.../internal/dialect/phoenix/PhoenixDialect.java | 2 +-
.../internal/dialect/psql/PostgresDialect.java | 6 +-
.../internal/dialect/redshift/RedshiftDialect.java | 2 +-
.../internal/dialect/saphana/SapHanaDialect.java | 2 +-
.../internal/dialect/sqlite/SqliteDialect.java | 4 +-
.../dialect/sqlserver/SqlServerDialect.java | 5 +-
.../dialect/tablestore/TablestoreDialect.java | 8 +-
.../internal/dialect/teradata/TeradataDialect.java | 2 +-
.../internal/options/JdbcConnectionOptions.java | 205 ---------
.../seatunnel/jdbc/internal/xa/XaFacade.java | 6 +-
.../jdbc/internal/xa/XaFacadeImplAutoLoad.java | 14 +-
.../jdbc/sink/JdbcExactlyOnceSinkWriter.java | 12 +-
.../connectors/seatunnel/jdbc/sink/JdbcSink.java | 28 +-
.../jdbc/sink/JdbcSinkAggregatedCommitter.java | 14 +-
.../seatunnel/jdbc/sink/JdbcSinkCommitter.java | 14 +-
.../seatunnel/jdbc/sink/JdbcSinkFactory.java | 42 +-
.../seatunnel/jdbc/sink/JdbcSinkWriter.java | 8 +-
.../seatunnel/jdbc/source/JdbcSource.java | 48 ++-
.../seatunnel/jdbc/source/JdbcSourceFactory.java | 22 +-
.../seatunnel/jdbc/source/JdbcSourceReader.java | 17 +-
.../jdbc/source/JdbcSourceSplitEnumerator.java | 12 +-
.../jdbc/catalog/AbstractJdbcCatalogTest.java | 66 ---
.../jdbc/catalog/MysqlDataTypeConvertorTest.java | 56 +++
.../sql/MysqlCreateTableSqlBuilderTest.java | 106 +++++
.../seatunnel/kafka/catalog/KafkaCatalog.java | 265 ++++++++++++
.../kafka/catalog/KafkaCatalogFactory.java} | 22 +-
.../kafka/catalog/KafkaDataTypeConvertor.java | 72 ++++
.../connectors/seatunnel/kafka/config/Config.java | 3 +
.../connectors/seatunnel/kafka/sink/KafkaSink.java | 2 +-
.../seatunnel/kafka/source/KafkaSource.java | 9 +-
.../kafka/catalog/KafkaDataTypeConvertorTest.java | 46 ++
.../seatunnel/mongodb/sink/MongodbSink.java | 4 +-
.../seatunnel/mongodb/source/MongodbSource.java | 11 +-
.../mongodb/source/MongodbSourceFactory.java | 4 +-
.../seatunnel/neo4j/source/Neo4jSource.java | 9 +-
.../seatunnel/neo4j/source/Neo4jSourceFactory.java | 4 +-
.../seatunnel/pulsar/source/PulsarSource.java | 6 +-
.../pulsar/source/PulsarSourceFactory.java | 4 +-
.../seatunnel/rabbitmq/source/RabbitmqSource.java | 8 +-
.../rabbitmq/source/RabbitmqSourceFactory.java | 2 +-
.../seatunnel/redis/source/RedisSource.java | 9 +-
.../seatunnel/redis/source/RedisSourceFactory.java | 4 +-
.../seatunnel/redshift/sink/S3RedshiftFactory.java | 2 +-
.../connector-starrocks/pom.xml | 7 +
.../starrocks/catalog/StarRocksCatalog.java | 434 +++++++++++++++++++
.../catalog/StarRocksCatalogFactory.java} | 38 +-
.../catalog/StarRocksDataTypeConvertor.java | 204 +++++++++
.../starrocks/client/StarRocksSinkManager.java | 7 +-
.../client/StarRocksStreamLoadVisitor.java | 25 +-
.../seatunnel/starrocks/config/SinkConfig.java | 213 ++--------
.../starrocks/config/StarRocksOptions.java} | 22 +-
.../starrocks/config/StarRocksSinkOptions.java | 136 ++++++
.../starrocks/sink/StarRocksSaveModeUtil.java | 151 +++++++
.../seatunnel/starrocks/sink/StarRocksSink.java | 102 +++--
.../starrocks/sink/StarRocksSinkFactory.java | 47 ++-
.../starrocks/sink/StarRocksSinkWriter.java | 5 +-
.../starrocks/source/StarRocksSource.java | 8 +-
.../starrocks/source/StarRocksSourceFactory.java | 2 +-
.../starrocks/util/CreateTableParser.java | 85 ++++
.../seatunnel/starrocks/StarRocksCatalogTest.java | 84 ++++
.../tablestore/sink/TablestoreSinkFactory.java | 4 +-
.../flink/execution/SinkExecuteProcessor.java | 6 +-
.../flink/execution/SourceExecuteProcessor.java | 6 +-
.../flink/execution/SinkExecuteProcessor.java | 6 +-
.../flink/execution/SourceExecuteProcessor.java | 6 +-
.../spark/execution/SinkExecuteProcessor.java | 13 +-
.../spark/execution/SourceExecuteProcessor.java | 13 +-
.../spark/execution/SinkExecuteProcessor.java | 13 +-
.../spark/execution/SourceExecuteProcessor.java | 13 +-
.../seatunnel-starter/src/main/bin/seatunnel.sh | 2 +-
.../seatunnel/command/ClientExecuteCommand.java | 31 +-
seatunnel-dist/pom.xml | 6 +
seatunnel-dist/release-docs/LICENSE | 11 +-
.../resources/json/fake_to_local_file_json.conf | 2 +-
.../resources/json/local_file_json_to_assert.conf | 2 +-
.../test/resources/orc/fake_to_local_file_orc.conf | 2 +-
.../orc/local_file_orc_projection_to_assert.conf | 2 +-
.../resources/orc/local_file_orc_to_assert.conf | 4 +-
.../parquet/fake_to_local_file_parquet.conf | 2 +-
.../local_file_parquet_projection_to_assert.conf | 2 +-
.../parquet/local_file_parquet_to_assert.conf | 2 +-
.../resources/text/fake_to_local_file_text.conf | 2 +-
.../text/local_file_text_projection_to_assert.conf | 2 +-
.../text/local_file_text_skip_headers.conf | 2 +-
.../resources/text/local_file_text_to_assert.conf | 2 +-
.../src/test/resources/iceberg/iceberg_source.conf | 34 +-
.../jdbc/internal/xa/XaGroupOpsImplIT.java | 20 +-
.../resources/jdbc_sink_auto_generate_sql.conf | 3 +-
.../jdbc_sink_auto_generate_upsql_sql.conf | 4 +-
.../test/resources/jdbc_sink_cdc_changelog.conf | 4 +-
.../resources/kafkasource_canal_cdc_to_pgsql.conf | 2 +-
.../connector/starrocks/StarRocksCDCSinkIT.java | 2 +
.../e2e/connector/starrocks/StarRocksIT.java | 2 +
.../common/container/AbstractTestContainer.java | 2 +-
.../flink/AbstractTestFlinkContainer.java | 3 +
.../container/seatunnel/SeaTunnelContainer.java | 15 +-
.../spark/AbstractTestSparkContainer.java | 4 +
.../engine/e2e/ClusterFaultToleranceIT.java | 107 +++--
.../e2e/ClusterFaultToleranceTwoPipelineIT.java | 60 ++-
.../seatunnel/engine/e2e/JobExecutionIT.java | 2 +-
.../test/resources/batch_fakesource_to_file.conf | 2 +-
.../batch_fakesource_to_file_complex.conf | 2 +-
.../cluster_batch_fake_to_localfile_template.conf | 2 +-
...ch_fake_to_localfile_two_pipeline_template.conf | 2 +-
.../src/test/resources/hazelcast.yaml | 5 +-
.../src/test/resources}/jvm_options | 5 +-
.../src/test/resources/log4j2-test.properties | 2 +-
.../src/test/resources/seatunnel.yaml | 8 +-
.../streaming_fakesource_to_file_complex.conf | 2 +-
.../seatunnel/engine/client/SeaTunnelClient.java | 2 +-
.../engine/client/job/ClientJobProxy.java | 43 +-
.../seatunnel/engine/client/job/JobClient.java | 37 +-
.../engine/client/job/JobExecutionEnvironment.java | 10 +-
.../engine/client/util/ContentFormatUtil.java | 107 +++++
.../apache/seatunnel/engine/client/TestUtils.java | 44 ++
.../src/test/resources/custmoize-client.yaml} | 23 +-
.../src/test/resources/hazelcast-client.yaml | 37 +-
.../src/test/resources/hazelcast.yaml | 14 +-
.../src/test/resources/seatunnel.yaml | 9 +-
.../apache/seatunnel/engine/common/Constant.java | 3 -
.../engine/common/config/EngineConfig.java | 12 +-
.../config/YamlSeaTunnelDomConfigProcessor.java | 12 +-
.../common/config/server/ServerConfigOptions.java | 9 +-
.../common/config/server/ThreadShareMode.java | 17 +-
.../exception/SchedulerNotAllowException.java | 31 +-
.../src/main/resources/hazelcast.yaml | 1 +
.../src/main/resources/seatunnel.yaml | 18 +-
.../config/YamlSeaTunnelConfigParserTest.java | 4 +-
.../src/test/resources/seatunnel.yaml | 6 +-
.../engine/core/dag/actions/AbstractAction.java | 25 +-
.../seatunnel/engine/core/dag/actions/Action.java | 2 +
.../seatunnel/engine/core/dag/actions/Config.java | 10 +-
.../core/dag/actions/PartitionTransformAction.java | 54 ---
.../actions/{Action.java => ShuffleAction.java} | 28 +-
.../actions/{Action.java => ShuffleConfig.java} | 45 +-
.../dag/actions/ShuffleMultipleRowStrategy.java | 92 ++++
.../core/dag/actions/ShufflePartitionStrategy.java | 97 +++++
.../engine/core/dag/actions/ShuffleStrategy.java | 70 +++
.../engine/core/dag/actions/SinkAction.java | 23 +-
.../engine/core/dag/actions/SinkConfig.java | 20 +-
.../engine/core/job/JobImmutableInformation.java | 15 +-
.../seatunnel/engine/core/job/JobStatusData.java | 19 +-
.../engine/core/parse/JobConfigParser.java | 55 +--
.../core/parse/MultipleTableJobConfigParser.java | 322 ++++++++++++++
seatunnel-engine/seatunnel-engine-server/pom.xml | 6 -
.../engine/server/CoordinatorService.java | 31 +-
.../seatunnel/engine/server/SeaTunnelServer.java | 11 +-
.../engine/server/TaskExecutionService.java | 110 ++++-
.../server/checkpoint/CheckpointCoordinator.java | 95 ++---
.../server/checkpoint/CheckpointManager.java | 36 +-
.../server/checkpoint/PendingCheckpoint.java | 3 +-
.../CheckpointBarrierTriggerOperation.java | 6 +-
.../operation/CheckpointFinishedOperation.java | 3 +-
.../operation/NotifyTaskRestoreOperation.java | 8 +-
.../operation/NotifyTaskStartOperation.java | 4 +-
.../operation/TaskAcknowledgeOperation.java | 4 +
.../operation/TaskReportStatusOperation.java | 11 +-
.../seatunnel/engine/server/dag/DAGUtils.java | 4 +-
.../dag/execution/ExecutionPlanGenerator.java | 468 ++++++++++++++-------
.../server/dag/execution/PipelineGenerator.java | 9 +-
.../engine/server/dag/physical/PhysicalPlan.java | 85 ++--
.../server/dag/physical/PhysicalPlanGenerator.java | 216 +++++++---
.../engine/server/dag/physical/PhysicalVertex.java | 5 +-
.../engine/server/dag/physical/PlanUtils.java | 8 +-
.../engine/server/dag/physical/SubPlan.java | 105 +++--
.../TaskGroupContextNotFoundException.java | 17 +-
.../engine/server/execution/TaskCallTimer.java | 1 +
.../server/execution/TaskExecutionContext.java | 5 +
.../engine/server/master/JobHistoryService.java | 71 ++--
.../seatunnel/engine/server/master/JobMaster.java | 91 ++--
.../operation/GetJobDetailStatusOperation.java | 5 +-
.../server/operation/GetJobInfoOperation.java | 5 +-
.../server/operation/GetJobMetricsOperation.java | 5 +-
.../server/operation/GetJobStatusOperation.java | 5 +-
.../server/operation/ListJobStatusOperation.java | 6 +-
.../resourcemanager/AbstractResourceManager.java | 73 ++--
.../server/scheduler/PipelineBaseScheduler.java | 31 +-
.../server/serializable/RecordSerializer.java | 93 ++++
.../server/serializable/RecordSerializerHook.java | 26 +-
.../engine/server/serializable/TypeId.java | 15 +-
.../server/service/slot/DefaultSlotService.java | 2 +-
.../engine/server/task/SeaTunnelTask.java | 49 ++-
.../server/task/SinkAggregatedCommitterTask.java | 24 +-
.../server/task/SourceSplitEnumeratorTask.java | 21 +-
.../context/SeaTunnelSplitEnumeratorContext.java | 6 +-
.../flow/PartitionTransformSinkFlowLifeCycle.java | 67 ---
.../PartitionTransformSourceFlowLifeCycle.java | 85 ----
.../server/task/flow/ShuffleSinkFlowLifeCycle.java | 186 ++++++++
.../task/flow/ShuffleSourceFlowLifeCycle.java | 145 +++++++
.../server/task/flow/SourceFlowLifeCycle.java | 19 +-
.../TaskGroupWithIntermediateBlockingQueue.java | 2 +-
.../CheckTaskGroupIsExecutingOperation.java | 8 +-
.../operation/checkpoint/BarrierFlowOperation.java | 8 +-
.../checkpoint/CloseRequestOperation.java | 4 +-
.../task/operation/sink/SinkRegisterOperation.java | 12 +-
.../operation/source/AssignSplitOperation.java | 4 +-
.../operation/source/RequestSplitOperation.java | 4 +-
.../operation/source/RestoredSplitOperation.java | 25 +-
.../source/SourceNoMoreElementOperation.java | 4 +-
.../source/SourceReaderEventOperation.java | 4 +-
.../operation/source/SourceRegisterOperation.java | 12 +-
.../engine/server/CoordinatorServiceTest.java | 2 +
.../engine/server/TaskExecutionServiceTest.java | 27 ++
.../apache/seatunnel/engine/server/TestUtils.java | 12 +
.../server/checkpoint/CheckpointManagerTest.java | 10 +-
.../server/checkpoint/CheckpointPlanTest.java | 16 +-
.../server/checkpoint/CheckpointSerializeTest.java | 91 ++++
.../engine/server/checkpoint/SavePointTest.java | 1 +
.../seatunnel/engine/server/dag/TaskTest.java | 26 +-
.../engine/server/execution/BlockTask.java | 32 +-
.../server/master/JobHistoryServiceTest.java | 9 +-
.../engine/server/master/JobMasterTest.java | 1 +
.../engine/server/master/JobMetricsTest.java | 11 +-
.../src/test/resources/hazelcast-client.yaml | 37 +-
.../src/test/resources/hazelcast.yaml | 14 +-
.../src/test/resources/seatunnel.yaml | 8 +-
.../checkpoint/storage/api/CheckpointStorage.java | 3 +
.../checkpoint-storage-hdfs/pom.xml | 13 +
.../checkpoint/storage/hdfs/HdfsStorage.java | 60 ++-
.../storage/hdfs/S3FileCheckpointTest.java | 4 +-
.../storage/localfile/LocalFileStorage.java | 121 +++++-
.../storage/localfile/LocalFileStorageFactory.java | 1 -
.../json/canal/CanalJsonSerDeSchemaTest.java | 52 +--
.../plugin/discovery/AbstractPluginDiscovery.java | 4 +
.../seatunnel-hadoop3-3.1.4-uber/pom.xml | 5 +
.../seatunnel/transform/FieldMapperTransform.java | 5 +-
.../common/AbstractSeaTunnelTransform.java | 7 +-
.../transform/common/SeaTunnelRowAccessor.java | 2 +-
.../spark/source/SeaTunnelSourceSupport.java | 6 +-
.../spark/source/SeaTunnelSourceTable.java | 6 +-
tools/dependencies/known-dependencies.txt | 13 +-
411 files changed, 10077 insertions(+), 3934 deletions(-)