You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ki...@apache.org on 2022/07/01 08:26:52 UTC
[incubator-seatunnel] 01/01: Merge pull request #2083 from apache/api-draft
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
commit df251bf47773fa2f80c2d7008d9ff94b88c8c912
Merge: 770bade75 1b706c16a
Author: Kirs <ki...@apache.org>
AuthorDate: Fri Jul 1 16:26:43 2022 +0800
Merge pull request #2083 from apache/api-draft
[API-DRAFT] [MERGE] Merge api-draft to dev branch
.github/workflows/backend.yml | 2 +-
.licenserc.yaml | 2 +-
LICENSE | 1 +
config/flink.batch.conf.template | 2 +
config/spark.streaming.conf.template | 2 +-
docs/en/connector/sink/Assert.md | 47 +-
docs/en/images/seatunnel_architecture.png | Bin 0 -> 778394 bytes
docs/en/images/seatunnel_starter.png | Bin 0 -> 423840 bytes
.../en/{connector => new-connector}/sink/Assert.md | 51 +-
docs/en/new-connector/sink/Clickhouse.md | 104 ++++
docs/en/new-connector/sink/ClickhouseFile.md | 106 ++++
docs/en/new-connector/sink/Jdbc.md | 96 ++++
docs/en/new-connector/sink/common-options.md | 45 ++
docs/en/new-connector/source/Clickhouse.md | 63 ++
docs/en/new-connector/source/Jdbc.md | 75 +++
docs/en/new-connector/source/common-options.md | 33 ++
...mapping.properties => plugin-mapping.properties | 17 +
pom.xml | 105 ++--
{seatunnel-core => seatunnel-api}/pom.xml | 23 +-
.../api/common/PluginIdentifierInterface.java | 21 +-
.../seatunnel/api/common/PrepareFailException.java | 30 +-
.../seatunnel/api/common/SeaTunnelContext.java | 87 +++
.../api/common/SeaTunnelPluginLifeCycle.java | 35 +-
.../api/serialization/DefaultSerializer.java | 31 +-
.../api/serialization/DeserializationSchema.java | 36 +-
.../api/serialization/SerializationSchema.java | 21 +-
.../seatunnel/api/serialization/Serializer.java | 29 +-
.../api/sink/DefaultSinkWriterContext.java | 36 +-
.../apache/seatunnel/api/sink/SeaTunnelSink.java | 122 ++++
.../api/sink/SinkAggregatedCommitter.java | 66 +++
.../apache/seatunnel/api/sink/SinkCommitter.java | 50 ++
.../org/apache/seatunnel/api/sink/SinkWriter.java | 95 +++
.../apache/seatunnel/api/source/Boundedness.java | 22 +-
.../org/apache/seatunnel/api/source/Collector.java | 20 +-
.../api/source/SeaTunnelRuntimeEnvironment.java | 19 +-
.../seatunnel/api/source/SeaTunnelSource.java | 99 ++++
.../apache/seatunnel/api/source/SourceEvent.java | 18 +-
.../apache/seatunnel/api/source/SourceReader.java | 119 ++++
.../apache/seatunnel/api/source/SourceSplit.java | 18 +-
.../api/source/SourceSplitEnumerator.java | 124 ++++
.../seatunnel/api/source/SupportCoordinate.java | 18 +-
.../seatunnel/api/state/CheckpointListener.java | 18 +-
.../seatunnel/api/table/catalog/Catalog.java | 116 ++++
.../seatunnel/api/table/catalog/CatalogTable.java | 93 +++
.../apache/seatunnel/api/table/catalog/Column.java | 223 ++++++++
.../api/table/catalog/TableIdentifier.java | 81 +++
.../seatunnel/api/table/catalog/TablePath.java | 86 +++
.../seatunnel/api/table/catalog/TableSchema.java | 132 +++++
.../table/catalog/exception/CatalogException.java | 31 +-
.../exception/DatabaseNotExistException.java | 32 ++
.../catalog/exception/TableNotExistException.java | 35 ++
.../api/table/connector/DeserializationFormat.java | 28 +-
.../api/table/connector/SerializationFormat.java | 15 +-
.../table/connector/SupportReadingMetadata.java | 21 +-
.../seatunnel/api/table/connector/TableSink.java | 15 +-
.../seatunnel/api/table/connector/TableSource.java | 16 +-
.../api/table/factory/CatalogFactory.java | 19 +-
.../factory/DeserializationFormatFactory.java | 16 +-
.../seatunnel/api/table/factory/Factory.java | 21 +-
.../api/table/factory/FactoryException.java | 15 +-
.../seatunnel/api/table/factory/FactoryUtil.java | 155 +++++
.../table/factory/SerializationFormatFactory.java | 16 +-
.../api/table/factory/SupportMultipleTable.java | 60 ++
.../api/table/factory/TableFactoryContext.java | 58 ++
.../api/table/factory/TableSinkFactory.java | 20 +-
.../api/table/factory/TableSourceFactory.java | 31 +-
.../apache/seatunnel/api/table/type/ArrayType.java | 87 +++
.../apache/seatunnel/api/table/type/BasicType.java | 79 +++
.../seatunnel/api/table/type/CompositeType.java | 15 +-
.../seatunnel/api/table/type/DecimalType.java | 65 +++
.../seatunnel/api/table/type/LocalTimeType.java | 72 +++
.../apache/seatunnel/api/table/type/MapType.java | 104 ++++
.../api/table/type/PrimitiveByteArrayType.java | 38 +-
.../apache/seatunnel/api/table/type/RowKind.java | 117 ++++
.../api/table/type/SeaTunnelDataType.java | 19 +-
.../seatunnel/api/table/type/SeaTunnelRow.java | 106 ++++
.../seatunnel/api/table/type/SeaTunnelRowType.java | 121 ++++
.../apache/seatunnel/api/table/type/SqlType.java | 34 +-
.../apache/seatunnel/apis/base/env/RuntimeEnv.java | 4 +-
.../apache/seatunnel/apis/base/plugin/Plugin.java | 14 +-
.../apache/seatunnel/flink/FlinkEnvironment.java | 18 +-
.../apache/seatunnel/flink/util/SchemaUtil.java | 2 +
.../apache/seatunnel/spark/SparkEnvironment.java | 4 +-
seatunnel-common/pom.xml | 9 +
.../apache/seatunnel/common/PropertiesUtil.java | 37 ++
.../org/apache/seatunnel/common/config/Common.java | 33 +-
.../common/constants/CollectionConstants.java | 15 +-
.../seatunnel/common/constants}/PluginType.java | 6 +-
.../seatunnel/common/utils/ReflectionUtils.java | 49 ++
.../seatunnel/common/utils/SerializationUtils.java | 49 ++
.../apache/seatunnel/common/config/CommonTest.java | 2 +-
.../common/utils/VariablesSubstituteTest.java | 7 +-
seatunnel-connectors-v2-dist/pom.xml | 108 ++++
seatunnel-connectors-v2/README.md | 84 +++
seatunnel-connectors-v2/README.zh.md | 64 +++
.../connector-assert}/pom.xml | 33 +-
.../assertion/excecutor/AssertExecutor.java | 108 ++++
.../seatunnel/assertion/rule/AssertFieldRule.java | 65 +++
.../seatunnel/assertion/rule/AssertRuleParser.java | 83 +++
.../seatunnel/assertion/sink/AssertSink.java | 85 +++
.../seatunnel/assertion/sink/AssertSinkWriter.java | 55 ++
.../flink/assertion/AssertExecutorTest.java | 110 ++++
.../flink/assertion/rule/AssertRuleParserTest.java | 85 +++
.../connector-clickhouse/pom.xml | 70 +++
.../config/ClickhouseFileCopyMethod.java | 34 +-
.../seatunnel/clickhouse/config/Config.java | 96 ++++
.../clickhouse/config/FileReaderOption.java | 113 ++++
.../seatunnel/clickhouse/config/ReaderOption.java | 94 +++
.../seatunnel/clickhouse/shard/Shard.java | 98 ++++
.../seatunnel/clickhouse/shard/ShardMetadata.java | 145 +++++
.../clickhouse/sink/DistributedEngine.java | 58 ++
.../sink/client/ClickhouseBatchStatement.java | 52 ++
.../clickhouse/sink/client/ClickhouseProxy.java | 212 +++++++
.../clickhouse/sink/client/ClickhouseSink.java | 172 ++++++
.../sink/client/ClickhouseSinkWriter.java | 230 ++++++++
.../clickhouse/sink/client/ShardRouter.java | 97 ++++
.../clickhouse/sink/file/ClickhouseFileSink.java | 145 +++++
.../sink/file/ClickhouseFileSinkWriter.java | 237 ++++++++
.../clickhouse/sink/file/ClickhouseTable.java | 117 ++++
.../clickhouse/sink/file/FileTransfer.java | 19 +-
.../clickhouse/sink/file/FileTransferFactory.java | 36 +-
.../clickhouse/sink/file/RsyncFileTransfer.java | 144 +++++
.../clickhouse/sink/file/ScpFileTransfer.java | 124 ++++
.../sink/inject/ArrayInjectFunction.java | 28 +-
.../sink/inject/BigDecimalInjectFunction.java | 28 +-
.../sink/inject/ClickhouseFieldInjectFunction.java | 46 ++
.../clickhouse/sink/inject/DateInjectFunction.java | 37 +-
.../sink/inject/DateTimeInjectFunction.java | 34 +-
.../sink/inject/DoubleInjectFunction.java | 31 +-
.../sink/inject/FloatInjectFunction.java | 34 +-
.../clickhouse/sink/inject/IntInjectFunction.java | 47 ++
.../clickhouse/sink/inject/LongInjectFunction.java | 29 +-
.../sink/inject/StringInjectFunction.java | 18 +-
.../clickhouse/source/ClickhouseSource.java | 132 +++++
.../clickhouse/source/ClickhouseSourceReader.java | 108 ++++
.../clickhouse/source/ClickhouseSourceSplit.java | 17 +-
.../source/ClickhouseSourceSplitEnumerator.java | 100 ++++
.../clickhouse/state/CKAggCommitInfo.java | 15 +-
.../seatunnel/clickhouse/state/CKCommitInfo.java | 15 +-
.../clickhouse/state/ClickhouseSinkState.java | 15 +-
.../clickhouse/state/ClickhouseSourceState.java | 15 +-
.../seatunnel/clickhouse/tool/IntHolder.java | 19 +-
.../seatunnel/clickhouse/util/ClickhouseUtil.java | 40 ++
.../seatunnel/clickhouse/util/TypeConvertUtil.java | 100 ++++
.../connector-common}/pom.xml | 23 +-
.../seatunnel/common/sink/AbstractSimpleSink.java | 59 ++
.../seatunnel/common/sink/AbstractSinkWriter.java | 19 +-
.../common/source/AbstractSingleSplitReader.java | 67 +++
.../common/source/AbstractSingleSplitSource.java | 58 ++
.../seatunnel/common/source/SingleSplit.java | 21 +-
.../common/source/SingleSplitEnumerator.java | 94 +++
.../common/source/SingleSplitEnumeratorState.java | 15 +-
.../common/source/SingleSplitReaderContext.java | 26 +-
.../connector-console}/pom.xml | 23 +-
.../seatunnel/console/sink/ConsoleSink.java | 63 ++
.../seatunnel/console/sink/ConsoleSinkWriter.java | 37 +-
.../connector-fake}/pom.xml | 21 +-
.../seatunnel/fake/source/FakeSource.java | 73 +++
.../seatunnel/fake/source/FakeSourceReader.java | 73 +++
.../connector-hive}/pom.xml | 25 +-
.../connectors/seatunnel/hive/config/Config.java | 15 +-
.../hive/sink/HiveAggregatedCommitInfo.java | 25 +-
.../seatunnel/hive/sink/HiveCommitInfo.java | 25 +-
.../connectors/seatunnel/hive/sink/HiveSink.java | 94 +++
.../hive/sink/HiveSinkAggregatedCommitter.java | 88 +++
.../seatunnel/hive/sink/HiveSinkConfig.java | 106 ++++
.../seatunnel/hive/sink/HiveSinkState.java | 19 +-
.../seatunnel/hive/sink/HiveSinkWriter.java | 99 ++++
.../hive/sink/file/writer/AbstractFileWriter.java | 155 +++++
.../hive/sink/file/writer/FileWriter.java | 49 ++
.../hive/sink/file/writer/HdfsTxtFileWriter.java | 151 +++++
.../seatunnel/hive/sink/file/writer/HdfsUtils.java | 96 ++++
.../connector-http}/pom.xml | 23 +-
.../seatunnel/http/client/HttpClientProvider.java | 283 +++++++++
.../seatunnel/http/client/HttpResponse.java | 74 +++
.../connectors/seatunnel/http/config/Config.java | 18 +-
.../seatunnel/http/source/HttpSource.java | 109 ++++
.../seatunnel/http/source/HttpSourceParameter.java | 70 +++
.../seatunnel/http/source/HttpSourceReader.java | 79 +++
.../connector-jdbc}/pom.xml | 26 +-
.../jdbc/catalog/AbstractJdbcCatalog.java | 211 +++++++
.../seatunnel/jdbc/catalog/MySqlCatalog.java | 230 ++++++++
.../seatunnel/jdbc/config/JdbcConfig.java | 102 ++++
.../seatunnel/jdbc/config/JdbcSinkOptions.java | 35 +-
.../seatunnel/jdbc/config/JdbcSourceOptions.java | 77 +++
.../seatunnel/jdbc/internal/JdbcInputFormat.java | 220 +++++++
.../seatunnel/jdbc/internal/JdbcOutputFormat.java | 257 +++++++++
.../jdbc/internal/connection/DataSourceUtils.java | 104 ++++
.../connection/JdbcConnectionProvider.java | 70 +++
.../connection/SimpleJdbcConnectionProvider.java | 154 +++++
.../converter/AbstractJdbcRowConverter.java | 97 ++++
.../jdbc/internal/converter/JdbcRowConverter.java | 25 +-
.../jdbc/internal/dialect/JdbcDialect.java | 52 ++
.../jdbc/internal/dialect/JdbcDialectFactory.java | 25 +-
.../jdbc/internal/dialect/JdbcDialectLoader.java | 102 ++++
.../internal/dialect/JdbcDialectTypeMapper.java | 23 +-
.../dialect/mysql/MySqlDialectFactory.java | 29 +-
.../internal/dialect/mysql/MySqlTypeMapper.java | 170 ++++++
.../jdbc/internal/dialect/mysql/MysqlDialect.java | 30 +-
.../dialect/mysql/MysqlJdbcRowConverter.java | 31 +-
.../internal/dialect/psql/PostgresDialect.java | 30 +-
.../dialect/psql/PostgresDialectFactory.java | 29 +-
.../dialect/psql/PostgresJdbcRowConverter.java | 31 +-
.../internal/dialect/psql/PostgresTypeMapper.java | 159 +++++
.../internal/executor/BiConsumerWithException.java | 61 ++
.../executor/JdbcBatchStatementExecutor.java | 29 +-
.../internal/executor/JdbcStatementBuilder.java | 22 +-
.../executor/SimpleBatchStatementExecutor.java | 79 +++
.../jdbc/internal/options/JdbcConfig.java | 55 ++
.../internal/options/JdbcConnectionOptions.java | 200 +++++++
.../internal/options/JdbcConnectorOptions.java | 253 ++++++++
.../split/JdbcGenericParameterValuesProvider.java | 30 +-
.../JdbcNumericBetweenParametersProvider.java | 116 ++++
.../split/JdbcParameterValuesProvider.java | 32 +-
.../jdbc/internal/xa/GroupXaOperationResult.java | 80 +++
.../jdbc/internal/xa/SemanticXidGenerator.java | 113 ++++
.../seatunnel/jdbc/internal/xa/XaFacade.java | 113 ++++
.../jdbc/internal/xa/XaFacadeImplAutoLoad.java | 477 +++++++++++++++
.../seatunnel/jdbc/internal/xa/XaGroupOps.java | 43 ++
.../seatunnel/jdbc/internal/xa/XaGroupOpsImpl.java | 151 +++++
.../seatunnel/jdbc/internal/xa/XidGenerator.java | 62 ++
.../seatunnel/jdbc/internal/xa/XidImpl.java | 136 +++++
.../jdbc/sink/JdbcExactlyOnceSinkWriter.java | 190 ++++++
.../connectors/seatunnel/jdbc/sink/JdbcSink.java | 143 +++++
.../jdbc/sink/JdbcSinkAggregatedCommitter.java | 91 +++
.../seatunnel/jdbc/sink/JdbcSinkCommitter.java | 70 +++
.../seatunnel/jdbc/sink/JdbcSinkWriter.java | 96 ++++
.../seatunnel/jdbc/source/JdbcSource.java | 216 +++++++
.../seatunnel/jdbc/source/JdbcSourceReader.java | 97 ++++
.../seatunnel/jdbc/source/JdbcSourceSplit.java | 22 +-
.../jdbc/source/JdbcSourceSplitEnumerator.java | 100 ++++
.../seatunnel/jdbc/source/PartitionParameter.java | 20 +-
.../jdbc/state/JdbcAggregatedCommitInfo.java | 20 +-
.../seatunnel/jdbc/state/JdbcSinkState.java | 19 +-
.../seatunnel/jdbc/state/JdbcSourceState.java | 15 +-
.../connectors/seatunnel/jdbc/state/XidInfo.java | 23 +-
.../seatunnel/jdbc/utils/ExceptionUtils.java | 53 ++
.../connectors/seatunnel/jdbc/utils/JdbcUtils.java | 158 +++++
.../seatunnel/jdbc/utils/ThrowingRunnable.java | 52 ++
.../jdbc/catalog/AbstractJdbcCatalogTest.java | 48 ++
.../connector-kafka}/pom.xml | 25 +-
.../connectors/seatunnel/kafka/config/Config.java | 53 ++
.../seatunnel/kafka/config/KafkaSemantics.java | 27 +-
.../serialize/DefaultSeaTunnelRowSerializer.java | 50 ++
.../kafka/serialize/SeaTunnelRowSerializer.java | 22 +-
.../kafka/sink/KafkaInternalProducer.java | 164 ++++++
.../kafka/sink/KafkaNoTransactionSender.java | 83 +++
.../seatunnel/kafka/sink/KafkaProduceSender.java | 64 +++
.../connectors/seatunnel/kafka/sink/KafkaSink.java | 95 +++
.../seatunnel/kafka/sink/KafkaSinkCommitter.java | 85 +++
.../seatunnel/kafka/sink/KafkaSinkWriter.java | 161 ++++++
.../kafka/sink/KafkaTransactionSender.java | 126 ++++
.../seatunnel/kafka/source/ConsumerMetadata.java | 82 +++
.../seatunnel/kafka/source/KafkaSource.java | 132 +++++
.../seatunnel/kafka/source/KafkaSourceReader.java | 171 ++++++
.../seatunnel/kafka/source/KafkaSourceSplit.java | 72 +++
.../kafka/source/KafkaSourceSplitEnumerator.java | 167 ++++++
.../kafka/state/KafkaAggregatedCommitInfo.java | 19 +-
.../seatunnel/kafka/state/KafkaCommitInfo.java | 23 +-
.../seatunnel/kafka/state/KafkaSinkState.java | 23 +-
.../seatunnel/kafka/state/KafkaSourceState.java | 35 +-
seatunnel-connectors-v2/connector-pulsar/pom.xml | 99 ++++
.../seatunnel/pulsar/config/BasePulsarConfig.java | 27 +-
.../seatunnel/pulsar/config/PulsarAdminConfig.java | 76 +++
.../pulsar/config/PulsarClientConfig.java | 80 +++
.../seatunnel/pulsar/config/PulsarConfigUtil.java | 81 +++
.../pulsar/config/PulsarConsumerConfig.java | 60 ++
.../seatunnel/pulsar/config/SourceProperties.java | 119 ++++
.../seatunnel/pulsar/source/PulsarSource.java | 281 +++++++++
.../source/enumerator/PulsarSplitEnumerator.java | 281 +++++++++
.../enumerator/PulsarSplitEnumeratorState.java | 36 ++
.../cursor/start/MessageIdStartCursor.java | 65 +++
.../enumerator/cursor/start/StartCursor.java | 78 +++
.../cursor/start/SubscriptionStartCursor.java | 63 ++
.../cursor/start/TimestampStartCursor.java | 40 ++
.../cursor/stop/LatestMessageStopCursor.java | 59 ++
.../cursor/stop/MessageIdStopCursor.java | 58 ++
.../enumerator/cursor/stop/NeverStopCursor.java | 49 ++
.../source/enumerator/cursor/stop/StopCursor.java | 63 ++
.../cursor/stop/TimestampStopCursor.java | 44 ++
.../enumerator/discoverer/PulsarDiscoverer.java | 48 ++
.../enumerator/discoverer/TopicListDiscoverer.java | 63 ++
.../discoverer/TopicPatternDiscoverer.java | 98 ++++
.../source/enumerator/topic/TopicPartition.java | 91 +++
.../seatunnel/pulsar/source/reader/Handover.java | 88 +++
.../pulsar/source/reader/PulsarSourceReader.java | 221 +++++++
.../source/reader/PulsarSplitReaderThread.java | 143 +++++
.../pulsar/source/reader/RecordWithSplitId.java | 22 +-
.../pulsar/source/split/PulsarPartitionSplit.java | 94 +++
.../connector-socket}/pom.xml | 21 +-
.../seatunnel/socket/source/SocketSource.java | 72 +++
.../socket/source/SocketSourceParameter.java | 32 +-
.../socket/source/SocketSourceReader.java | 91 +++
.../seatunnel/socket/state/SocketState.java | 15 +-
.../pom.xml | 40 +-
.../seatunnel-connector-flink-clickhouse/pom.xml | 5 -
.../flink/clickhouse/sink/ClickhouseBatchSink.java | 3 -
.../clickhouse/sink/ClickhouseOutputFormat.java | 2 +-
.../org/apache/seatunnel/flink/fake/Config.java | 2 +-
.../flink/fake/source/FakeSourceStream.java | 2 +-
seatunnel-core/README.md | 8 +
seatunnel-core/pom.xml | 3 +
seatunnel-core/seatunnel-core-base/pom.xml | 6 +
.../org/apache/seatunnel/core/base/Seatunnel.java | 3 +-
.../core/base/command/AbstractCommandArgs.java | 1 +
.../core/base/command/BaseTaskExecuteCommand.java | 5 +-
.../seatunnel/core/base/command/Command.java | 3 +-
...nContext.java => AbstractExecutionContext.java} | 43 +-
.../seatunnel/core/base/config/ConfigBuilder.java | 21 +-
.../seatunnel/core/base/config/ConfigChecker.java | 36 +-
.../core/base/config/EnvironmentFactory.java | 2 +
.../core/base/config/ExecutionFactory.java | 4 +-
.../seatunnel/core/base/config/PluginFactory.java | 237 --------
.../CommandException.java} | 16 +-
.../CommandExecuteException.java} | 16 +-
.../ConfigCheckException.java} | 16 +-
.../core/base/config/PluginFactoryTest.java | 65 ---
.../seatunnel/core/base/utils/FileUtilsTest.java | 1 +
.../apache/seatunnel/core/sql/FlinkSqlStarter.java | 6 +-
.../apache/seatunnel/core/sql/job/Executor.java | 16 +-
seatunnel-core/seatunnel-core-flink/pom.xml | 24 +-
.../apache/seatunnel/core/flink/FlinkStarter.java | 4 +-
.../seatunnel/core/flink/SeatunnelFlink.java | 3 +-
.../core/flink/args/FlinkCommandArgs.java | 4 +-
...mmand.java => FlinkApiConfValidateCommand.java} | 14 +-
...ommand.java => FlinkApiTaskExecuteCommand.java} | 37 +-
.../core/flink/command/FlinkCommandBuilder.java | 9 +-
.../core/flink/config/FlinkApiConfigChecker.java | 44 ++
.../core/flink/config/FlinkExecutionContext.java | 104 ++++
.../core/flink/utils/CommandLineUtils.java | 17 +-
.../flink/command/FlinkTaskExecuteCommandTest.java | 2 +-
.../core/flink/utils/CommandLineUtilsTest.java | 17 +-
seatunnel-core/seatunnel-core-spark/pom.xml | 26 +
.../seatunnel/core/spark/SeatunnelSpark.java | 3 +-
.../apache/seatunnel/core/spark/SparkStarter.java | 36 +-
.../core/spark/command/SparkCommandBuilder.java | 5 +-
.../spark/command/SparkConfValidateCommand.java | 8 +-
.../spark/command/SparkTaskExecuteCommand.java | 59 +-
.../core/spark/config/SparkApiConfigChecker.java | 44 ++
.../core/spark/config/SparkExecutionContext.java | 103 ++++
seatunnel-core/seatunnel-core-starter/README.md | 12 +
.../pom.xml | 7 +-
.../apache/seatunnel/core/starter}/Seatunnel.java | 9 +-
.../apache/seatunnel/core/starter/Starter.java} | 16 +-
.../core/starter}/command/AbstractCommandArgs.java | 6 +-
.../seatunnel/core/starter}/command/Command.java | 8 +-
.../core/starter/command/CommandArgs.java} | 18 +-
.../core/starter/command/CommandBuilder.java} | 16 +-
.../core/starter/command/DeployModeConverter.java} | 20 +-
.../starter/config/AbstractExecutionContext.java} | 45 +-
.../core/starter}/config/ConfigBuilder.java | 22 +-
.../core/starter/config/ConfigChecker.java | 36 +-
.../seatunnel/core/starter/config/EngineType.java} | 18 +-
.../core/starter}/config/EnvironmentFactory.java | 3 +-
.../core/starter}/config/ExecutionFactory.java | 6 +-
.../seatunnel/core/starter}/config/PluginType.java | 2 +-
.../starter/constants/CommonParamConstants.java} | 16 +-
.../core/starter/exception/CommandException.java} | 16 +-
.../exception/CommandExecuteException.java} | 16 +-
.../starter/exception/ConfigCheckException.java} | 16 +-
.../starter/exception/TaskExecuteException.java} | 15 +-
.../core/starter/execution/TaskExecution.java} | 15 +-
.../core/starter/utils/AsciiArtUtils.java | 72 +++
.../core/starter/utils/CompressionUtils.java | 158 +++++
.../seatunnel/core/starter/utils/FileUtils.java | 70 +++
.../core/starter/utils/CompressionUtilsTest.java | 50 ++
.../core/starter}/utils/FileUtilsTest.java | 4 +-
.../src/test/resources/flink.batch.conf | 5 +-
.../pom.xml | 64 ++-
.../bin/start-seatunnel-flink-new-connector.sh | 46 ++
.../src/main/docker/Dockerfile | 37 ++
.../core/starter}/flink/FlinkStarter.java | 16 +-
.../core/starter}/flink/SeatunnelFlink.java | 17 +-
.../core/starter}/flink/args/FlinkCommandArgs.java | 9 +-
.../command/FlinkApiConfValidateCommand.java} | 27 +-
.../flink/command/FlinkApiTaskExecuteCommand.java | 61 ++
.../flink/command/FlinkCommandBuilder.java | 18 +-
.../flink/config/FlinkApiConfigChecker.java} | 17 +-
.../starter/flink/config/FlinkApiEnvironment.java | 75 +++
.../core/starter/flink/config/FlinkJobType.java} | 12 +-
.../core/starter/flink/config/FlinkRunMode.java} | 22 +-
.../starter/flink/constant/FlinkConstant.java} | 16 +-
.../core/starter/flink/env}/FlinkEnvironment.java | 63 +-
.../execution/AbstractPluginExecuteProcessor.java | 76 +++
.../starter/flink/execution/FlinkExecution.java | 73 +++
.../flink/execution/PluginExecuteProcessor.java} | 25 +-
.../flink/execution/SinkExecuteProcessor.java | 84 +++
.../flink/execution/SourceExecuteProcessor.java | 121 ++++
.../flink/execution/TransformExecuteProcessor.java | 86 +++
.../starter}/flink/utils/CommandLineUtils.java | 28 +-
.../core/starter/flink/FlinkStarterTest.java | 65 +++
.../starter/flink/args/FlinkCommandArgsTest.java | 30 +-
.../pom.xml | 58 +-
.../bin/start-seatunnel-spark-new-connector.sh | 45 ++
.../src/main/docker/Dockerfile | 37 ++
.../core/starter}/spark/SeatunnelSpark.java | 15 +-
.../core/starter}/spark/SparkStarter.java | 68 ++-
.../core/starter/spark/args/SparkCommandArgs.java | 62 ++
.../command/SparkApiConfValidateCommand.java} | 29 +-
.../spark/command/SparkApiTaskExecuteCommand.java | 60 ++
.../spark/command/SparkCommandBuilder.java | 26 +-
.../spark/config/SparkApiConfigChecker.java} | 17 +-
.../starter/spark/config/SparkEnvironment.java | 45 +-
.../execution/AbstractPluginExecuteProcessor.java | 68 +++
.../spark/execution/PluginExecuteProcessor.java} | 25 +-
.../spark/execution/SinkExecuteProcessor.java | 84 +++
.../spark/execution/SourceExecuteProcessor.java | 81 +++
.../starter/spark/execution/SparkExecution.java | 63 ++
.../spark/execution/TransformExecuteProcessor.java | 80 +++
.../core/starter/spark/utils/CommandLineUtils.java | 29 +-
.../core/starter/spark/SparkStarterTest.java | 25 +-
.../starter/spark/args/SparkCommandArgsTest.java | 67 +++
.../starter/spark/utils/CommandLineUtilsTest.java | 27 +-
.../src/test/resources/spark_application.conf | 66 +++
seatunnel-dist/release-docs/LICENSE | 52 +-
seatunnel-dist/release-docs/NOTICE | 20 +
.../release-docs/licenses/LICENSE-bouncycastle.txt | 7 +
.../release-docs/licenses/LICENSE-jaxrs-api.txt | 637 +++++++++++++++++++++
seatunnel-dist/src/main/assembly/assembly-bin.xml | 48 +-
seatunnel-e2e/pom.xml | 4 +-
.../apache/seatunnel/e2e/flink/FlinkContainer.java | 2 +-
.../seatunnel-flink-new-connector-e2e}/pom.xml | 27 +-
.../apache/seatunnel/e2e/flink/FlinkContainer.java | 44 +-
.../e2e/flink/assertion/FakeSourceToAssertIT.java | 24 +-
.../e2e/flink/fake/FakeSourceToConsoleIT.java | 25 +-
.../resources/assertion/fakesource_to_assert.conf | 88 +++
.../test/resources/fake/fakesource_to_console.conf | 35 +-
.../src/test/resources/log4j.properties | 23 +
.../apache/seatunnel/e2e/spark/SparkContainer.java | 2 +-
.../seatunnel-spark-new-connector-e2e}/pom.xml | 28 +-
.../apache/seatunnel/e2e/spark/SparkContainer.java | 21 +-
.../e2e/spark/fake/FakeSourceToConsoleIT.java | 29 +-
.../test/resources/fake/fakesource_to_console.conf | 34 +-
.../src/test/resources/log4j.properties | 22 +
seatunnel-examples/pom.xml | 2 +
.../seatunnel-flink-examples/pom.xml | 6 +
.../seatunnel/example/flink/LocalFlinkExample.java | 3 +-
.../pom.xml | 22 +-
.../example/flink/SeaTunnelApiExample.java} | 15 +-
.../main/resources/examples/fake_to_console.conf | 35 +-
.../src/main/resources/log4j.properties | 22 +
.../seatunnel-flink-sql-examples/pom.xml | 2 +-
.../seatunnel/example/spark/LocalSparkExample.java | 3 +-
.../seatunnel-spark-new-connector-example}/pom.xml | 54 +-
.../example/spark/SeaTunnelApiExample.java} | 33 +-
.../src/main/resources/examples/spark.batch.conf | 33 +-
{seatunnel-e2e => seatunnel-formats}/pom.xml | 8 +-
.../seatunnel-format-json}/pom.xml | 28 +-
.../format/json/JsonDeserializationSchema.java | 122 ++++
.../seatunnel/format/json/JsonToRowConverters.java | 281 +++++++++
.../pom.xml | 28 +-
.../plugin/discovery/AbstractPluginDiscovery.java | 181 ++++++
.../plugin/discovery/PluginDiscovery.java | 70 +++
.../plugin/discovery/PluginIdentifier.java | 89 +++
.../discovery/flink/FlinkSinkPluginDiscovery.java | 17 +-
.../flink/FlinkSourcePluginDiscovery.java | 18 +-
.../flink/FlinkTransformPluginDiscovery.java | 40 +-
.../SeaTunnelFlinkTransformPluginDiscovery.java | 29 +-
.../seatunnel/SeaTunnelSinkPluginDiscovery.java | 17 +-
.../seatunnel/SeaTunnelSourcePluginDiscovery.java | 18 +-
.../SeaTunnelSparkTransformPluginDiscovery.java | 18 +-
.../discovery/spark/SparkSinkPluginDiscovery.java | 17 +-
.../spark/SparkSourcePluginDiscovery.java | 17 +-
.../spark/SparkTransformPluginDiscovery.java | 39 +-
{seatunnel-e2e => seatunnel-translation}/pom.xml | 8 +-
.../seatunnel-translation-base}/pom.xml | 18 +-
.../translation/serialization/RowConverter.java | 128 +++++
.../serialization/SerializerConverter.java | 20 +-
.../sink/SinkAggregatedCommitterConverter.java | 19 +-
.../translation/sink/SinkCommitterConverter.java | 20 +-
.../seatunnel/translation/sink/SinkConverter.java | 22 +-
.../translation/sink/SinkWriterConverter.java | 14 +-
.../translation/source/BaseSourceFunction.java | 24 +-
.../source/CoordinatedEnumeratorContext.java | 60 ++
.../source/CoordinatedReaderContext.java | 62 ++
.../translation/source/CoordinatedSource.java | 273 +++++++++
.../source/ParallelEnumeratorContext.java | 77 +++
.../translation/source/ParallelReaderContext.java | 63 ++
.../translation/source/ParallelSource.java | 201 +++++++
.../util/ThreadPoolExecutorFactory.java | 31 +-
.../seatunnel-translation-flink}/pom.xml | 45 +-
.../serialization/CommitWrapperSerializer.java | 66 +++
.../flink/serialization/FlinkRowConverter.java | 115 ++++
.../FlinkSimpleVersionedSerializer.java | 33 +-
.../serialization/FlinkWriterStateSerializer.java | 68 +++
.../flink/serialization/KryoTypeInfo.java | 41 ++
.../flink/serialization/WrappedRow.java | 40 +-
.../translation/flink/sink/CommitWrapper.java | 16 +-
.../translation/flink/sink/FlinkCommitter.java | 56 ++
.../flink/sink/FlinkGlobalCommitter.java | 72 +++
.../translation/flink/sink/FlinkSink.java | 90 +++
.../translation/flink/sink/FlinkSinkWriter.java | 75 +++
.../translation/flink/sink/FlinkWriterState.java | 33 +-
.../flink/source/BaseSeaTunnelSourceFunction.java | 136 +++++
.../translation/flink/source/RowCollector.java | 55 ++
.../flink/source/SeaTunnelCoordinatedSource.java | 43 ++
.../flink/source/SeaTunnelParallelSource.java | 49 ++
.../flink/utils/TypeConverterUtils.java | 146 +++++
.../flink/utils/TypeConverterUtilsTest.java | 129 +++++
.../seatunnel-translation-spark}/pom.xml | 44 +-
.../spark/serialization/InternalRowConverter.java | 178 ++++++
.../spark/sink/SparkDataSourceWriter.java | 95 +++
.../translation/spark/sink/SparkDataWriter.java | 101 ++++
.../spark/sink/SparkDataWriterFactory.java | 62 ++
.../translation/spark/sink/SparkSink.java | 76 +++
.../translation/spark/sink/SparkSinkInjector.java | 49 ++
.../translation/spark/sink/SparkStreamWriter.java | 73 +++
.../spark/sink/SparkWriterCommitMessage.java | 23 +-
.../translation/spark/source/Handover.java | 88 +++
.../spark/source/InternalRowCollector.java | 53 ++
.../translation/spark/source/ReaderState.java | 32 +-
.../spark/source/SeaTunnelSourceSupport.java | 92 +++
.../spark/source/batch/BatchPartition.java | 47 ++
.../spark/source/batch/BatchSourceReader.java | 62 ++
.../batch/CoordinatedBatchPartitionReader.java | 97 ++++
.../source/batch/ParallelBatchPartitionReader.java | 147 +++++
.../source/continnous/ContinuousPartition.java | 53 ++
.../source/continnous/ContinuousSourceReader.java | 116 ++++
.../spark/source/continnous/CoordinationState.java | 49 ++
.../ParallelContinuousPartitionReader.java | 78 +++
.../CoordinatedMicroBatchPartitionReader.java | 138 +++++
.../spark/source/micro/MicroBatchPartition.java | 64 +++
.../spark/source/micro/MicroBatchSourceReader.java | 111 ++++
.../spark/source/micro/MicroBatchState.java | 27 +-
.../micro/ParallelMicroBatchPartitionReader.java | 182 ++++++
.../spark/utils/TypeConverterUtils.java | 173 ++++++
...org.apache.spark.sql.sources.DataSourceRegister | 18 +
tools/dependencies/known-dependencies.txt | 30 +-
528 files changed, 28870 insertions(+), 3208 deletions(-)