You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by fa...@apache.org on 2022/08/01 05:52:35 UTC
[incubator-seatunnel] branch dev updated (4ab3c21b8 -> 263239def)
This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
from 4ab3c21b8 [Feature][Connector-V2-Hive] Add parquet file format support to Hive Sink (#2310)
add 263239def [Feature][seatunnel-translation] Add spark-common module to implement support spark multiple version (#2252)
No new revisions were added by this update.
Summary of changes:
seatunnel-core/seatunnel-core-spark/pom.xml | 8 ++++-
seatunnel-core/seatunnel-spark-starter/pom.xml | 2 +-
.../spark/execution/SinkExecuteProcessor.java | 2 +-
.../spark/execution/SourceExecuteProcessor.java | 2 +-
.../seatunnel-translation-spark/pom.xml | 40 +++-------------------
.../{ => seatunnel-translation-spark-2.4}/pom.xml | 10 +++---
.../spark/sink/SparkDataSourceWriter.java | 0
.../translation/spark/sink/SparkDataWriter.java | 2 +-
.../spark/sink/SparkDataWriterFactory.java | 0
.../translation/spark/sink/SparkSink.java | 0
.../translation/spark/sink/SparkSinkInjector.java | 0
.../translation/spark/sink/SparkStreamWriter.java | 0
.../spark/sink/SparkWriterCommitMessage.java | 0
.../spark/source/SeaTunnelSourceSupport.java | 0
.../source/SeatunnelInputPartitionReader.java} | 27 ++++++++-------
.../spark/source/batch/BatchPartition.java | 9 +++--
.../spark/source/batch/BatchSourceReader.java | 2 +-
.../source/continnous/ContinuousPartition.java | 1 +
.../source/continnous/ContinuousSourceReader.java | 5 +--
.../spark/source/micro/MicroBatchPartition.java | 10 ++++--
.../spark/source/micro/MicroBatchSourceReader.java | 3 +-
...org.apache.spark.sql.sources.DataSourceRegister | 0
.../pom.xml | 11 ++----
.../translation/spark/common}/Handover.java | 2 +-
.../spark/common}/InternalRowCollector.java | 4 +--
.../translation/spark/common}/ReaderState.java | 2 +-
.../serialization/InternalRowConverter.java | 4 +--
.../batch/CoordinatedBatchPartitionReader.java | 4 +--
.../source/batch/ParallelBatchPartitionReader.java | 12 +++----
.../source/continnous/CoordinationState.java | 4 +--
.../ParallelContinuousPartitionReader.java | 6 ++--
.../CoordinatedMicroBatchPartitionReader.java | 6 ++--
.../common}/source/micro/MicroBatchState.java | 2 +-
.../micro/ParallelMicroBatchPartitionReader.java | 6 ++--
.../spark/common}/utils/TypeConverterUtils.java | 2 +-
35 files changed, 86 insertions(+), 102 deletions(-)
copy seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/pom.xml (89%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataSourceWriter.java (100%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataWriter.java (97%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataWriterFactory.java (100%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSink.java (100%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java (100%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkStreamWriter.java (100%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkWriterCommitMessage.java (100%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceSupport.java (100%)
copy seatunnel-translation/{seatunnel-translation-flink/src/main/java/org/apache/seatunnel/translation/flink/serialization/FlinkSimpleVersionedSerializer.java => seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/SeatunnelInputPartitionReader.java} (53%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/batch/BatchPartition.java (75%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/batch/BatchSourceReader.java (96%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/continnous/ContinuousPartition.java (95%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/continnous/ContinuousSourceReader.java (95%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/micro/MicroBatchPartition.java (75%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/micro/MicroBatchSourceReader.java (96%)
rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (100%)
copy seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-common}/pom.xml (90%)
rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark/source => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/Handover.java (98%)
rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark/source => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/InternalRowCollector.java (92%)
rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark/source => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/ReaderState.java (96%)
rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/serialization/InternalRowConverter.java (97%)
rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/batch/CoordinatedBatchPartitionReader.java (96%)
rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/batch/ParallelBatchPartitionReader.java (92%)
rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/continnous/CoordinationState.java (92%)
rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/continnous/ParallelContinuousPartitionReader.java (92%)
rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/micro/CoordinatedMicroBatchPartitionReader.java (96%)
rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/micro/MicroBatchState.java (95%)
rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/micro/ParallelMicroBatchPartitionReader.java (96%)
rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/utils/TypeConverterUtils.java (99%)