You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by fa...@apache.org on 2022/08/01 05:52:35 UTC

[incubator-seatunnel] branch dev updated (4ab3c21b8 -> 263239def)

This is an automated email from the ASF dual-hosted git repository.

fanjia pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git


    from 4ab3c21b8 [Feature][Connector-V2-Hive] Add parquet file format support to Hive Sink (#2310)
     add 263239def [Feature][seatunnel-translation] Add spark-common module to implement support spark multiple version (#2252)

No new revisions were added by this update.

Summary of changes:
 seatunnel-core/seatunnel-core-spark/pom.xml        |  8 ++++-
 seatunnel-core/seatunnel-spark-starter/pom.xml     |  2 +-
 .../spark/execution/SinkExecuteProcessor.java      |  2 +-
 .../spark/execution/SourceExecuteProcessor.java    |  2 +-
 .../seatunnel-translation-spark/pom.xml            | 40 +++-------------------
 .../{ => seatunnel-translation-spark-2.4}/pom.xml  | 10 +++---
 .../spark/sink/SparkDataSourceWriter.java          |  0
 .../translation/spark/sink/SparkDataWriter.java    |  2 +-
 .../spark/sink/SparkDataWriterFactory.java         |  0
 .../translation/spark/sink/SparkSink.java          |  0
 .../translation/spark/sink/SparkSinkInjector.java  |  0
 .../translation/spark/sink/SparkStreamWriter.java  |  0
 .../spark/sink/SparkWriterCommitMessage.java       |  0
 .../spark/source/SeaTunnelSourceSupport.java       |  0
 .../source/SeatunnelInputPartitionReader.java}     | 27 ++++++++-------
 .../spark/source/batch/BatchPartition.java         |  9 +++--
 .../spark/source/batch/BatchSourceReader.java      |  2 +-
 .../source/continnous/ContinuousPartition.java     |  1 +
 .../source/continnous/ContinuousSourceReader.java  |  5 +--
 .../spark/source/micro/MicroBatchPartition.java    | 10 ++++--
 .../spark/source/micro/MicroBatchSourceReader.java |  3 +-
 ...org.apache.spark.sql.sources.DataSourceRegister |  0
 .../pom.xml                                        | 11 ++----
 .../translation/spark/common}/Handover.java        |  2 +-
 .../spark/common}/InternalRowCollector.java        |  4 +--
 .../translation/spark/common}/ReaderState.java     |  2 +-
 .../serialization/InternalRowConverter.java        |  4 +--
 .../batch/CoordinatedBatchPartitionReader.java     |  4 +--
 .../source/batch/ParallelBatchPartitionReader.java | 12 +++----
 .../source/continnous/CoordinationState.java       |  4 +--
 .../ParallelContinuousPartitionReader.java         |  6 ++--
 .../CoordinatedMicroBatchPartitionReader.java      |  6 ++--
 .../common}/source/micro/MicroBatchState.java      |  2 +-
 .../micro/ParallelMicroBatchPartitionReader.java   |  6 ++--
 .../spark/common}/utils/TypeConverterUtils.java    |  2 +-
 35 files changed, 86 insertions(+), 102 deletions(-)
 copy seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/pom.xml (89%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataSourceWriter.java (100%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataWriter.java (97%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataWriterFactory.java (100%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSink.java (100%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java (100%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkStreamWriter.java (100%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkWriterCommitMessage.java (100%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceSupport.java (100%)
 copy seatunnel-translation/{seatunnel-translation-flink/src/main/java/org/apache/seatunnel/translation/flink/serialization/FlinkSimpleVersionedSerializer.java => seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/SeatunnelInputPartitionReader.java} (53%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/batch/BatchPartition.java (75%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/batch/BatchSourceReader.java (96%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/continnous/ContinuousPartition.java (95%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/continnous/ContinuousSourceReader.java (95%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/micro/MicroBatchPartition.java (75%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/java/org/apache/seatunnel/translation/spark/source/micro/MicroBatchSourceReader.java (96%)
 rename seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-2.4}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (100%)
 copy seatunnel-translation/seatunnel-translation-spark/{ => seatunnel-translation-spark-common}/pom.xml (90%)
 rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark/source => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/Handover.java (98%)
 rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark/source => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/InternalRowCollector.java (92%)
 rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark/source => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/ReaderState.java (96%)
 rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/serialization/InternalRowConverter.java (97%)
 rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/batch/CoordinatedBatchPartitionReader.java (96%)
 rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/batch/ParallelBatchPartitionReader.java (92%)
 rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/continnous/CoordinationState.java (92%)
 rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/continnous/ParallelContinuousPartitionReader.java (92%)
 rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/micro/CoordinatedMicroBatchPartitionReader.java (96%)
 rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/micro/MicroBatchState.java (95%)
 rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/source/micro/ParallelMicroBatchPartitionReader.java (96%)
 rename seatunnel-translation/seatunnel-translation-spark/{src/main/java/org/apache/seatunnel/translation/spark => seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common}/utils/TypeConverterUtils.java (99%)