You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by fa...@apache.org on 2024/08/14 06:13:46 UTC
(seatunnel) branch dev updated: [Feature][Spark] Support multiple tables read and write (#7283)
This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new ff107ac6d2 [Feature][Spark] Support multiple tables read and write (#7283)
ff107ac6d2 is described below
commit ff107ac6d252ba4b50b47b5d014b1400c9679915
Author: Carl-Zhou-CN <10...@qq.com>
AuthorDate: Wed Aug 14 14:13:41 2024 +0800
[Feature][Spark] Support multiple tables read and write (#7283)
---
.../core/starter/execution/PluginUtil.java | 98 ++-
.../spark/execution/SinkExecuteProcessor.java | 51 +-
.../spark/execution/SinkExecuteProcessor.java | 52 +-
.../spark/execution/SourceExecuteProcessor.java | 7 +-
.../SparkAbstractPluginExecuteProcessor.java | 3 +-
.../starter/spark/execution/SparkExecution.java | 1 +
.../spark/execution/TransformExecuteProcessor.java | 17 +-
.../clickhouse/ClickhouseSinkCDCChangelogIT.java | 6 -
.../seatunnel/e2e/connector/druid/DruidIT.java | 4 +-
.../connector/elasticsearch/ElasticsearchIT.java | 4 +-
.../e2e/connector/fake/FakeWithMultiTableTT.java | 2 +-
.../file/local/LocalFileWithMultipleTableIT.java | 7 +-
.../seatunnel/e2e/connector/http/HttpIT.java | 8 +-
.../e2e/connector/influxdb/InfluxdbIT.java | 4 +-
.../seatunnel/jdbc/JdbcMysqlMultipleTablesIT.java | 2 +-
.../seatunnel/e2e/connector/kudu/KuduIT.java | 8 +-
.../seatunnel/e2e/connector/redis/RedisIT.java | 4 +-
.../translation/spark/sink/SparkSink.java | 10 +-
.../translation/spark/sink/SparkSinkInjector.java | 8 +-
.../spark/sink/writer/SparkDataSourceWriter.java | 8 +-
.../spark/sink/writer/SparkDataWriter.java | 25 +-
.../spark/sink/writer/SparkDataWriterFactory.java | 9 +-
.../spark/sink/writer/SparkStreamWriter.java | 4 +-
.../spark/source/SeaTunnelSourceSupport.java | 33 +-
.../source/partition/batch/BatchPartition.java | 11 +-
.../partition/micro/MicroBatchPartition.java | 13 +-
.../source/reader/batch/BatchSourceReader.java | 21 +-
.../batch/CoordinatedBatchPartitionReader.java | 9 +-
.../reader/batch/ParallelBatchPartitionReader.java | 11 +-
.../CoordinatedMicroBatchPartitionReader.java | 10 +-
.../reader/micro/MicroBatchSourceReader.java | 15 +-
.../micro/ParallelMicroBatchPartitionReader.java | 6 +-
.../spark/sink/SeaTunnelBatchWrite.java | 8 +-
.../translation/spark/sink/SeaTunnelSinkTable.java | 10 +-
.../translation/spark/sink/SparkSinkInjector.java | 10 +-
.../spark/sink/write/SeaTunnelSparkDataWriter.java | 25 +-
.../write/SeaTunnelSparkDataWriterFactory.java | 9 +-
.../spark/sink/write/SeaTunnelWrite.java | 10 +-
.../spark/sink/write/SeaTunnelWriteBuilder.java | 8 +-
.../spark/source/SeaTunnelSourceTable.java | 22 +-
.../batch/CoordinatedBatchPartitionReader.java | 9 +-
.../batch/ParallelBatchPartitionReader.java | 10 +-
.../source/partition/batch/SeaTunnelBatch.java | 10 +-
.../SeaTunnelBatchPartitionReaderFactory.java | 11 +-
.../CoordinatedMicroBatchPartitionReader.java | 10 +-
.../micro/ParallelMicroBatchPartitionReader.java | 6 +-
.../partition/micro/SeaTunnelMicroBatch.java | 14 +-
.../SeaTunnelMicroBatchPartitionReaderFactory.java | 13 +-
.../spark/source/scan/SeaTunnelScan.java | 19 +-
.../spark/source/scan/SeaTunnelScanBuilder.java | 10 +-
.../translation/spark/sink/SparkSinkTest.java | 19 +-
.../spark/execution/ColumnWithIndex.java | 42 +-
.../spark/execution/DatasetTableInfo.java | 17 +-
.../translation/spark/execution/IndexQueue.java | 41 +-
.../spark/execution/MultiTableManager.java | 160 ++++
...llector.java => InternalMultiRowCollector.java} | 48 +-
.../spark/serialization/InternalRowCollector.java | 19 +-
.../spark/serialization/InternalRowConverter.java | 56 +-
.../spark/serialization/SeaTunnelRowConverter.java | 33 +-
.../spark/utils/TypeConverterUtils.java | 17 +-
.../spark/execution/MultiTableManagerTest.java | 950 +++++++++++++++++++++
61 files changed, 1743 insertions(+), 344 deletions(-)
diff --git a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/execution/PluginUtil.java b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/execution/PluginUtil.java
index c47ea0b121..e8a682062e 100644
--- a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/execution/PluginUtil.java
+++ b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/execution/PluginUtil.java
@@ -23,17 +23,21 @@ import org.apache.seatunnel.api.common.CommonOptions;
import org.apache.seatunnel.api.common.JobContext;
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
import org.apache.seatunnel.api.configuration.util.ConfigValidator;
+import org.apache.seatunnel.api.sink.SeaTunnelSink;
import org.apache.seatunnel.api.source.SeaTunnelSource;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
import org.apache.seatunnel.api.table.connector.TableSource;
import org.apache.seatunnel.api.table.factory.Factory;
import org.apache.seatunnel.api.table.factory.FactoryException;
+import org.apache.seatunnel.api.table.factory.FactoryUtil;
+import org.apache.seatunnel.api.table.factory.TableSinkFactory;
+import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext;
import org.apache.seatunnel.api.table.factory.TableSourceFactory;
import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.common.constants.JobMode;
-import org.apache.seatunnel.common.utils.SeaTunnelException;
+import org.apache.seatunnel.core.starter.enums.PluginType;
import org.apache.seatunnel.plugin.discovery.PluginIdentifier;
import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelFactoryDiscovery;
import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelSinkPluginDiscovery;
@@ -43,7 +47,9 @@ import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelTransformPluginD
import com.google.common.collect.Lists;
import java.net.URL;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import static org.apache.seatunnel.api.common.CommonOptions.PLUGIN_NAME;
@@ -97,10 +103,11 @@ public class PluginUtil {
CatalogTableUtil.convertDataTypeToCatalogTables(seaTunnelDataType, tableId);
}
- if (catalogTables.size() != 1) {
- throw new SeaTunnelException(
- String.format("Unsupported table number: %d on flink", catalogTables.size()));
- }
+ // if (catalogTables.size() != 1) {
+ // throw new SeaTunnelException(
+ // String.format("Unsupported table number: %d on flink",
+ // catalogTables.size()));
+ // }
return new SourceTableInfo(source, catalogTables);
}
@@ -162,6 +169,87 @@ public class PluginUtil {
}
}
+ public static SeaTunnelSink createSink(
+ Optional<? extends Factory> factory,
+ Config sinkConfig,
+ SeaTunnelSinkPluginDiscovery sinkPluginDiscovery,
+ JobContext jobContext,
+ List<CatalogTable> catalogTables,
+ ClassLoader classLoader) {
+ boolean fallBack = !factory.isPresent() || isFallback(factory.get());
+ if (fallBack) {
+ SeaTunnelSink sink =
+ fallbackCreateSink(
+ sinkPluginDiscovery,
+ PluginIdentifier.of(
+ ENGINE_TYPE,
+ PluginType.SINK.getType(),
+ sinkConfig.getString(PLUGIN_NAME.key())),
+ sinkConfig);
+ sink.setJobContext(jobContext);
+ sink.setTypeInfo(catalogTables.get(0).getSeaTunnelRowType());
+ return sink;
+ } else {
+ if (catalogTables.size() > 1) {
+ Map<String, SeaTunnelSink> sinks = new HashMap<>();
+ ReadonlyConfig readonlyConfig = ReadonlyConfig.fromConfig(sinkConfig);
+ catalogTables.forEach(
+ catalogTable -> {
+ TableSinkFactoryContext context =
+ TableSinkFactoryContext.replacePlaceholderAndCreate(
+ catalogTable,
+ ReadonlyConfig.fromConfig(sinkConfig),
+ classLoader,
+ ((TableSinkFactory) factory.get())
+ .excludeTablePlaceholderReplaceKeys());
+ ConfigValidator.of(context.getOptions())
+ .validate(factory.get().optionRule());
+ SeaTunnelSink action =
+ ((TableSinkFactory) factory.get())
+ .createSink(context)
+ .createSink();
+ action.setJobContext(jobContext);
+ sinks.put(catalogTable.getTablePath().toString(), action);
+ });
+ return FactoryUtil.createMultiTableSink(sinks, readonlyConfig, classLoader);
+ }
+ TableSinkFactoryContext context =
+ TableSinkFactoryContext.replacePlaceholderAndCreate(
+ catalogTables.get(0),
+ ReadonlyConfig.fromConfig(sinkConfig),
+ classLoader,
+ ((TableSinkFactory) factory.get())
+ .excludeTablePlaceholderReplaceKeys());
+ ConfigValidator.of(context.getOptions()).validate(factory.get().optionRule());
+ SeaTunnelSink sink =
+ ((TableSinkFactory) factory.get()).createSink(context).createSink();
+ sink.setJobContext(jobContext);
+ return sink;
+ }
+ }
+
+ public static boolean isFallback(Factory factory) {
+ try {
+ ((TableSinkFactory) factory).createSink(null);
+ } catch (Exception e) {
+ if (e instanceof UnsupportedOperationException
+ && "The Factory has not been implemented and the deprecated Plugin will be used."
+ .equals(e.getMessage())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static SeaTunnelSink fallbackCreateSink(
+ SeaTunnelSinkPluginDiscovery sinkPluginDiscovery,
+ PluginIdentifier pluginIdentifier,
+ Config pluginConfig) {
+ SeaTunnelSink source = sinkPluginDiscovery.createPluginInstance(pluginIdentifier);
+ source.prepare(pluginConfig);
+ return source;
+ }
+
public static void ensureJobModeMatch(JobContext jobContext, SeaTunnelSource source) {
if (jobContext.getJobMode() == JobMode.BATCH
&& source.getBoundedness()
diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SinkExecuteProcessor.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SinkExecuteProcessor.java
index 7751286b22..0f3fc75096 100644
--- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SinkExecuteProcessor.java
+++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SinkExecuteProcessor.java
@@ -21,17 +21,13 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.api.common.CommonOptions;
import org.apache.seatunnel.api.common.JobContext;
-import org.apache.seatunnel.api.configuration.ReadonlyConfig;
-import org.apache.seatunnel.api.configuration.util.ConfigValidator;
import org.apache.seatunnel.api.sink.SaveModeExecuteWrapper;
import org.apache.seatunnel.api.sink.SaveModeHandler;
import org.apache.seatunnel.api.sink.SeaTunnelSink;
import org.apache.seatunnel.api.sink.SupportSaveMode;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.seatunnel.api.table.factory.Factory;
import org.apache.seatunnel.api.table.factory.TableSinkFactory;
-import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext;
-import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
-import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException;
import org.apache.seatunnel.core.starter.enums.PluginType;
import org.apache.seatunnel.core.starter.exception.TaskExecuteException;
@@ -39,6 +35,7 @@ import org.apache.seatunnel.core.starter.execution.PluginUtil;
import org.apache.seatunnel.plugin.discovery.PluginIdentifier;
import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelFactoryDiscovery;
import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelSinkPluginDiscovery;
+import org.apache.seatunnel.translation.spark.execution.DatasetTableInfo;
import org.apache.seatunnel.translation.spark.sink.SparkSinkInjector;
import org.apache.spark.sql.Dataset;
@@ -50,7 +47,6 @@ import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
-import static org.apache.seatunnel.api.common.CommonOptions.PLUGIN_NAME;
import static org.apache.seatunnel.api.common.SeaTunnelAPIErrorCode.HANDLE_SAVE_MODE_FAILED;
public class SinkExecuteProcessor
@@ -97,8 +93,6 @@ public class SinkExecuteProcessor
DatasetTableInfo datasetTableInfo =
fromSourceTable(sinkConfig, sparkRuntimeEnvironment, upstreamDataStreams)
.orElse(input);
- SeaTunnelDataType<?> inputType =
- datasetTableInfo.getCatalogTable().getSeaTunnelRowType();
Dataset<Row> dataset = datasetTableInfo.getDataset();
int parallelism;
@@ -114,31 +108,14 @@ public class SinkExecuteProcessor
}
dataset.sparkSession().read().option(CommonOptions.PARALLELISM.key(), parallelism);
Optional<? extends Factory> factory = plugins.get(i);
- boolean fallBack = !factory.isPresent() || isFallback(factory.get());
- SeaTunnelSink sink;
- if (fallBack) {
- sink =
- fallbackCreateSink(
- sinkPluginDiscovery,
- PluginIdentifier.of(
- ENGINE_TYPE,
- PLUGIN_TYPE,
- sinkConfig.getString(PLUGIN_NAME.key())),
- sinkConfig);
- sink.setJobContext(jobContext);
- sink.setTypeInfo((SeaTunnelRowType) inputType);
- } else {
- TableSinkFactoryContext context =
- TableSinkFactoryContext.replacePlaceholderAndCreate(
- datasetTableInfo.getCatalogTable(),
- ReadonlyConfig.fromConfig(sinkConfig),
- classLoader,
- ((TableSinkFactory) factory.get())
- .excludeTablePlaceholderReplaceKeys());
- ConfigValidator.of(context.getOptions()).validate(factory.get().optionRule());
- sink = ((TableSinkFactory) factory.get()).createSink(context).createSink();
- sink.setJobContext(jobContext);
- }
+ SeaTunnelSink sink =
+ PluginUtil.createSink(
+ factory,
+ sinkConfig,
+ sinkPluginDiscovery,
+ jobContext,
+ datasetTableInfo.getCatalogTables(),
+ classLoader);
// TODO modify checkpoint location
if (SupportSaveMode.class.isAssignableFrom(sink.getClass())) {
SupportSaveMode saveModeSink = (SupportSaveMode) sink;
@@ -153,11 +130,9 @@ public class SinkExecuteProcessor
}
String applicationId =
sparkRuntimeEnvironment.getSparkSession().sparkContext().applicationId();
- SparkSinkInjector.inject(
- dataset.write(),
- sink,
- datasetTableInfo.getCatalogTable(),
- applicationId)
+ CatalogTable[] catalogTables =
+ datasetTableInfo.getCatalogTables().toArray(new CatalogTable[0]);
+ SparkSinkInjector.inject(dataset.write(), sink, catalogTables, applicationId)
.option("checkpointLocation", "/tmp")
.save();
}
diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SinkExecuteProcessor.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SinkExecuteProcessor.java
index 46b3233b00..29ff90cfb8 100644
--- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SinkExecuteProcessor.java
+++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SinkExecuteProcessor.java
@@ -21,17 +21,13 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.api.common.CommonOptions;
import org.apache.seatunnel.api.common.JobContext;
-import org.apache.seatunnel.api.configuration.ReadonlyConfig;
-import org.apache.seatunnel.api.configuration.util.ConfigValidator;
import org.apache.seatunnel.api.sink.SaveModeExecuteWrapper;
import org.apache.seatunnel.api.sink.SaveModeHandler;
import org.apache.seatunnel.api.sink.SeaTunnelSink;
import org.apache.seatunnel.api.sink.SupportSaveMode;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.seatunnel.api.table.factory.Factory;
import org.apache.seatunnel.api.table.factory.TableSinkFactory;
-import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext;
-import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
-import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException;
import org.apache.seatunnel.core.starter.enums.PluginType;
import org.apache.seatunnel.core.starter.exception.TaskExecuteException;
@@ -39,6 +35,7 @@ import org.apache.seatunnel.core.starter.execution.PluginUtil;
import org.apache.seatunnel.plugin.discovery.PluginIdentifier;
import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelFactoryDiscovery;
import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelSinkPluginDiscovery;
+import org.apache.seatunnel.translation.spark.execution.DatasetTableInfo;
import org.apache.seatunnel.translation.spark.sink.SparkSinkInjector;
import org.apache.spark.sql.Dataset;
@@ -51,7 +48,6 @@ import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
-import static org.apache.seatunnel.api.common.CommonOptions.PLUGIN_NAME;
import static org.apache.seatunnel.api.common.SeaTunnelAPIErrorCode.HANDLE_SAVE_MODE_FAILED;
public class SinkExecuteProcessor
@@ -98,10 +94,7 @@ public class SinkExecuteProcessor
DatasetTableInfo datasetTableInfo =
fromSourceTable(sinkConfig, sparkRuntimeEnvironment, upstreamDataStreams)
.orElse(input);
- SeaTunnelDataType<?> inputType =
- datasetTableInfo.getCatalogTable().getSeaTunnelRowType();
Dataset<Row> dataset = datasetTableInfo.getDataset();
-
int parallelism;
if (sinkConfig.hasPath(CommonOptions.PARALLELISM.key())) {
parallelism = sinkConfig.getInt(CommonOptions.PARALLELISM.key());
@@ -115,31 +108,14 @@ public class SinkExecuteProcessor
}
dataset.sparkSession().read().option(CommonOptions.PARALLELISM.key(), parallelism);
Optional<? extends Factory> factory = plugins.get(i);
- boolean fallBack = !factory.isPresent() || isFallback(factory.get());
- SeaTunnelSink sink;
- if (fallBack) {
- sink =
- fallbackCreateSink(
- sinkPluginDiscovery,
- PluginIdentifier.of(
- ENGINE_TYPE,
- PLUGIN_TYPE,
- sinkConfig.getString(PLUGIN_NAME.key())),
- sinkConfig);
- sink.setJobContext(jobContext);
- sink.setTypeInfo((SeaTunnelRowType) inputType);
- } else {
- TableSinkFactoryContext context =
- TableSinkFactoryContext.replacePlaceholderAndCreate(
- datasetTableInfo.getCatalogTable(),
- ReadonlyConfig.fromConfig(sinkConfig),
- classLoader,
- ((TableSinkFactory) factory.get())
- .excludeTablePlaceholderReplaceKeys());
- ConfigValidator.of(context.getOptions()).validate(factory.get().optionRule());
- sink = ((TableSinkFactory) factory.get()).createSink(context).createSink();
- sink.setJobContext(jobContext);
- }
+ SeaTunnelSink sink =
+ PluginUtil.createSink(
+ factory,
+ sinkConfig,
+ sinkPluginDiscovery,
+ jobContext,
+ datasetTableInfo.getCatalogTables(),
+ classLoader);
// TODO modify checkpoint location
if (SupportSaveMode.class.isAssignableFrom(sink.getClass())) {
SupportSaveMode saveModeSink = (SupportSaveMode) sink;
@@ -154,11 +130,9 @@ public class SinkExecuteProcessor
}
String applicationId =
sparkRuntimeEnvironment.getStreamingContext().sparkContext().applicationId();
- SparkSinkInjector.inject(
- dataset.write(),
- sink,
- datasetTableInfo.getCatalogTable(),
- applicationId)
+ CatalogTable[] catalogTables =
+ datasetTableInfo.getCatalogTables().toArray(new CatalogTable[0]);
+ SparkSinkInjector.inject(dataset.write(), sink, catalogTables, applicationId)
.option("checkpointLocation", "/tmp")
.mode(SaveMode.Append)
.save();
diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SourceExecuteProcessor.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SourceExecuteProcessor.java
index cf81ab60a0..d4a4038e48 100644
--- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SourceExecuteProcessor.java
+++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SourceExecuteProcessor.java
@@ -31,11 +31,10 @@ import org.apache.seatunnel.core.starter.execution.SourceTableInfo;
import org.apache.seatunnel.plugin.discovery.PluginIdentifier;
import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelFactoryDiscovery;
import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelSourcePluginDiscovery;
-import org.apache.seatunnel.translation.spark.utils.TypeConverterUtils;
+import org.apache.seatunnel.translation.spark.execution.DatasetTableInfo;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.types.StructType;
import com.google.common.collect.Lists;
@@ -87,7 +86,6 @@ public class SourceExecuteProcessor extends SparkAbstractPluginExecuteProcessor<
CommonOptions.PARALLELISM.key(),
CommonOptions.PARALLELISM.defaultValue());
}
- StructType schema = (StructType) TypeConverterUtils.convert(source.getProducedType());
Dataset<Row> dataset =
sparkRuntimeEnvironment
.getSparkSession()
@@ -98,12 +96,11 @@ public class SourceExecuteProcessor extends SparkAbstractPluginExecuteProcessor<
Constants.SOURCE_SERIALIZATION,
SerializationUtils.objectToString(source))
.options(envOption)
- .schema(schema)
.load();
sources.add(
new DatasetTableInfo(
dataset,
- sourceTableInfo.getCatalogTables().get(0),
+ sourceTableInfo.getCatalogTables(),
pluginConfig.hasPath(RESULT_TABLE_NAME.key())
? pluginConfig.getString(RESULT_TABLE_NAME.key())
: null));
diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkAbstractPluginExecuteProcessor.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkAbstractPluginExecuteProcessor.java
index a431f23dd1..f1c138285a 100644
--- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkAbstractPluginExecuteProcessor.java
+++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkAbstractPluginExecuteProcessor.java
@@ -22,6 +22,7 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.api.common.JobContext;
import org.apache.seatunnel.common.utils.SeaTunnelException;
import org.apache.seatunnel.core.starter.execution.PluginExecuteProcessor;
+import org.apache.seatunnel.translation.spark.execution.DatasetTableInfo;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
@@ -84,7 +85,7 @@ public abstract class SparkAbstractPluginExecuteProcessor<T>
return Optional.of(
new DatasetTableInfo(
sparkRuntimeEnvironment.getSparkSession().read().table(sourceTableName),
- datasetTableInfo.getCatalogTable(),
+ datasetTableInfo.getCatalogTables(),
sourceTableName));
}
diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkExecution.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkExecution.java
index 48aa9abb2a..55c145bc3b 100644
--- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkExecution.java
+++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkExecution.java
@@ -26,6 +26,7 @@ import org.apache.seatunnel.core.starter.exception.TaskExecuteException;
import org.apache.seatunnel.core.starter.execution.PluginExecuteProcessor;
import org.apache.seatunnel.core.starter.execution.RuntimeEnvironment;
import org.apache.seatunnel.core.starter.execution.TaskExecution;
+import org.apache.seatunnel.translation.spark.execution.DatasetTableInfo;
import lombok.extern.slf4j.Slf4j;
diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/TransformExecuteProcessor.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/TransformExecuteProcessor.java
index fc4a9e00d0..43ef1afcee 100644
--- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/TransformExecuteProcessor.java
+++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/TransformExecuteProcessor.java
@@ -31,6 +31,7 @@ import org.apache.seatunnel.core.starter.exception.TaskExecuteException;
import org.apache.seatunnel.core.starter.execution.PluginUtil;
import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelFactoryDiscovery;
import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelTransformPluginDiscovery;
+import org.apache.seatunnel.translation.spark.execution.DatasetTableInfo;
import org.apache.seatunnel.translation.spark.serialization.SeaTunnelRowConverter;
import org.apache.seatunnel.translation.spark.utils.TypeConverterUtils;
@@ -110,7 +111,7 @@ public class TransformExecuteProcessor
TableTransformFactory factory = plugins.get(i);
TableTransformFactoryContext context =
new TableTransformFactoryContext(
- Collections.singletonList(dataset.getCatalogTable()),
+ dataset.getCatalogTables(),
ReadonlyConfig.fromConfig(pluginConfig),
classLoader);
ConfigValidator.of(context.getOptions()).validate(factory.optionRule());
@@ -121,7 +122,7 @@ public class TransformExecuteProcessor
upstreamDataStreams.add(
new DatasetTableInfo(
inputDataset,
- transform.getProducedCatalogTable(),
+ Collections.singletonList(transform.getProducedCatalogTable()),
pluginConfig.hasPath(RESULT_TABLE_NAME.key())
? pluginConfig.getString(RESULT_TABLE_NAME.key())
: null));
@@ -138,10 +139,11 @@ public class TransformExecuteProcessor
private Dataset<Row> sparkTransform(SeaTunnelTransform transform, DatasetTableInfo tableInfo) {
Dataset<Row> stream = tableInfo.getDataset();
- SeaTunnelDataType<?> inputDataType = tableInfo.getCatalogTable().getSeaTunnelRowType();
+ SeaTunnelDataType<?> inputDataType =
+ tableInfo.getCatalogTables().get(0).getSeaTunnelRowType();
SeaTunnelDataType<?> outputDataTYpe =
transform.getProducedCatalogTable().getSeaTunnelRowType();
- StructType outputSchema = (StructType) TypeConverterUtils.convert(outputDataTYpe);
+ StructType outputSchema = (StructType) TypeConverterUtils.parcel(outputDataTYpe);
SeaTunnelRowConverter inputRowConverter = new SeaTunnelRowConverter(inputDataType);
SeaTunnelRowConverter outputRowConverter = new SeaTunnelRowConverter(outputDataTYpe);
ExpressionEncoder<Row> encoder = RowEncoder.apply(outputSchema);
@@ -187,15 +189,12 @@ public class TransformExecuteProcessor
public Row next() {
try {
Row row = sourceIterator.next();
- SeaTunnelRow seaTunnelRow =
- inputRowConverter.reconvert(
- new SeaTunnelRow(((GenericRowWithSchema) row).values()));
+ SeaTunnelRow seaTunnelRow = inputRowConverter.unpack((GenericRowWithSchema) row);
seaTunnelRow = (SeaTunnelRow) transform.map(seaTunnelRow);
if (seaTunnelRow == null) {
return null;
}
- seaTunnelRow = outputRowConverter.convert(seaTunnelRow);
- return new GenericRowWithSchema(seaTunnelRow.getFields(), structType);
+ return outputRowConverter.parcel(seaTunnelRow);
} catch (Exception e) {
throw new TaskExecuteException("Row convert failed, caused: " + e.getMessage(), e);
}
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/ClickhouseSinkCDCChangelogIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/ClickhouseSinkCDCChangelogIT.java
index 4c2b9cedb1..3d46ac8c55 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/ClickhouseSinkCDCChangelogIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/ClickhouseSinkCDCChangelogIT.java
@@ -19,9 +19,7 @@ package org.apache.seatunnel.connectors.seatunnel.clickhouse;
import org.apache.seatunnel.e2e.common.TestResource;
import org.apache.seatunnel.e2e.common.TestSuiteBase;
-import org.apache.seatunnel.e2e.common.container.EngineType;
import org.apache.seatunnel.e2e.common.container.TestContainer;
-import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.AfterAll;
@@ -50,10 +48,6 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-@DisabledOnContainer(
- value = {},
- type = {EngineType.SPARK},
- disabledReason = "Spark engine will lose the row kind of record")
@Slf4j
public class ClickhouseSinkCDCChangelogIT extends TestSuiteBase implements TestResource {
private static final String CLICKHOUSE_DOCKER_IMAGE = "clickhouse/clickhouse-server:23.3.13.6";
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-druid-e2e/src/test/java/org/apache/seatunnel/e2e/connector/druid/DruidIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-druid-e2e/src/test/java/org/apache/seatunnel/e2e/connector/druid/DruidIT.java
index 21bf3c5d5c..21463e4917 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-druid-e2e/src/test/java/org/apache/seatunnel/e2e/connector/druid/DruidIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-druid-e2e/src/test/java/org/apache/seatunnel/e2e/connector/druid/DruidIT.java
@@ -119,8 +119,8 @@ public class DruidIT extends TestSuiteBase implements TestResource {
@DisabledOnContainer(
value = {},
- type = {EngineType.SPARK, EngineType.FLINK},
- disabledReason = "Currently SPARK/FLINK do not support multiple table read")
+ type = {EngineType.FLINK},
+ disabledReason = "Currently FLINK do not support multiple table read")
@TestTemplate
public void testDruidMultiSink(TestContainer container) throws Exception {
Container.ExecResult execResult =
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-elasticsearch-e2e/src/test/java/org/apache/seatunnel/e2e/connector/elasticsearch/ElasticsearchIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-elasticsearch-e2e/src/test/java/org/apache/seatunnel/e2e/connector/elasticsearch/ElasticsearchIT.java
index 623dd9d221..375a6cc049 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-elasticsearch-e2e/src/test/java/org/apache/seatunnel/e2e/connector/elasticsearch/ElasticsearchIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-elasticsearch-e2e/src/test/java/org/apache/seatunnel/e2e/connector/elasticsearch/ElasticsearchIT.java
@@ -186,8 +186,8 @@ public class ElasticsearchIT extends TestSuiteBase implements TestResource {
@DisabledOnContainer(
value = {},
- type = {EngineType.SPARK, EngineType.FLINK},
- disabledReason = "Currently SPARK/FLINK do not support multiple table read")
+ type = {EngineType.FLINK},
+ disabledReason = "Currently FLINK do not support multiple table read")
@TestTemplate
public void testElasticsearchWithMultiSink(TestContainer container)
throws IOException, InterruptedException {
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-fake-e2e/src/test/java/org/apache/seatunnel/e2e/connector/fake/FakeWithMultiTableTT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-fake-e2e/src/test/java/org/apache/seatunnel/e2e/connector/fake/FakeWithMultiTableTT.java
index 2825f9d7da..798a2a8921 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-fake-e2e/src/test/java/org/apache/seatunnel/e2e/connector/fake/FakeWithMultiTableTT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-fake-e2e/src/test/java/org/apache/seatunnel/e2e/connector/fake/FakeWithMultiTableTT.java
@@ -30,7 +30,7 @@ import java.io.IOException;
@DisabledOnContainer(
value = {},
- type = {EngineType.SPARK, EngineType.FLINK},
+ type = {EngineType.FLINK},
disabledReason = "Currently SPARK and FLINK do not support multi-table")
public class FakeWithMultiTableTT extends TestSuiteBase {
@TestTemplate
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileWithMultipleTableIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileWithMultipleTableIT.java
index 35f29e635f..77e635c2f7 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileWithMultipleTableIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileWithMultipleTableIT.java
@@ -21,6 +21,7 @@ import org.apache.seatunnel.e2e.common.TestSuiteBase;
import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory;
import org.apache.seatunnel.e2e.common.container.EngineType;
import org.apache.seatunnel.e2e.common.container.TestContainer;
+import org.apache.seatunnel.e2e.common.container.TestContainerId;
import org.apache.seatunnel.e2e.common.container.TestHelper;
import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
import org.apache.seatunnel.e2e.common.junit.TestContainerExtension;
@@ -31,9 +32,9 @@ import org.junit.jupiter.api.TestTemplate;
import java.io.IOException;
@DisabledOnContainer(
- value = {},
- type = {EngineType.SPARK, EngineType.FLINK},
- disabledReason = "Currently SPARK and FLINK do not support multi table")
+ value = {TestContainerId.SPARK_2_4},
+ type = {EngineType.FLINK},
+ disabledReason = "Currently FLINK do not support multi table")
public class LocalFileWithMultipleTableIT extends TestSuiteBase {
/** Copy data files to container */
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java
index af1d7125ea..ab8acd1f86 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java
@@ -32,6 +32,7 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.TestTemplate;
import org.mockserver.client.MockServerClient;
+import org.mockserver.model.ClearType;
import org.mockserver.model.Format;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.GenericContainer;
@@ -232,8 +233,8 @@ public class HttpIT extends TestSuiteBase implements TestResource {
@DisabledOnContainer(
value = {},
- type = {EngineType.SPARK, EngineType.FLINK},
- disabledReason = "Currently SPARK/FLINK do not support multiple table read")
+ type = {EngineType.FLINK},
+ disabledReason = "Currently FLINK do not support multiple table read")
@TestTemplate
public void testMultiTableHttp(TestContainer container)
throws IOException, InterruptedException {
@@ -245,6 +246,9 @@ public class HttpIT extends TestSuiteBase implements TestResource {
mockServerClient.retrieveRecordedRequests(
request().withPath("/example/httpMultiTableContentSink").withMethod("POST"),
Format.JSON);
+ mockServerClient.clear(
+ request().withPath("/example/httpMultiTableContentSink").withMethod("POST"),
+ ClearType.LOG);
List<Record> recordResponse =
objectMapper.readValue(mockResponse, new TypeReference<List<Record>>() {});
recordResponse =
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-influxdb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/influxdb/InfluxdbIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-influxdb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/influxdb/InfluxdbIT.java
index c139afc5e8..74310f7868 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-influxdb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/influxdb/InfluxdbIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-influxdb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/influxdb/InfluxdbIT.java
@@ -249,8 +249,8 @@ public class InfluxdbIT extends TestSuiteBase implements TestResource {
@TestTemplate
@DisabledOnContainer(
value = {},
- type = {EngineType.SPARK, EngineType.FLINK},
- disabledReason = "Currently SPARK/FLINK do not support multiple table read")
+ type = {EngineType.FLINK},
+ disabledReason = "Currently FLINK do not support multiple table read")
public void testInfluxdbMultipleWrite(TestContainer container)
throws IOException, InterruptedException {
Container.ExecResult execResult =
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlMultipleTablesIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlMultipleTablesIT.java
index 9cb23f2c51..494d273879 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlMultipleTablesIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlMultipleTablesIT.java
@@ -171,7 +171,7 @@ public class JdbcMysqlMultipleTablesIT extends TestSuiteBase implements TestReso
@DisabledOnContainer(
value = {},
- type = {EngineType.SPARK, EngineType.FLINK},
+ type = {EngineType.FLINK},
disabledReason = "Currently SPARK and FLINK do not support multiple tables")
@TestTemplate
public void testMysqlJdbcMultipleTableE2e(TestContainer container)
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kudu-e2e/src/test/java/org/apache/seatunnel/e2e/connector/kudu/KuduIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kudu-e2e/src/test/java/org/apache/seatunnel/e2e/connector/kudu/KuduIT.java
index dc7ab43386..e8e61d3931 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kudu-e2e/src/test/java/org/apache/seatunnel/e2e/connector/kudu/KuduIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kudu-e2e/src/test/java/org/apache/seatunnel/e2e/connector/kudu/KuduIT.java
@@ -387,8 +387,8 @@ public class KuduIT extends TestSuiteBase implements TestResource {
@DisabledOnContainer(
value = {},
- type = {EngineType.SPARK, EngineType.FLINK},
- disabledReason = "Currently SPARK/FLINK do not support multiple table read")
+ type = {EngineType.FLINK},
+ disabledReason = "Currently FLINK do not support multiple table read")
@TestTemplate
public void testKuduMultipleRead(TestContainer container)
throws IOException, InterruptedException {
@@ -405,8 +405,8 @@ public class KuduIT extends TestSuiteBase implements TestResource {
@DisabledOnContainer(
value = {},
- type = {EngineType.SPARK, EngineType.FLINK},
- disabledReason = "Currently SPARK/FLINK do not support multiple table read")
+ type = {EngineType.FLINK},
+ disabledReason = "Currently FLINK do not support multiple table read")
@TestTemplate
public void testKuduMultipleWrite(TestContainer container)
throws IOException, InterruptedException {
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java
index 0b9ed1bedb..280ca94faf 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java
@@ -339,8 +339,8 @@ public class RedisIT extends TestSuiteBase implements TestResource {
@TestTemplate
@DisabledOnContainer(
value = {},
- type = {EngineType.SPARK, EngineType.FLINK},
- disabledReason = "Currently SPARK/FLINK do not support multiple table read")
+ type = {EngineType.FLINK},
+ disabledReason = "Currently FLINK do not support multiple table read")
public void testMultipletableRedisSink(TestContainer container)
throws IOException, InterruptedException {
Container.ExecResult execResult =
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSink.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSink.java
index 96438d3ac7..0315f275ff 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSink.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSink.java
@@ -43,7 +43,7 @@ public class SparkSink<StateT, CommitInfoT, AggregatedCommitInfoT>
private volatile SeaTunnelSink<SeaTunnelRow, StateT, CommitInfoT, AggregatedCommitInfoT> sink;
- private volatile CatalogTable catalogTable;
+ private volatile CatalogTable[] catalogTables;
private volatile String jobId;
@@ -58,8 +58,8 @@ public class SparkSink<StateT, CommitInfoT, AggregatedCommitInfoT>
"can not find sink "
+ "class string in DataSourceOptions")));
}
- if (catalogTable == null) {
- this.catalogTable =
+ if (catalogTables == null) {
+ this.catalogTables =
SerializationUtils.stringToObject(
options.get(SparkSinkInjector.SINK_CATALOG_TABLE)
.orElseThrow(
@@ -79,7 +79,7 @@ public class SparkSink<StateT, CommitInfoT, AggregatedCommitInfoT>
init(options);
try {
- return new SparkStreamWriter<>(sink, catalogTable, jobId);
+ return new SparkStreamWriter<>(sink, catalogTables, jobId);
} catch (IOException e) {
throw new RuntimeException("find error when createStreamWriter", e);
}
@@ -91,7 +91,7 @@ public class SparkSink<StateT, CommitInfoT, AggregatedCommitInfoT>
init(options);
try {
- return Optional.of(new SparkDataSourceWriter<>(sink, catalogTable, jobId));
+ return Optional.of(new SparkDataSourceWriter<>(sink, catalogTables, jobId));
} catch (IOException e) {
throw new RuntimeException("find error when createStreamWriter", e);
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java
index 44aec8f70a..740188339c 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java
@@ -38,23 +38,23 @@ public class SparkSinkInjector {
public static DataStreamWriter<Row> inject(
DataStreamWriter<Row> dataset,
SeaTunnelSink<?, ?, ?, ?> sink,
- CatalogTable catalogTable,
+ CatalogTable[] catalogTables,
String applicationId) {
return dataset.format(SPARK_SINK_CLASS_NAME)
.outputMode(OutputMode.Append())
.option(Constants.SINK_SERIALIZATION, SerializationUtils.objectToString(sink))
- .option(SINK_CATALOG_TABLE, SerializationUtils.objectToString(catalogTable))
+ .option(SINK_CATALOG_TABLE, SerializationUtils.objectToString(catalogTables))
.option(JOB_ID, applicationId);
}
public static DataFrameWriter<Row> inject(
DataFrameWriter<Row> dataset,
SeaTunnelSink<?, ?, ?, ?> sink,
- CatalogTable catalogTable,
+ CatalogTable[] catalogTables,
String applicationId) {
return dataset.format(SPARK_SINK_CLASS_NAME)
.option(Constants.SINK_SERIALIZATION, SerializationUtils.objectToString(sink))
- .option(SINK_CATALOG_TABLE, SerializationUtils.objectToString(catalogTable))
+ .option(SINK_CATALOG_TABLE, SerializationUtils.objectToString(catalogTables))
.option(JOB_ID, applicationId);
}
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkDataSourceWriter.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkDataSourceWriter.java
index b02ae32fbc..29755c6db4 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkDataSourceWriter.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkDataSourceWriter.java
@@ -47,18 +47,18 @@ public class SparkDataSourceWriter<StateT, CommitInfoT, AggregatedCommitInfoT>
@Nullable protected final SinkAggregatedCommitter<CommitInfoT, AggregatedCommitInfoT>
sinkAggregatedCommitter;
- protected final CatalogTable catalogTable;
+ protected final CatalogTable[] catalogTables;
protected final String jobId;
private MultiTableResourceManager resourceManager;
public SparkDataSourceWriter(
SeaTunnelSink<SeaTunnelRow, StateT, CommitInfoT, AggregatedCommitInfoT> sink,
- CatalogTable catalogTable,
+ CatalogTable[] catalogTables,
String jobId)
throws IOException {
this.sink = sink;
- this.catalogTable = catalogTable;
+ this.catalogTables = catalogTables;
this.jobId = jobId;
this.sinkAggregatedCommitter = sink.createAggregatedCommitter().orElse(null);
if (sinkAggregatedCommitter != null) {
@@ -78,7 +78,7 @@ public class SparkDataSourceWriter<StateT, CommitInfoT, AggregatedCommitInfoT>
@Override
public DataWriterFactory<InternalRow> createWriterFactory() {
- return new SparkDataWriterFactory<>(sink, catalogTable, jobId);
+ return new SparkDataWriterFactory<>(sink, catalogTables, jobId);
}
@Override
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkDataWriter.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkDataWriter.java
index 95d4c840c1..434b1ef979 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkDataWriter.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkDataWriter.java
@@ -21,10 +21,8 @@ import org.apache.seatunnel.api.sink.MultiTableResourceManager;
import org.apache.seatunnel.api.sink.SinkCommitter;
import org.apache.seatunnel.api.sink.SinkWriter;
import org.apache.seatunnel.api.sink.SupportResourceShare;
-import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
-import org.apache.seatunnel.translation.serialization.RowConverter;
-import org.apache.seatunnel.translation.spark.serialization.InternalRowConverter;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.sources.v2.writer.DataWriter;
@@ -41,32 +39,33 @@ import java.util.Optional;
@Slf4j
public class SparkDataWriter<CommitInfoT, StateT> implements DataWriter<InternalRow> {
- private final SinkWriter<SeaTunnelRow, CommitInfoT, StateT> sinkWriter;
+ protected final SinkWriter<SeaTunnelRow, CommitInfoT, StateT> sinkWriter;
- @Nullable private final SinkCommitter<CommitInfoT> sinkCommitter;
- private final RowConverter<InternalRow> rowConverter;
- private CommitInfoT latestCommitInfoT;
- private long epochId;
- private MultiTableResourceManager resourceManager;
+ @Nullable protected final SinkCommitter<CommitInfoT> sinkCommitter;
+ protected CommitInfoT latestCommitInfoT;
+ protected long epochId;
+ protected volatile MultiTableResourceManager resourceManager;
+
+ private final MultiTableManager multiTableManager;
SparkDataWriter(
SinkWriter<SeaTunnelRow, CommitInfoT, StateT> sinkWriter,
@Nullable SinkCommitter<CommitInfoT> sinkCommitter,
- SeaTunnelDataType<?> dataType,
+ MultiTableManager multiTableManager,
long epochId) {
this.sinkWriter = sinkWriter;
this.sinkCommitter = sinkCommitter;
- this.rowConverter = new InternalRowConverter(dataType);
this.epochId = epochId == 0 ? 1 : epochId;
+ this.multiTableManager = multiTableManager;
initResourceManger();
}
@Override
public void write(InternalRow record) throws IOException {
- sinkWriter.write(rowConverter.reconvert(record));
+ sinkWriter.write(multiTableManager.reconvert(record));
}
- private void initResourceManger() {
+ protected void initResourceManger() {
if (sinkWriter instanceof SupportResourceShare) {
resourceManager =
((SupportResourceShare) sinkWriter).initMultiTableResourceManager(1, 1);
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkDataWriterFactory.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkDataWriterFactory.java
index a9c6cd445a..3a646f3aca 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkDataWriterFactory.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkDataWriterFactory.java
@@ -23,6 +23,7 @@ import org.apache.seatunnel.api.sink.SinkCommitter;
import org.apache.seatunnel.api.sink.SinkWriter;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.sources.v2.writer.DataWriter;
@@ -33,15 +34,15 @@ import java.io.IOException;
public class SparkDataWriterFactory<CommitInfoT, StateT> implements DataWriterFactory<InternalRow> {
private final SeaTunnelSink<SeaTunnelRow, StateT, CommitInfoT, ?> sink;
- private final CatalogTable catalogTable;
+ private final CatalogTable[] catalogTables;
private final String jobId;
SparkDataWriterFactory(
SeaTunnelSink<SeaTunnelRow, StateT, CommitInfoT, ?> sink,
- CatalogTable catalogTable,
+ CatalogTable[] catalogTables,
String jobId) {
this.sink = sink;
- this.catalogTable = catalogTable;
+ this.catalogTables = catalogTables;
this.jobId = jobId;
}
@@ -62,6 +63,6 @@ public class SparkDataWriterFactory<CommitInfoT, StateT> implements DataWriterFa
throw new RuntimeException("Failed to create SinkCommitter.", e);
}
return new SparkDataWriter<>(
- writer, committer, catalogTable.getSeaTunnelRowType(), epochId);
+ writer, committer, new MultiTableManager(catalogTables), epochId);
}
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkStreamWriter.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkStreamWriter.java
index 4d297454f8..1c8bbb4b87 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkStreamWriter.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/sink/writer/SparkStreamWriter.java
@@ -34,10 +34,10 @@ public class SparkStreamWriter<StateT, CommitInfoT, AggregatedCommitInfoT>
public SparkStreamWriter(
SeaTunnelSink<SeaTunnelRow, StateT, CommitInfoT, AggregatedCommitInfoT> sink,
- CatalogTable catalogTable,
+ CatalogTable[] catalogTables,
String jobId)
throws IOException {
- super(sink, catalogTable, jobId);
+ super(sink, catalogTables, jobId);
}
@Override
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceSupport.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceSupport.java
index be97f39752..ad6ee7a723 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceSupport.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceSupport.java
@@ -20,9 +20,13 @@ package org.apache.seatunnel.translation.spark.source;
import org.apache.seatunnel.api.common.CommonOptions;
import org.apache.seatunnel.api.env.EnvCommonOptions;
import org.apache.seatunnel.api.source.SeaTunnelSource;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.Constants;
import org.apache.seatunnel.common.utils.SerializationUtils;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.source.reader.batch.BatchSourceReader;
import org.apache.seatunnel.translation.spark.source.reader.micro.MicroBatchSourceReader;
@@ -42,6 +46,7 @@ import org.apache.spark.sql.types.StructType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -67,7 +72,19 @@ public class SeaTunnelSourceSupport
int parallelism = options.getInt(CommonOptions.PARALLELISM.key(), 1);
Map<String, String> envOptions = options.asMap();
String applicationId = SparkSession.getActiveSession().get().sparkContext().applicationId();
- return new BatchSourceReader(seaTunnelSource, applicationId, parallelism, envOptions);
+ List<CatalogTable> catalogTables;
+ try {
+ catalogTables = seaTunnelSource.getProducedCatalogTables();
+ } catch (UnsupportedOperationException e) {
+ // TODO remove it when all connector use `getProducedCatalogTables`
+ SeaTunnelDataType<?> seaTunnelDataType = seaTunnelSource.getProducedType();
+ catalogTables =
+ CatalogTableUtil.convertDataTypeToCatalogTables(seaTunnelDataType, "default");
+ }
+ MultiTableManager multiTableManager =
+ new MultiTableManager(catalogTables.toArray(new CatalogTable[0]));
+ return new BatchSourceReader(
+ seaTunnelSource, applicationId, parallelism, envOptions, multiTableManager);
}
@Override
@@ -91,6 +108,17 @@ public class SeaTunnelSourceSupport
String hdfsUser = options.get(Constants.HDFS_USER).orElse("");
Integer checkpointId = options.getInt(Constants.CHECKPOINT_ID, 1);
Map<String, String> envOptions = options.asMap();
+ List<CatalogTable> catalogTables;
+ try {
+ catalogTables = seaTunnelSource.getProducedCatalogTables();
+ } catch (UnsupportedOperationException e) {
+ // TODO remove it when all connector use `getProducedCatalogTables`
+ SeaTunnelDataType<?> seaTunnelDataType = seaTunnelSource.getProducedType();
+ catalogTables =
+ CatalogTableUtil.convertDataTypeToCatalogTables(seaTunnelDataType, "default");
+ }
+ MultiTableManager multiTableManager =
+ new MultiTableManager(catalogTables.toArray(new CatalogTable[0]));
return new MicroBatchSourceReader(
seaTunnelSource,
parallelism,
@@ -100,7 +128,8 @@ public class SeaTunnelSourceSupport
checkpointPath,
hdfsRoot,
hdfsUser,
- envOptions);
+ envOptions,
+ multiTableManager);
}
private SeaTunnelSource<SeaTunnelRow, ?, ?> getSeaTunnelSource(DataSourceOptions options) {
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/BatchPartition.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/BatchPartition.java
index 965706024c..090a8a1cf7 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/BatchPartition.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/BatchPartition.java
@@ -20,6 +20,7 @@ package org.apache.seatunnel.translation.spark.source.partition.batch;
import org.apache.seatunnel.api.source.SeaTunnelSource;
import org.apache.seatunnel.api.source.SupportCoordinate;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.source.reader.SeaTunnelInputPartitionReader;
import org.apache.seatunnel.translation.spark.source.reader.batch.CoordinatedBatchPartitionReader;
import org.apache.seatunnel.translation.spark.source.reader.batch.ParallelBatchPartitionReader;
@@ -37,17 +38,21 @@ public class BatchPartition implements InputPartition<InternalRow> {
protected final Integer subtaskId;
private Map<String, String> envOptions;
+ private final MultiTableManager multiTableManager;
+
public BatchPartition(
SeaTunnelSource<SeaTunnelRow, ?, ?> source,
Integer parallelism,
String jobId,
Integer subtaskId,
- Map<String, String> envOptions) {
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
this.source = source;
this.parallelism = parallelism;
this.jobId = jobId;
this.subtaskId = subtaskId;
this.envOptions = envOptions;
+ this.multiTableManager = multiTableManager;
}
@Override
@@ -56,11 +61,11 @@ public class BatchPartition implements InputPartition<InternalRow> {
if (source instanceof SupportCoordinate) {
partitionReader =
new CoordinatedBatchPartitionReader(
- source, parallelism, jobId, subtaskId, envOptions);
+ source, parallelism, jobId, subtaskId, envOptions, multiTableManager);
} else {
partitionReader =
new ParallelBatchPartitionReader(
- source, parallelism, jobId, subtaskId, envOptions);
+ source, parallelism, jobId, subtaskId, envOptions, multiTableManager);
}
return new SeaTunnelInputPartitionReader(partitionReader);
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/MicroBatchPartition.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/MicroBatchPartition.java
index 8f28160357..264d17eeeb 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/MicroBatchPartition.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/MicroBatchPartition.java
@@ -20,6 +20,7 @@ package org.apache.seatunnel.translation.spark.source.partition.micro;
import org.apache.seatunnel.api.source.SeaTunnelSource;
import org.apache.seatunnel.api.source.SupportCoordinate;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.source.reader.SeaTunnelInputPartitionReader;
import org.apache.seatunnel.translation.spark.source.reader.batch.ParallelBatchPartitionReader;
import org.apache.seatunnel.translation.spark.source.reader.micro.CoordinatedMicroBatchPartitionReader;
@@ -43,6 +44,8 @@ public class MicroBatchPartition implements InputPartition<InternalRow> {
protected final String hdfsUser;
private Map<String, String> envOptions;
+ protected final MultiTableManager multiTableManager;
+
public MicroBatchPartition(
SeaTunnelSource<SeaTunnelRow, ?, ?> source,
Integer parallelism,
@@ -53,7 +56,8 @@ public class MicroBatchPartition implements InputPartition<InternalRow> {
String checkpointPath,
String hdfsRoot,
String hdfsUser,
- Map<String, String> envOptions) {
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
this.source = source;
this.parallelism = parallelism;
this.jobId = jobId;
@@ -64,6 +68,7 @@ public class MicroBatchPartition implements InputPartition<InternalRow> {
this.hdfsRoot = hdfsRoot;
this.hdfsUser = hdfsUser;
this.envOptions = envOptions;
+ this.multiTableManager = multiTableManager;
}
@Override
@@ -81,7 +86,8 @@ public class MicroBatchPartition implements InputPartition<InternalRow> {
checkpointPath,
hdfsRoot,
hdfsUser,
- envOptions);
+ envOptions,
+ multiTableManager);
} else {
partitionReader =
new ParallelMicroBatchPartitionReader(
@@ -94,7 +100,8 @@ public class MicroBatchPartition implements InputPartition<InternalRow> {
checkpointPath,
hdfsRoot,
hdfsUser,
- envOptions);
+ envOptions,
+ multiTableManager);
}
return new SeaTunnelInputPartitionReader(partitionReader);
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/BatchSourceReader.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/BatchSourceReader.java
index 4f5528622d..4bef013a60 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/BatchSourceReader.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/BatchSourceReader.java
@@ -20,8 +20,8 @@ package org.apache.seatunnel.translation.spark.source.reader.batch;
import org.apache.seatunnel.api.source.SeaTunnelSource;
import org.apache.seatunnel.api.source.SupportCoordinate;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.source.partition.batch.BatchPartition;
-import org.apache.seatunnel.translation.spark.utils.TypeConverterUtils;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.sources.v2.reader.DataSourceReader;
@@ -38,21 +38,24 @@ public class BatchSourceReader implements DataSourceReader {
protected final String jobId;
protected final Integer parallelism;
private Map<String, String> envOptions;
+ private final MultiTableManager multiTableManager;
public BatchSourceReader(
SeaTunnelSource<SeaTunnelRow, ?, ?> source,
String jobId,
Integer parallelism,
- Map<String, String> envOptions) {
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
this.source = source;
this.jobId = jobId;
this.parallelism = parallelism;
this.envOptions = envOptions;
+ this.multiTableManager = multiTableManager;
}
@Override
public StructType readSchema() {
- return (StructType) TypeConverterUtils.convert(source.getProducedType());
+ return multiTableManager.getTableSchema();
}
@Override
@@ -60,12 +63,20 @@ public class BatchSourceReader implements DataSourceReader {
List<InputPartition<InternalRow>> virtualPartitions;
if (source instanceof SupportCoordinate) {
virtualPartitions = new ArrayList<>(1);
- virtualPartitions.add(new BatchPartition(source, parallelism, jobId, 0, envOptions));
+ virtualPartitions.add(
+ new BatchPartition(
+ source, parallelism, jobId, 0, envOptions, multiTableManager));
} else {
virtualPartitions = new ArrayList<>(parallelism);
for (int subtaskId = 0; subtaskId < parallelism; subtaskId++) {
virtualPartitions.add(
- new BatchPartition(source, parallelism, jobId, subtaskId, envOptions));
+ new BatchPartition(
+ source,
+ parallelism,
+ jobId,
+ subtaskId,
+ envOptions,
+ multiTableManager));
}
}
return virtualPartitions;
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/CoordinatedBatchPartitionReader.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/CoordinatedBatchPartitionReader.java
index bf7dd22142..47d55767ff 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/CoordinatedBatchPartitionReader.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/CoordinatedBatchPartitionReader.java
@@ -24,6 +24,7 @@ import org.apache.seatunnel.api.source.SourceSplit;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.translation.source.BaseSourceFunction;
import org.apache.seatunnel.translation.source.CoordinatedSource;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.serialization.InternalRowCollector;
import java.io.Serializable;
@@ -41,14 +42,14 @@ public class CoordinatedBatchPartitionReader extends ParallelBatchPartitionReade
Integer parallelism,
String jobId,
Integer subtaskId,
- Map<String, String> envOptions) {
- super(source, parallelism, jobId, subtaskId, envOptions);
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
+ super(source, parallelism, jobId, subtaskId, envOptions, multiTableManager);
this.collectorMap = new HashMap<>(parallelism);
for (int i = 0; i < parallelism; i++) {
collectorMap.put(
i,
- new InternalRowCollector(
- handover, new Object(), source.getProducedType(), envOptions));
+ multiTableManager.getInternalRowCollector(handover, new Object(), envOptions));
}
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/ParallelBatchPartitionReader.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/ParallelBatchPartitionReader.java
index 71ed2f7c22..2024063870 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/ParallelBatchPartitionReader.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/ParallelBatchPartitionReader.java
@@ -23,6 +23,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.Handover;
import org.apache.seatunnel.translation.source.BaseSourceFunction;
import org.apache.seatunnel.translation.source.ParallelSource;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.serialization.InternalRowCollector;
import org.apache.seatunnel.translation.util.ThreadPoolExecutorFactory;
@@ -58,12 +59,15 @@ public class ParallelBatchPartitionReader {
protected volatile InternalRowCollector internalRowCollector;
private Map<String, String> envOptions;
+ protected final MultiTableManager multiTableManager;
+
public ParallelBatchPartitionReader(
SeaTunnelSource<SeaTunnelRow, ?, ?> source,
Integer parallelism,
String jobId,
Integer subtaskId,
- Map<String, String> envOptions) {
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
this.source = source;
this.parallelism = parallelism;
this.jobId = jobId;
@@ -73,6 +77,7 @@ public class ParallelBatchPartitionReader {
1, getEnumeratorThreadName());
this.handover = new Handover<>();
this.envOptions = envOptions;
+ this.multiTableManager = multiTableManager;
}
protected String getEnumeratorThreadName() {
@@ -105,8 +110,8 @@ public class ParallelBatchPartitionReader {
}
this.internalRowCollector =
- new InternalRowCollector(
- handover, checkpointLock, source.getProducedType(), envOptions);
+ multiTableManager.getInternalRowCollector(handover, checkpointLock, envOptions);
+
executorService.execute(
() -> {
try {
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/micro/CoordinatedMicroBatchPartitionReader.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/micro/CoordinatedMicroBatchPartitionReader.java
index 11e57cc437..9023ba3651 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/micro/CoordinatedMicroBatchPartitionReader.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/micro/CoordinatedMicroBatchPartitionReader.java
@@ -24,6 +24,7 @@ import org.apache.seatunnel.api.source.SourceSplit;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.translation.source.BaseSourceFunction;
import org.apache.seatunnel.translation.source.CoordinatedSource;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.serialization.InternalRowCollector;
import org.apache.seatunnel.translation.spark.source.state.ReaderState;
@@ -47,7 +48,8 @@ public class CoordinatedMicroBatchPartitionReader extends ParallelMicroBatchPart
String checkpointPath,
String hdfsRoot,
String hdfsUser,
- Map<String, String> envOptions) {
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
super(
source,
parallelism,
@@ -58,13 +60,13 @@ public class CoordinatedMicroBatchPartitionReader extends ParallelMicroBatchPart
checkpointPath,
hdfsRoot,
hdfsUser,
- envOptions);
+ envOptions,
+ multiTableManager);
this.collectorMap = new HashMap<>(parallelism);
for (int i = 0; i < parallelism; i++) {
collectorMap.put(
i,
- new InternalRowCollector(
- handover, new Object(), source.getProducedType(), envOptions));
+ multiTableManager.getInternalRowCollector(handover, new Object(), envOptions));
}
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/micro/MicroBatchSourceReader.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/micro/MicroBatchSourceReader.java
index d49e5bd6a6..f28c8c1b79 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/micro/MicroBatchSourceReader.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/micro/MicroBatchSourceReader.java
@@ -21,9 +21,9 @@ import org.apache.seatunnel.api.source.SeaTunnelSource;
import org.apache.seatunnel.api.source.SupportCoordinate;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.utils.SerializationUtils;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.source.partition.micro.MicroBatchPartition;
import org.apache.seatunnel.translation.spark.source.state.MicroBatchState;
-import org.apache.seatunnel.translation.spark.utils.TypeConverterUtils;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.sources.v2.reader.InputPartition;
@@ -50,6 +50,7 @@ public class MicroBatchSourceReader implements MicroBatchReader {
protected MicroBatchState startOffset;
protected MicroBatchState endOffset;
private Map<String, String> envOptions;
+ private final MultiTableManager multiTableManager;
public MicroBatchSourceReader(
SeaTunnelSource<SeaTunnelRow, ?, ?> source,
@@ -60,7 +61,8 @@ public class MicroBatchSourceReader implements MicroBatchReader {
String checkpointPath,
String hdfsRoot,
String hdfsUser,
- Map<String, String> envOptions) {
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
this.source = source;
this.parallelism = parallelism;
this.jobId = jobId;
@@ -70,6 +72,7 @@ public class MicroBatchSourceReader implements MicroBatchReader {
this.hdfsRoot = hdfsRoot;
this.hdfsUser = hdfsUser;
this.envOptions = envOptions;
+ this.multiTableManager = multiTableManager;
}
@Override
@@ -108,7 +111,7 @@ public class MicroBatchSourceReader implements MicroBatchReader {
@Override
public StructType readSchema() {
- return (StructType) TypeConverterUtils.convert(source.getProducedType());
+ return multiTableManager.getTableSchema();
}
@Override
@@ -127,7 +130,8 @@ public class MicroBatchSourceReader implements MicroBatchReader {
checkpointPath,
hdfsRoot,
hdfsUser,
- envOptions));
+ envOptions,
+ multiTableManager));
} else {
virtualPartitions = new ArrayList<>(parallelism);
for (int subtaskId = 0; subtaskId < parallelism; subtaskId++) {
@@ -142,7 +146,8 @@ public class MicroBatchSourceReader implements MicroBatchReader {
checkpointPath,
hdfsRoot,
hdfsUser,
- envOptions));
+ envOptions,
+ multiTableManager));
}
}
checkpointId++;
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/micro/ParallelMicroBatchPartitionReader.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/micro/ParallelMicroBatchPartitionReader.java
index d3457246f7..b820a2caf7 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/micro/ParallelMicroBatchPartitionReader.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/micro/ParallelMicroBatchPartitionReader.java
@@ -21,6 +21,7 @@ import org.apache.seatunnel.api.source.SeaTunnelSource;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.utils.SerializationUtils;
import org.apache.seatunnel.translation.source.BaseSourceFunction;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.source.reader.batch.ParallelBatchPartitionReader;
import org.apache.seatunnel.translation.spark.source.state.ReaderState;
import org.apache.seatunnel.translation.util.ThreadPoolExecutorFactory;
@@ -65,8 +66,9 @@ public class ParallelMicroBatchPartitionReader extends ParallelBatchPartitionRea
String checkpointPath,
String hdfsRoot,
String hdfsUser,
- Map<String, String> envOptions) {
- super(source, parallelism, jobId, subtaskId, envOptions);
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
+ super(source, parallelism, jobId, subtaskId, envOptions, multiTableManager);
this.checkpointId = checkpointId;
this.checkpointInterval = checkpointInterval;
this.checkpointPath = checkpointPath;
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/SeaTunnelBatchWrite.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/SeaTunnelBatchWrite.java
index a650854e2a..f1c343ba56 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/SeaTunnelBatchWrite.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/SeaTunnelBatchWrite.java
@@ -49,17 +49,17 @@ public class SeaTunnelBatchWrite<StateT, CommitInfoT, AggregatedCommitInfoT>
private MultiTableResourceManager resourceManager;
- private final CatalogTable catalogTable;
+ private final CatalogTable[] catalogTables;
private final String jobId;
public SeaTunnelBatchWrite(
SeaTunnelSink<SeaTunnelRow, StateT, CommitInfoT, AggregatedCommitInfoT> sink,
- CatalogTable catalogTable,
+ CatalogTable[] catalogTables,
String jobId)
throws IOException {
this.sink = sink;
- this.catalogTable = catalogTable;
+ this.catalogTables = catalogTables;
this.jobId = jobId;
this.aggregatedCommitter = sink.createAggregatedCommitter().orElse(null);
if (aggregatedCommitter != null) {
@@ -78,7 +78,7 @@ public class SeaTunnelBatchWrite<StateT, CommitInfoT, AggregatedCommitInfoT>
@Override
public DataWriterFactory createBatchWriterFactory(PhysicalWriteInfo info) {
- return new SeaTunnelSparkDataWriterFactory<>(sink, catalogTable, jobId);
+ return new SeaTunnelSparkDataWriterFactory<>(sink, catalogTables, jobId);
}
@Override
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/SeaTunnelSinkTable.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/SeaTunnelSinkTable.java
index 1fb8109eb5..f3652be673 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/SeaTunnelSinkTable.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/SeaTunnelSinkTable.java
@@ -22,8 +22,8 @@ import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.Constants;
import org.apache.seatunnel.common.utils.SerializationUtils;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.sink.write.SeaTunnelWriteBuilder;
-import org.apache.seatunnel.translation.spark.utils.TypeConverterUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.spark.sql.connector.catalog.SupportsWrite;
@@ -46,7 +46,7 @@ public class SeaTunnelSinkTable implements Table, SupportsWrite {
private final SeaTunnelSink<SeaTunnelRow, ?, ?, ?> sink;
- private final CatalogTable catalogTable;
+ private final CatalogTable[] catalogTables;
private final String jobId;
public SeaTunnelSinkTable(Map<String, String> properties) {
@@ -62,13 +62,13 @@ public class SeaTunnelSinkTable implements Table, SupportsWrite {
throw new IllegalArgumentException(
SparkSinkInjector.SINK_CATALOG_TABLE + " must be specified");
}
- this.catalogTable = SerializationUtils.stringToObject(sinkCatalogTableSerialization);
+ this.catalogTables = SerializationUtils.stringToObject(sinkCatalogTableSerialization);
this.jobId = properties.getOrDefault(SparkSinkInjector.JOB_ID, null);
}
@Override
public WriteBuilder newWriteBuilder(LogicalWriteInfo info) {
- return new SeaTunnelWriteBuilder<>(sink, catalogTable, jobId);
+ return new SeaTunnelWriteBuilder<>(sink, catalogTables, jobId);
}
@Override
@@ -78,7 +78,7 @@ public class SeaTunnelSinkTable implements Table, SupportsWrite {
@Override
public StructType schema() {
- return (StructType) TypeConverterUtils.convert(catalogTable.getSeaTunnelRowType());
+ return new MultiTableManager(catalogTables).getTableSchema();
}
@Override
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java
index d6d148c3fe..b975c0913b 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java
@@ -38,23 +38,25 @@ public class SparkSinkInjector {
public static DataStreamWriter<Row> inject(
DataStreamWriter<Row> dataset,
SeaTunnelSink<?, ?, ?, ?> sink,
- CatalogTable catalogTable,
+ CatalogTable[] catalogTables,
String applicationId) {
return dataset.format(SINK_NAME)
.outputMode(OutputMode.Append())
.option(Constants.SINK_SERIALIZATION, SerializationUtils.objectToString(sink))
- .option(SINK_CATALOG_TABLE, SerializationUtils.objectToString(catalogTable))
+ // TODO this should require fetching the catalog table in sink
+ .option(SINK_CATALOG_TABLE, SerializationUtils.objectToString(catalogTables))
.option(JOB_ID, applicationId);
}
public static DataFrameWriter<Row> inject(
DataFrameWriter<Row> dataset,
SeaTunnelSink<?, ?, ?, ?> sink,
- CatalogTable catalogTable,
+ CatalogTable[] catalogTables,
String applicationId) {
return dataset.format(SINK_NAME)
.option(Constants.SINK_SERIALIZATION, SerializationUtils.objectToString(sink))
- .option(SINK_CATALOG_TABLE, SerializationUtils.objectToString(catalogTable))
+ // TODO this should require fetching the catalog table in sink
+ .option(SINK_CATALOG_TABLE, SerializationUtils.objectToString(catalogTables))
.option(JOB_ID, applicationId);
}
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelSparkDataWriter.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelSparkDataWriter.java
index 0ce7896bdd..59f931e38f 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelSparkDataWriter.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelSparkDataWriter.java
@@ -21,10 +21,8 @@ import org.apache.seatunnel.api.sink.MultiTableResourceManager;
import org.apache.seatunnel.api.sink.SinkCommitter;
import org.apache.seatunnel.api.sink.SinkWriter;
import org.apache.seatunnel.api.sink.SupportResourceShare;
-import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
-import org.apache.seatunnel.translation.serialization.RowConverter;
-import org.apache.seatunnel.translation.spark.serialization.InternalRowConverter;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.connector.write.DataWriter;
@@ -41,32 +39,33 @@ import java.util.Optional;
@Slf4j
public class SeaTunnelSparkDataWriter<CommitInfoT, StateT> implements DataWriter<InternalRow> {
- private final SinkWriter<SeaTunnelRow, CommitInfoT, StateT> sinkWriter;
+ protected final SinkWriter<SeaTunnelRow, CommitInfoT, StateT> sinkWriter;
- @Nullable private final SinkCommitter<CommitInfoT> sinkCommitter;
- private final RowConverter<InternalRow> rowConverter;
- private CommitInfoT latestCommitInfoT;
- private long epochId;
- private volatile MultiTableResourceManager resourceManager;
+ @Nullable protected final SinkCommitter<CommitInfoT> sinkCommitter;
+ protected CommitInfoT latestCommitInfoT;
+ protected long epochId;
+ protected volatile MultiTableResourceManager resourceManager;
+
+ private final MultiTableManager multiTableManager;
public SeaTunnelSparkDataWriter(
SinkWriter<SeaTunnelRow, CommitInfoT, StateT> sinkWriter,
@Nullable SinkCommitter<CommitInfoT> sinkCommitter,
- SeaTunnelDataType<?> dataType,
+ MultiTableManager multiTableManager,
long epochId) {
this.sinkWriter = sinkWriter;
this.sinkCommitter = sinkCommitter;
- this.rowConverter = new InternalRowConverter(dataType);
+ this.multiTableManager = multiTableManager;
this.epochId = epochId == 0 ? 1 : epochId;
initResourceManger();
}
@Override
public void write(InternalRow record) throws IOException {
- sinkWriter.write(rowConverter.reconvert(record));
+ sinkWriter.write(multiTableManager.reconvert(record));
}
- private void initResourceManger() {
+ protected void initResourceManger() {
if (sinkWriter instanceof SupportResourceShare) {
resourceManager =
((SupportResourceShare) sinkWriter).initMultiTableResourceManager(1, 1);
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelSparkDataWriterFactory.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelSparkDataWriterFactory.java
index 5046c2decc..b83787cac1 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelSparkDataWriterFactory.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelSparkDataWriterFactory.java
@@ -23,6 +23,7 @@ import org.apache.seatunnel.api.sink.SinkCommitter;
import org.apache.seatunnel.api.sink.SinkWriter;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.connector.write.DataWriter;
@@ -35,15 +36,15 @@ public class SeaTunnelSparkDataWriterFactory<CommitInfoT, StateT>
implements DataWriterFactory, StreamingDataWriterFactory {
private final SeaTunnelSink<SeaTunnelRow, StateT, CommitInfoT, ?> sink;
- private final CatalogTable catalogTable;
+ private final CatalogTable[] catalogTables;
private final String jobId;
public SeaTunnelSparkDataWriterFactory(
SeaTunnelSink<SeaTunnelRow, StateT, CommitInfoT, ?> sink,
- CatalogTable catalogTable,
+ CatalogTable[] catalogTables,
String jobId) {
this.sink = sink;
- this.catalogTable = catalogTable;
+ this.catalogTables = catalogTables;
this.jobId = jobId;
}
@@ -63,7 +64,7 @@ public class SeaTunnelSparkDataWriterFactory<CommitInfoT, StateT>
throw new RuntimeException("Failed to create SinkCommitter.", e);
}
return new SeaTunnelSparkDataWriter<>(
- writer, committer, catalogTable.getSeaTunnelRowType(), 0);
+ writer, committer, new MultiTableManager(catalogTables), 0);
}
@Override
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelWrite.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelWrite.java
index 21ae52f128..2bd4388cc2 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelWrite.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelWrite.java
@@ -31,22 +31,22 @@ import java.io.IOException;
public class SeaTunnelWrite<AggregatedCommitInfoT, CommitInfoT, StateT> implements Write {
private final SeaTunnelSink<SeaTunnelRow, StateT, CommitInfoT, AggregatedCommitInfoT> sink;
- private final CatalogTable catalogTable;
+ private final CatalogTable[] catalogTables;
private final String jobId;
public SeaTunnelWrite(
SeaTunnelSink<SeaTunnelRow, StateT, CommitInfoT, AggregatedCommitInfoT> sink,
- CatalogTable catalogTable,
+ CatalogTable[] catalogTables,
String jobId) {
this.sink = sink;
- this.catalogTable = catalogTable;
+ this.catalogTables = catalogTables;
this.jobId = jobId;
}
@Override
public BatchWrite toBatch() {
try {
- return new SeaTunnelBatchWrite<>(sink, catalogTable, jobId);
+ return new SeaTunnelBatchWrite<>(sink, catalogTables, jobId);
} catch (IOException e) {
throw new RuntimeException("SeaTunnel Spark sink create batch failed", e);
}
@@ -55,7 +55,7 @@ public class SeaTunnelWrite<AggregatedCommitInfoT, CommitInfoT, StateT> implemen
@Override
public StreamingWrite toStreaming() {
try {
- return new SeaTunnelBatchWrite<>(sink, catalogTable, jobId);
+ return new SeaTunnelBatchWrite<>(sink, catalogTables, jobId);
} catch (IOException e) {
throw new RuntimeException("SeaTunnel Spark sink create batch failed", e);
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelWriteBuilder.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelWriteBuilder.java
index d1ff3cfca1..896982c351 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelWriteBuilder.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/sink/write/SeaTunnelWriteBuilder.java
@@ -28,20 +28,20 @@ public class SeaTunnelWriteBuilder<StateT, CommitInfoT, AggregatedCommitInfoT>
implements WriteBuilder {
private final SeaTunnelSink<SeaTunnelRow, StateT, CommitInfoT, AggregatedCommitInfoT> sink;
- private final CatalogTable catalogTable;
+ private final CatalogTable[] catalogTables;
private final String jobId;
public SeaTunnelWriteBuilder(
SeaTunnelSink<SeaTunnelRow, StateT, CommitInfoT, AggregatedCommitInfoT> sink,
- CatalogTable catalogTable,
+ CatalogTable[] catalogTables,
String jobId) {
this.sink = sink;
- this.catalogTable = catalogTable;
+ this.catalogTables = catalogTables;
this.jobId = jobId;
}
@Override
public Write build() {
- return new SeaTunnelWrite<>(sink, catalogTable, jobId);
+ return new SeaTunnelWrite<>(sink, catalogTables, jobId);
}
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceTable.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceTable.java
index fb5eb6d6a9..ef707095e6 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceTable.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceTable.java
@@ -19,11 +19,14 @@ package org.apache.seatunnel.translation.spark.source;
import org.apache.seatunnel.api.common.CommonOptions;
import org.apache.seatunnel.api.source.SeaTunnelSource;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.Constants;
import org.apache.seatunnel.common.utils.SerializationUtils;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.source.scan.SeaTunnelScanBuilder;
-import org.apache.seatunnel.translation.spark.utils.TypeConverterUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.spark.sql.SparkSession;
@@ -37,6 +40,7 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap;
import com.google.common.collect.Sets;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -48,6 +52,8 @@ public class SeaTunnelSourceTable implements Table, SupportsRead {
private final SeaTunnelSource<SeaTunnelRow, ?, ?> source;
+ private final MultiTableManager multiTableManager;
+
public SeaTunnelSourceTable(Map<String, String> properties) {
this.properties = properties;
String sourceSerialization = properties.getOrDefault(Constants.SOURCE_SERIALIZATION, "");
@@ -55,6 +61,16 @@ public class SeaTunnelSourceTable implements Table, SupportsRead {
throw new IllegalArgumentException("source.serialization must be specified");
}
this.source = SerializationUtils.stringToObject(sourceSerialization);
+ List<CatalogTable> catalogTables;
+ try {
+ catalogTables = source.getProducedCatalogTables();
+ } catch (UnsupportedOperationException e) {
+ // TODO remove it when all connector use `getProducedCatalogTables`
+ SeaTunnelDataType<?> seaTunnelDataType = source.getProducedType();
+ catalogTables =
+ CatalogTableUtil.convertDataTypeToCatalogTables(seaTunnelDataType, "default");
+ }
+ multiTableManager = new MultiTableManager(catalogTables.toArray(new CatalogTable[0]));
}
/**
@@ -69,7 +85,7 @@ public class SeaTunnelSourceTable implements Table, SupportsRead {
Integer.parseInt(properties.getOrDefault(CommonOptions.PARALLELISM.key(), "1"));
String applicationId = SparkSession.getActiveSession().get().sparkContext().applicationId();
return new SeaTunnelScanBuilder(
- source, parallelism, applicationId, caseInsensitiveStringMap);
+ source, parallelism, applicationId, caseInsensitiveStringMap, multiTableManager);
}
/** A name to identify this table */
@@ -81,7 +97,7 @@ public class SeaTunnelSourceTable implements Table, SupportsRead {
/** Returns the schema of this table */
@Override
public StructType schema() {
- return (StructType) TypeConverterUtils.convert(source.getProducedType());
+ return multiTableManager.getTableSchema();
}
/** Returns the set of capabilities for this table */
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/CoordinatedBatchPartitionReader.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/CoordinatedBatchPartitionReader.java
index 746cf34838..69dc64c234 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/CoordinatedBatchPartitionReader.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/CoordinatedBatchPartitionReader.java
@@ -24,6 +24,7 @@ import org.apache.seatunnel.api.source.SourceSplit;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.translation.source.BaseSourceFunction;
import org.apache.seatunnel.translation.source.CoordinatedSource;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.serialization.InternalRowCollector;
import java.io.Serializable;
@@ -41,14 +42,14 @@ public class CoordinatedBatchPartitionReader extends ParallelBatchPartitionReade
Integer parallelism,
String jobId,
Integer subtaskId,
- Map<String, String> envOptions) {
- super(source, parallelism, jobId, subtaskId, envOptions);
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
+ super(source, parallelism, jobId, subtaskId, envOptions, multiTableManager);
this.collectorMap = new HashMap<>(parallelism);
for (int i = 0; i < parallelism; i++) {
collectorMap.put(
i,
- new InternalRowCollector(
- handover, new Object(), source.getProducedType(), envOptions));
+ multiTableManager.getInternalRowCollector(handover, new Object(), envOptions));
}
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/ParallelBatchPartitionReader.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/ParallelBatchPartitionReader.java
index 8bbbcc71c1..27ab9f42d2 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/ParallelBatchPartitionReader.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/ParallelBatchPartitionReader.java
@@ -23,6 +23,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.Handover;
import org.apache.seatunnel.translation.source.BaseSourceFunction;
import org.apache.seatunnel.translation.source.ParallelSource;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.serialization.InternalRowCollector;
import org.apache.seatunnel.translation.util.ThreadPoolExecutorFactory;
@@ -58,12 +59,15 @@ public class ParallelBatchPartitionReader {
protected volatile InternalRowCollector internalRowCollector;
private final Map<String, String> envOptions;
+ private final MultiTableManager multiTableManager;
+
public ParallelBatchPartitionReader(
SeaTunnelSource<SeaTunnelRow, ?, ?> source,
Integer parallelism,
String jobId,
Integer subtaskId,
- Map<String, String> envOptions) {
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
this.source = source;
this.parallelism = parallelism;
this.jobId = jobId;
@@ -73,6 +77,7 @@ public class ParallelBatchPartitionReader {
1, getEnumeratorThreadName());
this.handover = new Handover<>();
this.envOptions = envOptions;
+ this.multiTableManager = multiTableManager;
}
protected String getEnumeratorThreadName() {
@@ -105,8 +110,7 @@ public class ParallelBatchPartitionReader {
}
this.internalRowCollector =
- new InternalRowCollector(
- handover, checkpointLock, source.getProducedType(), envOptions);
+ multiTableManager.getInternalRowCollector(handover, checkpointLock, envOptions);
executorService.execute(
() -> {
try {
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/SeaTunnelBatch.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/SeaTunnelBatch.java
index 491555f32b..30c4644078 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/SeaTunnelBatch.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/SeaTunnelBatch.java
@@ -20,6 +20,7 @@ package org.apache.seatunnel.translation.spark.source.partition.batch;
import org.apache.seatunnel.api.source.SeaTunnelSource;
import org.apache.seatunnel.api.source.SupportCoordinate;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.spark.sql.connector.read.Batch;
import org.apache.spark.sql.connector.read.InputPartition;
@@ -36,15 +37,19 @@ public class SeaTunnelBatch implements Batch {
private final String jobId;
private final Map<String, String> envOptions;
+ private final MultiTableManager multiTableManager;
+
public SeaTunnelBatch(
SeaTunnelSource<SeaTunnelRow, ?, ?> source,
int parallelism,
String jobId,
- Map<String, String> envOptions) {
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
this.source = source;
this.parallelism = parallelism;
this.jobId = jobId;
this.envOptions = envOptions;
+ this.multiTableManager = multiTableManager;
}
@Override
@@ -64,6 +69,7 @@ public class SeaTunnelBatch implements Batch {
@Override
public PartitionReaderFactory createReaderFactory() {
- return new SeaTunnelBatchPartitionReaderFactory(source, parallelism, jobId, envOptions);
+ return new SeaTunnelBatchPartitionReaderFactory(
+ source, parallelism, jobId, envOptions, multiTableManager);
}
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/SeaTunnelBatchPartitionReaderFactory.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/SeaTunnelBatchPartitionReaderFactory.java
index f15ae051ec..5781262ef5 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/SeaTunnelBatchPartitionReaderFactory.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/SeaTunnelBatchPartitionReaderFactory.java
@@ -20,6 +20,7 @@ package org.apache.seatunnel.translation.spark.source.partition.batch;
import org.apache.seatunnel.api.source.SeaTunnelSource;
import org.apache.seatunnel.api.source.SupportCoordinate;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.connector.read.InputPartition;
@@ -36,15 +37,19 @@ public class SeaTunnelBatchPartitionReaderFactory implements PartitionReaderFact
private final String jobId;
private final Map<String, String> envOptions;
+ private final MultiTableManager multiTableManager;
+
public SeaTunnelBatchPartitionReaderFactory(
SeaTunnelSource<SeaTunnelRow, ?, ?> source,
int parallelism,
String jobId,
- Map<String, String> envOptions) {
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
this.source = source;
this.parallelism = parallelism;
this.jobId = jobId;
this.envOptions = envOptions;
+ this.multiTableManager = multiTableManager;
}
@Override
@@ -55,11 +60,11 @@ public class SeaTunnelBatchPartitionReaderFactory implements PartitionReaderFact
if (source instanceof SupportCoordinate) {
partitionReader =
new CoordinatedBatchPartitionReader(
- source, parallelism, jobId, partitionId, envOptions);
+ source, parallelism, jobId, partitionId, envOptions, multiTableManager);
} else {
partitionReader =
new ParallelBatchPartitionReader(
- source, parallelism, jobId, partitionId, envOptions);
+ source, parallelism, jobId, partitionId, envOptions, multiTableManager);
}
return new SeaTunnelBatchPartitionReader(partitionReader);
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/CoordinatedMicroBatchPartitionReader.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/CoordinatedMicroBatchPartitionReader.java
index 2df1b08926..ccf71692dc 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/CoordinatedMicroBatchPartitionReader.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/CoordinatedMicroBatchPartitionReader.java
@@ -24,6 +24,7 @@ import org.apache.seatunnel.api.source.SourceSplit;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.translation.source.BaseSourceFunction;
import org.apache.seatunnel.translation.source.CoordinatedSource;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.serialization.InternalRowCollector;
import java.io.Serializable;
@@ -46,7 +47,8 @@ public class CoordinatedMicroBatchPartitionReader extends ParallelMicroBatchPart
String checkpointPath,
String hdfsRoot,
String hdfsUser,
- Map<String, String> envOptions) {
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
super(
source,
parallelism,
@@ -57,13 +59,13 @@ public class CoordinatedMicroBatchPartitionReader extends ParallelMicroBatchPart
checkpointPath,
hdfsRoot,
hdfsUser,
- envOptions);
+ envOptions,
+ multiTableManager);
this.collectorMap = new HashMap<>(parallelism);
for (int i = 0; i < parallelism; i++) {
collectorMap.put(
i,
- new InternalRowCollector(
- handover, new Object(), source.getProducedType(), envOptions));
+ multiTableManager.getInternalRowCollector(handover, new Object(), envOptions));
}
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/ParallelMicroBatchPartitionReader.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/ParallelMicroBatchPartitionReader.java
index c62185d099..bfa7aa6079 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/ParallelMicroBatchPartitionReader.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/ParallelMicroBatchPartitionReader.java
@@ -21,6 +21,7 @@ import org.apache.seatunnel.api.source.SeaTunnelSource;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.utils.SerializationUtils;
import org.apache.seatunnel.translation.source.BaseSourceFunction;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.source.partition.batch.ParallelBatchPartitionReader;
import org.apache.seatunnel.translation.util.ThreadPoolExecutorFactory;
@@ -64,8 +65,9 @@ public class ParallelMicroBatchPartitionReader extends ParallelBatchPartitionRea
String checkpointPath,
String hdfsRoot,
String hdfsUser,
- Map<String, String> envOptions) {
- super(source, parallelism, jobId, subtaskId, envOptions);
+ Map<String, String> envOptions,
+ MultiTableManager multiTableManager) {
+ super(source, parallelism, jobId, subtaskId, envOptions, multiTableManager);
this.checkpointId = checkpointId;
this.checkpointInterval = checkpointInterval;
this.checkpointPath = checkpointPath;
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/SeaTunnelMicroBatch.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/SeaTunnelMicroBatch.java
index 0d9f6d4cdb..11eec4d7fc 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/SeaTunnelMicroBatch.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/SeaTunnelMicroBatch.java
@@ -23,6 +23,7 @@ import org.apache.seatunnel.api.source.SupportCoordinate;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.Constants;
import org.apache.seatunnel.common.utils.JsonUtils;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -56,17 +57,21 @@ public class SeaTunnelMicroBatch implements MicroBatchStream {
private Offset currentOffset = initialOffset;
+ private final MultiTableManager multiTableManager;
+
public SeaTunnelMicroBatch(
SeaTunnelSource<SeaTunnelRow, ?, ?> source,
int parallelism,
String jobId,
String checkpointLocation,
- CaseInsensitiveStringMap caseInsensitiveStringMap) {
+ CaseInsensitiveStringMap caseInsensitiveStringMap,
+ MultiTableManager multiTableManager) {
this.source = source;
this.parallelism = parallelism;
this.jobId = jobId;
this.checkpointLocation = checkpointLocation;
this.caseInsensitiveStringMap = caseInsensitiveStringMap;
+ this.multiTableManager = multiTableManager;
}
@Override
@@ -119,7 +124,12 @@ public class SeaTunnelMicroBatch implements MicroBatchStream {
@Override
public PartitionReaderFactory createReaderFactory() {
return new SeaTunnelMicroBatchPartitionReaderFactory(
- source, parallelism, jobId, checkpointLocation, caseInsensitiveStringMap);
+ source,
+ parallelism,
+ jobId,
+ checkpointLocation,
+ caseInsensitiveStringMap,
+ multiTableManager);
}
@Override
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/SeaTunnelMicroBatchPartitionReaderFactory.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/SeaTunnelMicroBatchPartitionReaderFactory.java
index 4742916894..a92a7dfb3b 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/SeaTunnelMicroBatchPartitionReaderFactory.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/micro/SeaTunnelMicroBatchPartitionReaderFactory.java
@@ -20,6 +20,7 @@ package org.apache.seatunnel.translation.spark.source.partition.micro;
import org.apache.seatunnel.api.source.SeaTunnelSource;
import org.apache.seatunnel.api.source.SupportCoordinate;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.source.partition.batch.ParallelBatchPartitionReader;
import org.apache.spark.sql.catalyst.InternalRow;
@@ -41,17 +42,21 @@ public class SeaTunnelMicroBatchPartitionReaderFactory implements PartitionReade
private final CaseInsensitiveStringMap caseInsensitiveStringMap;
+ private final MultiTableManager multiTableManager;
+
public SeaTunnelMicroBatchPartitionReaderFactory(
SeaTunnelSource<SeaTunnelRow, ?, ?> source,
int parallelism,
String jobId,
String checkpointLocation,
- CaseInsensitiveStringMap caseInsensitiveStringMap) {
+ CaseInsensitiveStringMap caseInsensitiveStringMap,
+ MultiTableManager multiTableManager) {
this.source = source;
this.parallelism = parallelism;
this.jobId = jobId;
this.checkpointLocation = checkpointLocation;
this.caseInsensitiveStringMap = caseInsensitiveStringMap;
+ this.multiTableManager = multiTableManager;
}
@Override
@@ -77,7 +82,8 @@ public class SeaTunnelMicroBatchPartitionReaderFactory implements PartitionReade
checkpointLocation,
hdfsRoot,
hdfsUser,
- envOptions);
+ envOptions,
+ multiTableManager);
} else {
partitionReader =
new ParallelMicroBatchPartitionReader(
@@ -90,7 +96,8 @@ public class SeaTunnelMicroBatchPartitionReaderFactory implements PartitionReade
checkpointLocation,
hdfsRoot,
hdfsUser,
- envOptions);
+ envOptions,
+ multiTableManager);
}
return new SeaTunnelMicroBatchPartitionReader(partitionReader);
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/scan/SeaTunnelScan.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/scan/SeaTunnelScan.java
index aa19699408..c2085d9127 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/scan/SeaTunnelScan.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/scan/SeaTunnelScan.java
@@ -19,9 +19,9 @@ package org.apache.seatunnel.translation.spark.source.scan;
import org.apache.seatunnel.api.source.SeaTunnelSource;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.seatunnel.translation.spark.source.partition.batch.SeaTunnelBatch;
import org.apache.seatunnel.translation.spark.source.partition.micro.SeaTunnelMicroBatch;
-import org.apache.seatunnel.translation.spark.utils.TypeConverterUtils;
import org.apache.spark.sql.connector.read.Batch;
import org.apache.spark.sql.connector.read.Scan;
@@ -40,31 +40,40 @@ public class SeaTunnelScan implements Scan {
private final CaseInsensitiveStringMap caseInsensitiveStringMap;
+ private final MultiTableManager multiTableManager;
+
public SeaTunnelScan(
SeaTunnelSource<SeaTunnelRow, ?, ?> source,
int parallelism,
String jobId,
- CaseInsensitiveStringMap caseInsensitiveStringMap) {
+ CaseInsensitiveStringMap caseInsensitiveStringMap,
+ MultiTableManager multiTableManager) {
this.source = source;
this.parallelism = parallelism;
this.jobId = jobId;
this.caseInsensitiveStringMap = caseInsensitiveStringMap;
+ this.multiTableManager = multiTableManager;
}
@Override
public StructType readSchema() {
- return (StructType) TypeConverterUtils.convert(source.getProducedType());
+ return multiTableManager.getTableSchema();
}
@Override
public Batch toBatch() {
Map<String, String> envOptions = caseInsensitiveStringMap.asCaseSensitiveMap();
- return new SeaTunnelBatch(source, parallelism, jobId, envOptions);
+ return new SeaTunnelBatch(source, parallelism, jobId, envOptions, multiTableManager);
}
@Override
public MicroBatchStream toMicroBatchStream(String checkpointLocation) {
return new SeaTunnelMicroBatch(
- source, parallelism, jobId, checkpointLocation, caseInsensitiveStringMap);
+ source,
+ parallelism,
+ jobId,
+ checkpointLocation,
+ caseInsensitiveStringMap,
+ multiTableManager);
}
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/scan/SeaTunnelScanBuilder.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/scan/SeaTunnelScanBuilder.java
index efff7956c1..932f83dac9 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/scan/SeaTunnelScanBuilder.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/scan/SeaTunnelScanBuilder.java
@@ -19,6 +19,7 @@ package org.apache.seatunnel.translation.spark.source.scan;
import org.apache.seatunnel.api.source.SeaTunnelSource;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.translation.spark.execution.MultiTableManager;
import org.apache.spark.sql.connector.read.Scan;
import org.apache.spark.sql.connector.read.ScanBuilder;
@@ -33,20 +34,25 @@ public class SeaTunnelScanBuilder implements ScanBuilder {
private final CaseInsensitiveStringMap caseInsensitiveStringMap;
+ private final MultiTableManager multiTableManager;
+
public SeaTunnelScanBuilder(
SeaTunnelSource<SeaTunnelRow, ?, ?> source,
int parallelism,
String jobId,
- CaseInsensitiveStringMap caseInsensitiveStringMap) {
+ CaseInsensitiveStringMap caseInsensitiveStringMap,
+ MultiTableManager multiTableManager) {
this.source = source;
this.parallelism = parallelism;
this.jobId = jobId;
this.caseInsensitiveStringMap = caseInsensitiveStringMap;
+ this.multiTableManager = multiTableManager;
}
/** Returns the {@link SeaTunnelScan} */
@Override
public Scan build() {
- return new SeaTunnelScan(source, parallelism, jobId, caseInsensitiveStringMap);
+ return new SeaTunnelScan(
+ source, parallelism, jobId, caseInsensitiveStringMap, multiTableManager);
}
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/sink/SparkSinkTest.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/sink/SparkSinkTest.java
index b426f17548..73084fb76b 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/sink/SparkSinkTest.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/sink/SparkSinkTest.java
@@ -17,11 +17,13 @@
package org.apache.seatunnel.translation.spark.sink;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
import org.apache.seatunnel.api.table.type.BasicType;
import org.apache.seatunnel.api.table.type.LocalTimeType;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.translation.spark.utils.TypeConverterUtils;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
@@ -131,6 +133,8 @@ public class SparkSinkTest {
GenericRow row1WithRow =
new GenericRow(
new Object[] {
+ (byte) 1,
+ "test.test.test",
42,
"string1",
true,
@@ -196,6 +200,8 @@ public class SparkSinkTest {
GenericRow row2WithRow =
new GenericRow(
new Object[] {
+ (byte) 1,
+ "test.test.test",
12,
"string2",
false,
@@ -261,6 +267,8 @@ public class SparkSinkTest {
GenericRow row3WithRow =
new GenericRow(
new Object[] {
+ (byte) 1,
+ "test.test.test",
233,
"string3",
true,
@@ -397,15 +405,18 @@ public class SparkSinkTest {
BasicType.STRING_TYPE, BasicType.STRING_TYPE)
})
});
-
+ structType.add("row", structType);
+ StructType parcelStructType = (StructType) TypeConverterUtils.parcel(rowType);
Dataset<Row> dataset =
spark.createDataFrame(
- Arrays.asList(row1WithRow, row2WithRow, row3WithRow),
- structType.add("row", structType));
+ Arrays.asList(row1WithRow, row2WithRow, row3WithRow), parcelStructType);
SparkSinkInjector.inject(
dataset.write(),
new SeaTunnelSinkWithBuffer(),
- CatalogTableUtil.getCatalogTable("test", "test", "test", "test", rowType),
+ new CatalogTable[] {
+ CatalogTableUtil.getCatalogTable(
+ "test", "test", "test", "test", rowType)
+ },
spark.sparkContext().applicationId())
.option("checkpointLocation", "/tmp")
.mode(SaveMode.Append)
diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/DatasetTableInfo.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/execution/ColumnWithIndex.java
similarity index 52%
copy from seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/DatasetTableInfo.java
copy to seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/execution/ColumnWithIndex.java
index 3af8a5ad6d..43628a6dff 100644
--- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/DatasetTableInfo.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/execution/ColumnWithIndex.java
@@ -14,24 +14,40 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.seatunnel.core.starter.spark.execution;
+package org.apache.seatunnel.translation.spark.execution;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-@Data
-@AllArgsConstructor
-public class DatasetTableInfo {
+import java.io.Serializable;
+import java.util.Arrays;
- private Dataset<Row> dataset;
+public class ColumnWithIndex implements Serializable {
+ private int[] index;
private CatalogTable catalogTable;
- private String tableName;
+ public ColumnWithIndex(int[] index, CatalogTable catalogTable) {
+ this.index = index;
+ this.catalogTable = catalogTable;
+ }
+
+ public int[] getIndex() {
+ return index;
+ }
+
+ public CatalogTable getCatalogTable() {
+ return catalogTable;
+ }
+
+ @Override
+ public String toString() {
+ return "ColumnWithIndex{"
+ + "table="
+ + catalogTable.getTablePath()
+ + ", index="
+ + Arrays.toString(index)
+ + ", schema="
+ + catalogTable.getSeaTunnelRowType()
+ + '}';
+ }
}
diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/DatasetTableInfo.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/execution/DatasetTableInfo.java
similarity index 74%
copy from seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/DatasetTableInfo.java
copy to seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/execution/DatasetTableInfo.java
index 3af8a5ad6d..cd2fa938d5 100644
--- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/DatasetTableInfo.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/execution/DatasetTableInfo.java
@@ -15,23 +15,28 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.starter.spark.execution;
+package org.apache.seatunnel.translation.spark.execution;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import lombok.AllArgsConstructor;
import lombok.Data;
+import java.util.List;
+
@Data
-@AllArgsConstructor
public class DatasetTableInfo {
private Dataset<Row> dataset;
-
- private CatalogTable catalogTable;
-
+ private List<CatalogTable> catalogTables;
private String tableName;
+
+ public DatasetTableInfo(
+ Dataset<Row> dataset, List<CatalogTable> catalogTables, String tableName) {
+ this.dataset = dataset;
+ this.catalogTables = catalogTables;
+ this.tableName = tableName;
+ }
}
diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/DatasetTableInfo.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/execution/IndexQueue.java
similarity index 52%
rename from seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/DatasetTableInfo.java
rename to seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/execution/IndexQueue.java
index 3af8a5ad6d..521b0854df 100644
--- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/DatasetTableInfo.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/execution/IndexQueue.java
@@ -14,24 +14,41 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.seatunnel.translation.spark.execution;
-package org.apache.seatunnel.core.starter.spark.execution;
+import java.util.List;
+import java.util.ListIterator;
-import org.apache.seatunnel.api.table.catalog.CatalogTable;
+public class IndexQueue<T> {
+ private List<T> list;
+ private ListIterator<T> listIterator;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
+ public IndexQueue(List<T> list) {
+ this.list = list;
+ this.listIterator = list.listIterator();
+ }
-import lombok.AllArgsConstructor;
-import lombok.Data;
+ public boolean hasNext() {
+ return listIterator.hasNext();
+ }
-@Data
-@AllArgsConstructor
-public class DatasetTableInfo {
+ public T next() {
+ return listIterator.next();
+ }
- private Dataset<Row> dataset;
+ public void add(T t) {
+ listIterator.add(t);
+ }
- private CatalogTable catalogTable;
+ public void append(T t) {
+ list.add(t);
+ }
- private String tableName;
+ public void set(T t) {
+ listIterator.set(t);
+ }
+
+ public void reset() {
+ listIterator = list.listIterator();
+ }
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/execution/MultiTableManager.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/execution/MultiTableManager.java
new file mode 100644
index 0000000000..ab470ce3dd
--- /dev/null
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/execution/MultiTableManager.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.translation.spark.execution;
+
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.common.Handover;
+import org.apache.seatunnel.translation.spark.serialization.InternalMultiRowCollector;
+import org.apache.seatunnel.translation.spark.serialization.InternalRowCollector;
+import org.apache.seatunnel.translation.spark.serialization.InternalRowConverter;
+import org.apache.seatunnel.translation.spark.utils.TypeConverterUtils;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.types.StructType;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+@Slf4j
+public class MultiTableManager implements Serializable {
+
+ private Map<String, InternalRowConverter> rowSerializationMap;
+
+ private InternalRowConverter rowSerialization;
+ private CatalogTable mergeCatalogTable;
+ private boolean isMultiTable = false;
+
+ public MultiTableManager(CatalogTable[] catalogTables) {
+ List<ColumnWithIndex> columnWithIndexes = mergeSchema(catalogTables);
+ if (catalogTables.length > 1) {
+ isMultiTable = true;
+ rowSerializationMap =
+ columnWithIndexes.stream()
+ .collect(
+ Collectors.toMap(
+ columnWithIndex ->
+ columnWithIndex
+ .getCatalogTable()
+ .getTablePath()
+ .toString(),
+ columnWithIndex ->
+ new InternalRowConverter(
+ mergeCatalogTable.getSeaTunnelRowType(),
+ columnWithIndex.getIndex())));
+ } else {
+ rowSerialization = new InternalRowConverter(catalogTables[0].getSeaTunnelRowType());
+ }
+ log.info("Multi-table enabled:{}", isMultiTable);
+ log.info(
+ "merged table {}, schema {}",
+ mergeCatalogTable.getTablePath(),
+ mergeCatalogTable.getSeaTunnelRowType());
+ for (ColumnWithIndex columnWithIndex : columnWithIndexes) {
+ log.info("MultiTableManager columnWithIndex:{}", columnWithIndex);
+ }
+ }
+
+ public SeaTunnelRow reconvert(InternalRow record) throws IOException {
+ if (isMultiTable) {
+ String tableId = record.getString(1);
+ return rowSerializationMap.get(tableId).reconvert(record);
+ }
+ return rowSerialization.reconvert(record);
+ }
+
+ public StructType getTableSchema() {
+ return (StructType) TypeConverterUtils.parcel(mergeCatalogTable.getSeaTunnelRowType());
+ }
+
+ public List<ColumnWithIndex> mergeSchema(CatalogTable[] catalogTables) {
+ List<ColumnWithIndex> columnWithIndexes = new ArrayList<>();
+ if (catalogTables.length == 1) {
+ CatalogTable catalogTable = catalogTables[0];
+ columnWithIndexes.add(
+ new ColumnWithIndex(
+ IntStream.rangeClosed(
+ 0, catalogTable.getSeaTunnelRowType().getTotalFields())
+ .toArray(),
+ catalogTable));
+ mergeCatalogTable = catalogTable;
+ return columnWithIndexes;
+ }
+ List<String> fieldNames = new ArrayList<>();
+ List<SeaTunnelDataType<?>> fieldTypes = new ArrayList<>();
+ int indexSize = -1;
+ HashMap<SeaTunnelDataType<?>, IndexQueue<Integer>> map = new HashMap<>();
+ for (int i = 0; i < catalogTables.length; i++) {
+ CatalogTable catalogTable = catalogTables[i];
+ SeaTunnelRowType seaTunnelRowType = catalogTable.getSeaTunnelRowType();
+ SeaTunnelDataType<?>[] seaTunnelDataTypes = seaTunnelRowType.getFieldTypes();
+ int[] indexes = new int[seaTunnelDataTypes.length];
+ for (int j = 0; j < seaTunnelDataTypes.length; j++) {
+ IndexQueue<Integer> indexQueue =
+ map.computeIfAbsent(
+ seaTunnelDataTypes[j], k -> new IndexQueue<>(new ArrayList<>()));
+ if (indexQueue.hasNext()) {
+ indexes[j] = indexQueue.next();
+ } else {
+ indexSize++;
+ indexes[j] = indexSize;
+ indexQueue.add(indexSize);
+ fieldNames.add(editColumnName(indexSize));
+ fieldTypes.add(seaTunnelDataTypes[j]);
+ }
+ }
+ map.forEach((k, v) -> v.reset());
+ columnWithIndexes.add(new ColumnWithIndex(indexes, catalogTable));
+ }
+ SeaTunnelRowType rowType =
+ new SeaTunnelRowType(
+ fieldNames.toArray(new String[0]),
+ fieldTypes.toArray(new SeaTunnelDataType[0]));
+ mergeCatalogTable =
+ CatalogTableUtil.getCatalogTable(
+ "spark", "default", "default", "merge_table", rowType);
+ return columnWithIndexes;
+ }
+
+ public static String editColumnName(int index) {
+ return "column" + index;
+ }
+
+ public InternalRowCollector getInternalRowCollector(
+ Handover<InternalRow> handover,
+ Object checkpointLock,
+ Map<String, String> envOptionsInfo) {
+ if (isMultiTable) {
+ return new InternalMultiRowCollector(
+ handover, checkpointLock, rowSerializationMap, envOptionsInfo);
+ } else {
+ return new InternalRowCollector(
+ handover, checkpointLock, rowSerialization, envOptionsInfo);
+ }
+ }
+}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalRowCollector.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalMultiRowCollector.java
similarity index 51%
copy from seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalRowCollector.java
copy to seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalMultiRowCollector.java
index 0bbcd3a6fe..5c5573b3e3 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalRowCollector.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalMultiRowCollector.java
@@ -17,44 +17,31 @@
package org.apache.seatunnel.translation.spark.serialization;
-import org.apache.seatunnel.api.source.Collector;
-import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.Handover;
-import org.apache.seatunnel.core.starter.flowcontrol.FlowControlGate;
-import org.apache.seatunnel.core.starter.flowcontrol.FlowControlStrategy;
import org.apache.spark.sql.catalyst.InternalRow;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-public class InternalRowCollector implements Collector<SeaTunnelRow> {
- private final Handover<InternalRow> handover;
- private final Object checkpointLock;
- private final InternalRowConverter rowSerialization;
- private final AtomicLong collectTotalCount;
- private Map<String, Object> envOptions;
- private FlowControlGate flowControlGate;
- private volatile boolean emptyThisPollNext;
+public class InternalMultiRowCollector extends InternalRowCollector {
+ private final Map<String, InternalRowConverter> rowSerializationMap;
- public InternalRowCollector(
+ public InternalMultiRowCollector(
Handover<InternalRow> handover,
Object checkpointLock,
- SeaTunnelDataType<?> dataType,
+ Map<String, InternalRowConverter> rowSerializationMap,
Map<String, String> envOptionsInfo) {
- this.handover = handover;
- this.checkpointLock = checkpointLock;
- this.rowSerialization = new InternalRowConverter(dataType);
- this.collectTotalCount = new AtomicLong(0);
- this.envOptions = (Map) envOptionsInfo;
- this.flowControlGate = FlowControlGate.create(FlowControlStrategy.fromMap(envOptions));
+ super(handover, checkpointLock, null, envOptionsInfo);
+ this.rowSerializationMap = rowSerializationMap;
}
@Override
public void collect(SeaTunnelRow record) {
try {
synchronized (checkpointLock) {
+ InternalRowConverter rowSerialization =
+ rowSerializationMap.get(record.getTableId());
flowControlGate.audit(record);
handover.produce(rowSerialization.convert(record));
}
@@ -65,22 +52,7 @@ public class InternalRowCollector implements Collector<SeaTunnelRow> {
}
}
- public long collectTotalCount() {
- return collectTotalCount.get();
- }
-
- @Override
- public Object getCheckpointLock() {
- return this.checkpointLock;
- }
-
- @Override
- public boolean isEmptyThisPollNext() {
- return emptyThisPollNext;
- }
-
- @Override
- public void resetEmptyThisPollNext() {
- this.emptyThisPollNext = true;
+ public Map<String, InternalRowConverter> getRowSerializationMap() {
+ return rowSerializationMap;
}
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalRowCollector.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalRowCollector.java
index 0bbcd3a6fe..d13b31dc85 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalRowCollector.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalRowCollector.java
@@ -18,7 +18,6 @@
package org.apache.seatunnel.translation.spark.serialization;
import org.apache.seatunnel.api.source.Collector;
-import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.Handover;
import org.apache.seatunnel.core.starter.flowcontrol.FlowControlGate;
@@ -30,22 +29,22 @@ import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
public class InternalRowCollector implements Collector<SeaTunnelRow> {
- private final Handover<InternalRow> handover;
- private final Object checkpointLock;
+ protected final Handover<InternalRow> handover;
+ protected final Object checkpointLock;
private final InternalRowConverter rowSerialization;
- private final AtomicLong collectTotalCount;
+ protected final AtomicLong collectTotalCount;
private Map<String, Object> envOptions;
- private FlowControlGate flowControlGate;
- private volatile boolean emptyThisPollNext;
+ protected FlowControlGate flowControlGate;
+ protected volatile boolean emptyThisPollNext;
public InternalRowCollector(
Handover<InternalRow> handover,
Object checkpointLock,
- SeaTunnelDataType<?> dataType,
+ InternalRowConverter rowSerialization,
Map<String, String> envOptionsInfo) {
this.handover = handover;
this.checkpointLock = checkpointLock;
- this.rowSerialization = new InternalRowConverter(dataType);
+ this.rowSerialization = rowSerialization;
this.collectTotalCount = new AtomicLong(0);
this.envOptions = (Map) envOptionsInfo;
this.flowControlGate = FlowControlGate.create(FlowControlStrategy.fromMap(envOptions));
@@ -83,4 +82,8 @@ public class InternalRowCollector implements Collector<SeaTunnelRow> {
public void resetEmptyThisPollNext() {
this.emptyThisPollNext = true;
}
+
+ public InternalRowConverter getRowSerialization() {
+ return rowSerialization;
+ }
}
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalRowConverter.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalRowConverter.java
index 79269ecf8e..8dbca1975a 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalRowConverter.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/InternalRowConverter.java
@@ -20,6 +20,7 @@ package org.apache.seatunnel.translation.spark.serialization;
import org.apache.seatunnel.api.table.type.ArrayType;
import org.apache.seatunnel.api.table.type.BasicType;
import org.apache.seatunnel.api.table.type.MapType;
+import org.apache.seatunnel.api.table.type.RowKind;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
@@ -63,17 +64,24 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.stream.IntStream;
public final class InternalRowConverter extends RowConverter<InternalRow> {
+ private final int[] indexes;
public InternalRowConverter(SeaTunnelDataType<?> dataType) {
super(dataType);
+ indexes = IntStream.range(0, ((SeaTunnelRowType) dataType).getTotalFields()).toArray();
+ }
+
+ public InternalRowConverter(SeaTunnelDataType<?> dataType, int[] indexes) {
+ super(dataType);
+ this.indexes = indexes;
}
@Override
public InternalRow convert(SeaTunnelRow seaTunnelRow) throws IOException {
- validate(seaTunnelRow);
- return (InternalRow) convert(seaTunnelRow, dataType);
+ return parcel(seaTunnelRow, (SeaTunnelRowType) dataType);
}
private static Object convert(Object field, SeaTunnelDataType<?> dataType) {
@@ -136,6 +144,30 @@ public final class InternalRowConverter extends RowConverter<InternalRow> {
return new SpecificInternalRow(values);
}
+ private InternalRow parcel(SeaTunnelRow seaTunnelRow, SeaTunnelRowType rowType) {
+ // 0 -> row kind, 1 -> table id
+ int arity = rowType.getTotalFields();
+ MutableValue[] values = new MutableValue[arity + 2];
+ for (int i = 0; i < indexes.length; i++) {
+ values[indexes[i] + 2] = createMutableValue(rowType.getFieldType(indexes[i]));
+ Object fieldValue = convert(seaTunnelRow.getField(i), rowType.getFieldType(indexes[i]));
+ if (fieldValue != null) {
+ values[indexes[i] + 2].update(fieldValue);
+ }
+ }
+ values[0] = new MutableByte();
+ values[0].update(seaTunnelRow.getRowKind().toByteValue());
+ values[1] = new MutableAny();
+ values[1].update(UTF8String.fromString(seaTunnelRow.getTableId()));
+ // Fill any remaining null values with MutableAny
+ for (int i = 0; i < values.length; i++) {
+ if (values[i] == null) {
+ values[i] = new MutableAny();
+ }
+ }
+ return new SpecificInternalRow(values);
+ }
+
private static ArrayBasedMapData convertMap(Map<?, ?> mapData, MapType<?, ?> mapType) {
if (mapData == null || mapData.size() == 0) {
return ArrayBasedMapData.apply(new Object[] {}, new Object[] {});
@@ -213,9 +245,27 @@ public final class InternalRowConverter extends RowConverter<InternalRow> {
}
}
+ public SeaTunnelRow unpack(InternalRow engineRow, SeaTunnelRowType rowType) throws IOException {
+ RowKind rowKind = RowKind.fromByteValue(engineRow.getByte(0));
+ String tableId = engineRow.getString(1);
+ Object[] fields = new Object[indexes.length];
+ for (int i = 0; i < indexes.length; i++) {
+ fields[i] =
+ reconvert(
+ engineRow.get(
+ indexes[i] + 2,
+ TypeConverterUtils.convert(rowType.getFieldType(indexes[i]))),
+ rowType.getFieldType(indexes[i]));
+ }
+ SeaTunnelRow seaTunnelRow = new SeaTunnelRow(fields);
+ seaTunnelRow.setRowKind(rowKind);
+ seaTunnelRow.setTableId(tableId);
+ return seaTunnelRow;
+ }
+
@Override
public SeaTunnelRow reconvert(InternalRow engineRow) throws IOException {
- return (SeaTunnelRow) reconvert(engineRow, dataType);
+ return unpack(engineRow, (SeaTunnelRowType) dataType);
}
private static Object reconvert(Object field, SeaTunnelDataType<?> dataType) {
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/SeaTunnelRowConverter.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/SeaTunnelRowConverter.java
index db23d4912a..33d1b49fb1 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/SeaTunnelRowConverter.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/SeaTunnelRowConverter.java
@@ -20,6 +20,7 @@ package org.apache.seatunnel.translation.spark.serialization;
import org.apache.seatunnel.api.table.type.ArrayType;
import org.apache.seatunnel.api.table.type.BasicType;
import org.apache.seatunnel.api.table.type.MapType;
+import org.apache.seatunnel.api.table.type.RowKind;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
@@ -56,11 +57,25 @@ public class SeaTunnelRowConverter extends RowConverter<SeaTunnelRow> {
validate(seaTunnelRow);
GenericRowWithSchema rowWithSchema = (GenericRowWithSchema) convert(seaTunnelRow, dataType);
SeaTunnelRow newRow = new SeaTunnelRow(rowWithSchema.values());
- newRow.setRowKind(seaTunnelRow.getRowKind());
- newRow.setTableId(seaTunnelRow.getTableId());
return newRow;
}
+ public GenericRowWithSchema parcel(SeaTunnelRow seaTunnelRow) {
+ SeaTunnelRowType rowType = (SeaTunnelRowType) dataType;
+ int arity = rowType.getTotalFields();
+ Object[] fields = new Object[arity + 2];
+ fields[0] = seaTunnelRow.getRowKind().toByteValue();
+ fields[1] = seaTunnelRow.getTableId();
+ StructType schema = (StructType) TypeConverterUtils.parcel(rowType);
+ for (int i = 0; i < arity; i++) {
+ Object fieldValue = convert(seaTunnelRow.getField(i), rowType.getFieldType(i));
+ if (fieldValue != null) {
+ fields[i + 2] = fieldValue;
+ }
+ }
+ return new GenericRowWithSchema(fields, schema);
+ }
+
private Object convert(Object field, SeaTunnelDataType<?> dataType) {
if (field == null) {
return null;
@@ -156,6 +171,20 @@ public class SeaTunnelRowConverter extends RowConverter<SeaTunnelRow> {
return (SeaTunnelRow) reconvert(engineRow, dataType);
}
+ public SeaTunnelRow unpack(GenericRowWithSchema engineRow) throws IOException {
+ SeaTunnelRowType rowType = (SeaTunnelRowType) dataType;
+ RowKind rowKind = RowKind.fromByteValue(engineRow.getByte(0));
+ String tableId = engineRow.getString(1);
+ Object[] fields = new Object[rowType.getTotalFields()];
+ for (int i = 0; i < fields.length; i++) {
+ fields[i] = reconvert(engineRow.get(i + 2), rowType.getFieldType(i));
+ }
+ SeaTunnelRow seaTunnelRow = new SeaTunnelRow(fields);
+ seaTunnelRow.setRowKind(rowKind);
+ seaTunnelRow.setTableId(tableId);
+ return seaTunnelRow;
+ }
+
private Object reconvert(Object field, SeaTunnelDataType<?> dataType) {
if (field == null) {
return null;
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/utils/TypeConverterUtils.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/utils/TypeConverterUtils.java
index a4dbcd0c72..627cde2834 100644
--- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/utils/TypeConverterUtils.java
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/utils/TypeConverterUtils.java
@@ -43,7 +43,9 @@ public class TypeConverterUtils {
private static final Map<DataType, SeaTunnelDataType<?>> TO_SEA_TUNNEL_TYPES =
new HashMap<>(16);
- public static final String ROW_KIND_FIELD = "op";
+ public static final String ROW_KIND_FIELD = "seatunnel_row_kind";
+ public static final String ROW = "row";
+ public static final String TABLE_ID = "seatunnel_table_id";
public static final String LOGICAL_TIME_TYPE_FLAG = "logical_time_type";
static {
@@ -129,6 +131,19 @@ public class TypeConverterUtils {
return new StructType(fields);
}
+ public static DataType parcel(SeaTunnelDataType<?> dataType) {
+ // 0 -> row kind, 1 -> table id
+ SeaTunnelRowType seaTunnelRowType = (SeaTunnelRowType) dataType;
+ StructField[] fields = new StructField[2 + seaTunnelRowType.getTotalFields()];
+ fields[0] = new StructField(ROW_KIND_FIELD, DataTypes.ByteType, true, Metadata.empty());
+ fields[1] = new StructField(TABLE_ID, DataTypes.StringType, true, Metadata.empty());
+ StructType structType = (StructType) convert(dataType);
+ for (int i = 0; i < seaTunnelRowType.getTotalFields(); i++) {
+ fields[i + 2] = structType.fields()[i];
+ }
+ return new StructType(fields);
+ }
+
public static SeaTunnelDataType<?> convert(DataType sparkType) {
checkNotNull(sparkType, "The Spark's data type is required.");
SeaTunnelDataType<?> dataType = TO_SEA_TUNNEL_TYPES.get(sparkType);
diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/test/java/org/apache/seatunnel/translation/spark/execution/MultiTableManagerTest.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/test/java/org/apache/seatunnel/translation/spark/execution/MultiTableManagerTest.java
new file mode 100644
index 0000000000..ad5fdfbea5
--- /dev/null
+++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/test/java/org/apache/seatunnel/translation/spark/execution/MultiTableManagerTest.java
@@ -0,0 +1,950 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.seatunnel.translation.spark.execution;
+
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.LocalTimeType;
+import org.apache.seatunnel.api.table.type.RowKind;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.translation.spark.serialization.InternalMultiRowCollector;
+import org.apache.seatunnel.translation.spark.serialization.InternalRowCollector;
+import org.apache.seatunnel.translation.spark.serialization.InternalRowConverter;
+import org.apache.seatunnel.translation.spark.utils.InstantConverterUtils;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.catalyst.expressions.MutableAny;
+import org.apache.spark.sql.catalyst.expressions.MutableBoolean;
+import org.apache.spark.sql.catalyst.expressions.MutableByte;
+import org.apache.spark.sql.catalyst.expressions.MutableDouble;
+import org.apache.spark.sql.catalyst.expressions.MutableFloat;
+import org.apache.spark.sql.catalyst.expressions.MutableInt;
+import org.apache.spark.sql.catalyst.expressions.MutableLong;
+import org.apache.spark.sql.catalyst.expressions.MutableShort;
+import org.apache.spark.sql.catalyst.expressions.MutableValue;
+import org.apache.spark.sql.catalyst.expressions.SpecificInternalRow;
+import org.apache.spark.sql.catalyst.util.ArrayBasedMapData;
+import org.apache.spark.sql.catalyst.util.ArrayData;
+import org.apache.spark.sql.types.ArrayType;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Decimal;
+import org.apache.spark.sql.types.DecimalType;
+import org.apache.spark.sql.types.MapType;
+import org.apache.spark.sql.types.StructType;
+import org.apache.spark.unsafe.types.UTF8String;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.seatunnel.translation.spark.utils.TypeConverterUtils.ROW_KIND_FIELD;
+import static org.apache.seatunnel.translation.spark.utils.TypeConverterUtils.TABLE_ID;
+import static org.apache.spark.sql.types.DataTypes.BooleanType;
+import static org.apache.spark.sql.types.DataTypes.ByteType;
+import static org.apache.spark.sql.types.DataTypes.DateType;
+import static org.apache.spark.sql.types.DataTypes.DoubleType;
+import static org.apache.spark.sql.types.DataTypes.FloatType;
+import static org.apache.spark.sql.types.DataTypes.IntegerType;
+import static org.apache.spark.sql.types.DataTypes.LongType;
+import static org.apache.spark.sql.types.DataTypes.NullType;
+import static org.apache.spark.sql.types.DataTypes.ShortType;
+import static org.apache.spark.sql.types.DataTypes.StringType;
+import static org.apache.spark.sql.types.DataTypes.TimestampType;
+
+public class MultiTableManagerTest {
+
+ private SeaTunnelRowType rowType1;
+ private CatalogTable catalogTable1;
+ private SeaTunnelRowType rowType2;
+ private CatalogTable catalogTable2;
+ private SeaTunnelRowType rowType3;
+ private CatalogTable catalogTable3;
+
+ private StructType structType1;
+ private StructType structType2;
+ private StructType structType3;
+
+ private SeaTunnelRow seaTunnelRow1;
+ private SeaTunnelRow seaTunnelRow3;
+
+ private SpecificInternalRow specificInternalRow1;
+ private SpecificInternalRow specificInternalRow2;
+ private SpecificInternalRow specificInternalRow3;
+
+ @Test
+ public void testMergeSchema() {
+ initSchema();
+ MultiTableManager multiTableManager1 =
+ new MultiTableManager(new CatalogTable[] {catalogTable1, catalogTable2});
+ StructType tableSchema1 = multiTableManager1.getTableSchema();
+ Assertions.assertEquals(structType1, tableSchema1);
+
+ MultiTableManager multiTableManager2 =
+ new MultiTableManager(new CatalogTable[] {catalogTable2, catalogTable1});
+ StructType tableSchema2 = multiTableManager2.getTableSchema();
+ Assertions.assertEquals(structType1, tableSchema2);
+
+ MultiTableManager multiTableManager3 =
+ new MultiTableManager(new CatalogTable[] {catalogTable2, catalogTable3});
+ StructType tableSchema3 = multiTableManager3.getTableSchema();
+ Assertions.assertEquals(structType2, tableSchema3);
+
+ MultiTableManager multiTableManager4 =
+ new MultiTableManager(
+ new CatalogTable[] {catalogTable1, catalogTable2, catalogTable3});
+ StructType tableSchema4 = multiTableManager4.getTableSchema();
+ Assertions.assertEquals(structType2, tableSchema4);
+
+ MultiTableManager multiTableManager5 =
+ new MultiTableManager(new CatalogTable[] {catalogTable1});
+ StructType tableSchema5 = multiTableManager5.getTableSchema();
+ Assertions.assertEquals(structType3, tableSchema5);
+ }
+
+ @Test
+ public void testWriteConverter() throws IOException {
+ initSchema();
+ initData();
+ MultiTableManager multiTableManager =
+ new MultiTableManager(new CatalogTable[] {catalogTable1});
+ SeaTunnelRow seaTunnelRow = multiTableManager.reconvert(specificInternalRow1);
+ for (int i = 0; i < seaTunnelRow.getFields().length; i++) {
+ Object[] values = seaTunnelRow.getFields();
+ Object[] actual = seaTunnelRow1.getFields();
+ for (int v = 0; v < values.length; v++) {
+ if (values[v] instanceof Object[]) {
+ Assertions.assertArrayEquals((Object[]) values[v], (Object[]) actual[v]);
+ } else {
+ Assertions.assertEquals(values[v], actual[v]);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testMultiWriteConverter() throws IOException {
+ initSchema();
+ initData();
+ MultiTableManager multiTableManager =
+ new MultiTableManager(
+ new CatalogTable[] {catalogTable1, catalogTable2, catalogTable3});
+ SeaTunnelRow seaTunnelRow = multiTableManager.reconvert(specificInternalRow1);
+ for (int i = 0; i < seaTunnelRow.getFields().length; i++) {
+ Object[] values = seaTunnelRow.getFields();
+ Object[] actual = seaTunnelRow1.getFields();
+ for (int v = 0; v < values.length; v++) {
+ if (values[v] instanceof Object[]) {
+ Assertions.assertArrayEquals((Object[]) values[v], (Object[]) actual[v]);
+ } else {
+ Assertions.assertEquals(values[v], actual[v]);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testMultiReaderConverter() throws IOException {
+ initSchema();
+ initData();
+ MultiTableManager multiTableManager =
+ new MultiTableManager(
+ new CatalogTable[] {catalogTable1, catalogTable2, catalogTable3});
+ InternalMultiRowCollector internalMultiRowCollector =
+ (InternalMultiRowCollector)
+ multiTableManager.getInternalRowCollector(null, null, null);
+ Map<String, InternalRowConverter> rowSerializationMap =
+ internalMultiRowCollector.getRowSerializationMap();
+ InternalRow internalRow =
+ rowSerializationMap.get(seaTunnelRow1.getTableId()).convert(seaTunnelRow1);
+ for (int v = 0; v < specificInternalRow2.numFields(); v++) {
+ if (specificInternalRow2.genericGet(v) instanceof ArrayBasedMapData) {
+ Assertions.assertEquals(
+ specificInternalRow2.getMap(v).keyArray(),
+ internalRow.getMap(v).keyArray());
+ Assertions.assertEquals(
+ specificInternalRow2.getMap(v).valueArray(),
+ internalRow.getMap(v).valueArray());
+ } else if (specificInternalRow2.genericGet(v) instanceof SpecificInternalRow) {
+ SpecificInternalRow expected =
+ (SpecificInternalRow) specificInternalRow2.genericGet(v);
+ SpecificInternalRow actual =
+ (SpecificInternalRow) ((SpecificInternalRow) internalRow).genericGet(v);
+ for (int o = 0; v < expected.numFields(); v++) {
+ if (expected.genericGet(o) instanceof ArrayBasedMapData) {
+ Assertions.assertEquals(
+ expected.getMap(o).keyArray(), actual.getMap(o).keyArray());
+ Assertions.assertEquals(
+ expected.getMap(o).valueArray(), actual.getMap(o).valueArray());
+ } else {
+ Assertions.assertEquals(
+ expected.genericGet(v),
+ ((SpecificInternalRow) actual).genericGet(v));
+ }
+ }
+ } else {
+ Assertions.assertEquals(
+ specificInternalRow2.genericGet(v),
+ ((SpecificInternalRow) internalRow).genericGet(v));
+ }
+ }
+ InternalRow internalRow3 =
+ rowSerializationMap.get(seaTunnelRow3.getTableId()).convert(seaTunnelRow3);
+ Assertions.assertEquals(specificInternalRow3, internalRow3);
+ for (int v = 0; v < specificInternalRow3.numFields(); v++) {
+ Assertions.assertEquals(
+ specificInternalRow3.genericGet(v),
+ ((SpecificInternalRow) internalRow3).genericGet(v));
+ }
+ }
+
+ @Test
+ public void testReaderConverter() throws IOException {
+ initSchema();
+ initData();
+ MultiTableManager multiTableManager =
+ new MultiTableManager(new CatalogTable[] {catalogTable1});
+ InternalRowCollector internalRowCollector =
+ multiTableManager.getInternalRowCollector(null, null, null);
+ InternalRowConverter rowSerialization = internalRowCollector.getRowSerialization();
+ InternalRow internalRow = rowSerialization.convert(seaTunnelRow1);
+ for (int v = 0; v < specificInternalRow1.numFields(); v++) {
+ if (specificInternalRow1.genericGet(v) instanceof ArrayBasedMapData) {
+ Assertions.assertEquals(
+ specificInternalRow1.getMap(v).keyArray(),
+ internalRow.getMap(v).keyArray());
+ Assertions.assertEquals(
+ specificInternalRow1.getMap(v).valueArray(),
+ internalRow.getMap(v).valueArray());
+ } else if (specificInternalRow1.genericGet(v) instanceof SpecificInternalRow) {
+ SpecificInternalRow expected =
+ (SpecificInternalRow) specificInternalRow1.genericGet(v);
+ SpecificInternalRow actual =
+ (SpecificInternalRow) ((SpecificInternalRow) internalRow).genericGet(v);
+ for (int o = 0; v < expected.numFields(); v++) {
+ if (expected.genericGet(o) instanceof ArrayBasedMapData) {
+ Assertions.assertEquals(
+ expected.getMap(o).keyArray(), actual.getMap(o).keyArray());
+ Assertions.assertEquals(
+ expected.getMap(o).valueArray(), actual.getMap(o).valueArray());
+ } else {
+ Assertions.assertEquals(
+ expected.genericGet(v),
+ ((SpecificInternalRow) actual).genericGet(v));
+ }
+ }
+ } else {
+ Assertions.assertEquals(
+ specificInternalRow1.genericGet(v),
+ ((SpecificInternalRow) internalRow).genericGet(v));
+ }
+ }
+ }
+
+ public void initSchema() {
+ this.rowType1 =
+ new SeaTunnelRowType(
+ new String[] {
+ "int",
+ "string",
+ "boolean",
+ "float",
+ "double",
+ "byte",
+ "short",
+ "long",
+ "decimal",
+ "date",
+ "timestamp",
+ "null",
+ "array_string",
+ "array_boolean",
+ "array_byte",
+ "array_short",
+ "array_int",
+ "array_long",
+ "array_float",
+ "array_double",
+ "map",
+ "row"
+ },
+ new SeaTunnelDataType[] {
+ BasicType.INT_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.BOOLEAN_TYPE,
+ BasicType.FLOAT_TYPE,
+ BasicType.DOUBLE_TYPE,
+ BasicType.BYTE_TYPE,
+ BasicType.SHORT_TYPE,
+ BasicType.LONG_TYPE,
+ new org.apache.seatunnel.api.table.type.DecimalType(10, 2),
+ LocalTimeType.LOCAL_DATE_TYPE,
+ LocalTimeType.LOCAL_DATE_TIME_TYPE,
+ BasicType.VOID_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.STRING_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.BOOLEAN_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.BYTE_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.SHORT_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.INT_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.LONG_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.FLOAT_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.DOUBLE_ARRAY_TYPE,
+ new org.apache.seatunnel.api.table.type.MapType<>(
+ BasicType.STRING_TYPE, BasicType.STRING_TYPE),
+ new SeaTunnelRowType(
+ new String[] {
+ "int",
+ "string",
+ "boolean",
+ "float",
+ "double",
+ "byte",
+ "short",
+ "long",
+ "decimal",
+ "date",
+ "timestamp",
+ "null",
+ "array_string",
+ "array_boolean",
+ "array_byte",
+ "array_short",
+ "array_int",
+ "array_long",
+ "array_float",
+ "array_double",
+ "map"
+ },
+ new SeaTunnelDataType[] {
+ BasicType.INT_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.BOOLEAN_TYPE,
+ BasicType.FLOAT_TYPE,
+ BasicType.DOUBLE_TYPE,
+ BasicType.BYTE_TYPE,
+ BasicType.SHORT_TYPE,
+ BasicType.LONG_TYPE,
+ new org.apache.seatunnel.api.table.type.DecimalType(10, 2),
+ LocalTimeType.LOCAL_DATE_TYPE,
+ LocalTimeType.LOCAL_DATE_TIME_TYPE,
+ BasicType.VOID_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .STRING_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .BOOLEAN_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .BYTE_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .SHORT_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .INT_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .LONG_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .FLOAT_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .DOUBLE_ARRAY_TYPE,
+ new org.apache.seatunnel.api.table.type.MapType<>(
+ BasicType.STRING_TYPE, BasicType.STRING_TYPE)
+ })
+ });
+
+ this.rowType2 =
+ new SeaTunnelRowType(
+ new String[] {
+ "int",
+ "string",
+ "boolean",
+ "float",
+ "double",
+ "byte",
+ "short",
+ "long",
+ "decimal",
+ "date",
+ "timestamp",
+ "null",
+ "array_string",
+ "array_boolean",
+ "array_byte",
+ "array_short",
+ "array_int",
+ "array_long",
+ "array_float",
+ "array_double",
+ "map",
+ "row"
+ },
+ new SeaTunnelDataType[] {
+ BasicType.INT_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.BOOLEAN_TYPE,
+ BasicType.FLOAT_TYPE,
+ BasicType.DOUBLE_TYPE,
+ BasicType.BYTE_TYPE,
+ BasicType.SHORT_TYPE,
+ BasicType.LONG_TYPE,
+ new org.apache.seatunnel.api.table.type.DecimalType(10, 2),
+ LocalTimeType.LOCAL_DATE_TYPE,
+ LocalTimeType.LOCAL_DATE_TIME_TYPE,
+ BasicType.VOID_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.STRING_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.BOOLEAN_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.BYTE_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.SHORT_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.INT_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.LONG_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.FLOAT_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType.DOUBLE_ARRAY_TYPE,
+ new org.apache.seatunnel.api.table.type.MapType<>(
+ BasicType.STRING_TYPE, BasicType.STRING_TYPE),
+ new SeaTunnelRowType(
+ new String[] {
+ "int",
+ "string",
+ "boolean",
+ "float",
+ "double",
+ "byte",
+ "short",
+ "long",
+ "decimal",
+ "date",
+ "timestamp",
+ "null",
+ "array_string",
+ "array_boolean",
+ "array_byte",
+ "array_short",
+ "array_int",
+ "array_long",
+ "array_float",
+ "array_double",
+ "map"
+ },
+ new SeaTunnelDataType[] {
+ BasicType.INT_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.BOOLEAN_TYPE,
+ BasicType.FLOAT_TYPE,
+ BasicType.DOUBLE_TYPE,
+ BasicType.BYTE_TYPE,
+ BasicType.SHORT_TYPE,
+ BasicType.LONG_TYPE,
+ new org.apache.seatunnel.api.table.type.DecimalType(10, 2),
+ LocalTimeType.LOCAL_DATE_TYPE,
+ LocalTimeType.LOCAL_DATE_TIME_TYPE,
+ BasicType.VOID_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .STRING_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .BOOLEAN_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .BYTE_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .SHORT_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .INT_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .LONG_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .FLOAT_ARRAY_TYPE,
+ org.apache.seatunnel.api.table.type.ArrayType
+ .DOUBLE_ARRAY_TYPE,
+ new org.apache.seatunnel.api.table.type.MapType<>(
+ BasicType.STRING_TYPE, BasicType.STRING_TYPE)
+ })
+ });
+
+ this.rowType3 =
+ new SeaTunnelRowType(
+ new String[] {
+ "int",
+ "string",
+ "float1",
+ "float2",
+ "boolean1",
+ "boolean2",
+ "double",
+ "byte1",
+ "byte2",
+ "long",
+ "short",
+ "decimal",
+ "timestamp",
+ "date",
+ "null"
+ },
+ new SeaTunnelDataType[] {
+ BasicType.INT_TYPE,
+ BasicType.STRING_TYPE,
+ BasicType.FLOAT_TYPE,
+ BasicType.FLOAT_TYPE,
+ BasicType.BOOLEAN_TYPE,
+ BasicType.BOOLEAN_TYPE,
+ BasicType.DOUBLE_TYPE,
+ BasicType.BYTE_TYPE,
+ BasicType.BYTE_TYPE,
+ BasicType.LONG_TYPE,
+ BasicType.SHORT_TYPE,
+ new org.apache.seatunnel.api.table.type.DecimalType(10, 2),
+ LocalTimeType.LOCAL_DATE_TIME_TYPE,
+ LocalTimeType.LOCAL_DATE_TYPE,
+ BasicType.VOID_TYPE,
+ });
+
+ catalogTable1 = CatalogTableUtil.getCatalogTable("test", "test", "test", "test1", rowType1);
+
+ catalogTable2 = CatalogTableUtil.getCatalogTable("test", "test", "test", "test2", rowType2);
+
+ catalogTable3 = CatalogTableUtil.getCatalogTable("test", "test", "test", "test3", rowType3);
+
+ StructType structType =
+ new StructType()
+ .add("int", IntegerType)
+ .add("string", StringType)
+ .add("boolean", BooleanType)
+ .add("float", FloatType)
+ .add("double", DoubleType)
+ .add("byte", ByteType)
+ .add("short", ShortType)
+ .add("long", LongType)
+ .add("decimal", new DecimalType(10, 2))
+ .add("date", DateType)
+ .add("timestamp", TimestampType)
+ .add("null", NullType)
+ .add("array_string", new ArrayType(StringType, true))
+ .add("array_boolean", new ArrayType(BooleanType, true))
+ .add("array_byte", new ArrayType(ByteType, true))
+ .add("array_short", new ArrayType(ShortType, true))
+ .add("array_int", new ArrayType(IntegerType, true))
+ .add("array_long", new ArrayType(LongType, true))
+ .add("array_float", new ArrayType(FloatType, true))
+ .add("array_double", new ArrayType(DoubleType, true))
+ .add("map", new MapType(StringType, StringType, true));
+
+ structType1 =
+ new StructType()
+ .add(ROW_KIND_FIELD, DataTypes.ByteType)
+ .add(TABLE_ID, DataTypes.StringType)
+ .add("column0", IntegerType)
+ .add("column1", StringType)
+ .add("column2", BooleanType)
+ .add("column3", FloatType)
+ .add("column4", DoubleType)
+ .add("column5", ByteType)
+ .add("column6", ShortType)
+ .add("column7", LongType)
+ .add("column8", new DecimalType(10, 2))
+ .add("column9", DateType)
+ .add("column10", TimestampType)
+ .add("column11", NullType)
+ .add("column12", new ArrayType(StringType, true))
+ .add("column13", new ArrayType(BooleanType, true))
+ .add("column14", new ArrayType(ByteType, true))
+ .add("column15", new ArrayType(ShortType, true))
+ .add("column16", new ArrayType(IntegerType, true))
+ .add("column17", new ArrayType(LongType, true))
+ .add("column18", new ArrayType(FloatType, true))
+ .add("column19", new ArrayType(DoubleType, true))
+ .add("column20", new MapType(StringType, StringType, true))
+ .add("column21", structType);
+
+ structType2 =
+ new StructType()
+ .add(ROW_KIND_FIELD, DataTypes.ByteType)
+ .add(TABLE_ID, DataTypes.StringType)
+ .add("column0", IntegerType)
+ .add("column1", StringType)
+ .add("column2", BooleanType)
+ .add("column3", FloatType)
+ .add("column4", DoubleType)
+ .add("column5", ByteType)
+ .add("column6", ShortType)
+ .add("column7", LongType)
+ .add("column8", new DecimalType(10, 2))
+ .add("column9", DateType)
+ .add("column10", TimestampType)
+ .add("column11", NullType)
+ .add("column12", new ArrayType(StringType, true))
+ .add("column13", new ArrayType(BooleanType, true))
+ .add("column14", new ArrayType(ByteType, true))
+ .add("column15", new ArrayType(ShortType, true))
+ .add("column16", new ArrayType(IntegerType, true))
+ .add("column17", new ArrayType(LongType, true))
+ .add("column18", new ArrayType(FloatType, true))
+ .add("column19", new ArrayType(DoubleType, true))
+ .add("column20", new MapType(StringType, StringType, true))
+ .add("column21", structType)
+ .add("column22", FloatType)
+ .add("column23", BooleanType)
+ .add("column24", ByteType);
+
+ structType3 =
+ new StructType()
+ .add(ROW_KIND_FIELD, DataTypes.ByteType)
+ .add(TABLE_ID, DataTypes.StringType)
+ .add("int", IntegerType)
+ .add("string", StringType)
+ .add("boolean", BooleanType)
+ .add("float", FloatType)
+ .add("double", DoubleType)
+ .add("byte", ByteType)
+ .add("short", ShortType)
+ .add("long", LongType)
+ .add("decimal", new DecimalType(10, 2))
+ .add("date", DateType)
+ .add("timestamp", TimestampType)
+ .add("null", NullType)
+ .add("array_string", new ArrayType(StringType, true))
+ .add("array_boolean", new ArrayType(BooleanType, true))
+ .add("array_byte", new ArrayType(ByteType, true))
+ .add("array_short", new ArrayType(ShortType, true))
+ .add("array_int", new ArrayType(IntegerType, true))
+ .add("array_long", new ArrayType(LongType, true))
+ .add("array_float", new ArrayType(FloatType, true))
+ .add("array_double", new ArrayType(DoubleType, true))
+ .add("map", new MapType(StringType, StringType, true))
+ .add("row", structType);
+ }
+
+ public void initData() {
+
+ SeaTunnelRow row1 =
+ new SeaTunnelRow(
+ new Object[] {
+ 233,
+ "string3",
+ true,
+ 231.1f,
+ 3533.33,
+ (byte) 7,
+ (short) 2,
+ Long.MAX_VALUE - 2,
+ new BigDecimal("65.55"),
+ LocalDate.parse("2001-01-01"),
+ LocalDateTime.parse("2031-01-01T00:00:00"),
+ null,
+ new Object[] {"string1fsa", "stringdsa2", "strfdsaing3"},
+ new Object[] {false, true, true},
+ new Object[] {(byte) 6, (byte) 2, (byte) 1},
+ new Object[] {(short) 7, (short) 8, (short) 9},
+ new Object[] {3, 77, 22},
+ new Object[] {143L, 642L, 533L},
+ new Object[] {24.1f, 54.2f, 1.3f},
+ new Object[] {431.11, 2422.22, 3243.33},
+ new HashMap<String, String>() {
+ {
+ put("keyfs1", "valfdsue1");
+ put("kedfasy2", "vafdslue2");
+ put("kefdsay3", "vfdasalue3");
+ }
+ }
+ });
+
+ seaTunnelRow1 =
+ new SeaTunnelRow(
+ new Object[] {
+ 233,
+ "string3",
+ true,
+ 231.1f,
+ 3533.33,
+ (byte) 7,
+ (short) 2,
+ Long.MAX_VALUE - 2,
+ new BigDecimal("65.55"),
+ LocalDate.parse("2001-01-01"),
+ LocalDateTime.parse("2031-01-01T00:00:00"),
+ null,
+ new Object[] {"string1fsa", "stringdsa2", "strfdsaing3"},
+ new Object[] {false, true, true},
+ new Object[] {(byte) 6, (byte) 2, (byte) 1},
+ new Object[] {(short) 7, (short) 8, (short) 9},
+ new Object[] {3, 77, 22},
+ new Object[] {143L, 642L, 533L},
+ new Object[] {24.1f, 54.2f, 1.3f},
+ new Object[] {431.11, 2422.22, 3243.33},
+ new HashMap<String, String>() {
+ {
+ put("keyfs1", "valfdsue1");
+ put("kedfasy2", "vafdslue2");
+ put("kefdsay3", "vfdasalue3");
+ }
+ },
+ row1
+ });
+ seaTunnelRow1.setRowKind(RowKind.INSERT);
+ seaTunnelRow1.setTableId("test.test.test1");
+
+ MutableValue[] mutableValues = new MutableValue[21];
+ mutableValues[0] = new MutableInt();
+ mutableValues[0].update(233);
+ mutableValues[1] = new MutableAny();
+ mutableValues[1].update(UTF8String.fromString("string3"));
+ mutableValues[2] = new MutableBoolean();
+ mutableValues[2].update(true);
+ mutableValues[3] = new MutableFloat();
+ mutableValues[3].update(231.1f);
+ mutableValues[4] = new MutableDouble();
+ mutableValues[4].update(3533.33);
+ mutableValues[5] = new MutableByte();
+ mutableValues[5].update((byte) 7);
+ mutableValues[6] = new MutableShort();
+ mutableValues[6].update((short) 2);
+ mutableValues[7] = new MutableLong();
+ mutableValues[7].update(Long.MAX_VALUE - 2);
+ mutableValues[8] = new MutableAny();
+ mutableValues[8].update(Decimal.apply(new BigDecimal("65.55")));
+ mutableValues[9] = new MutableInt();
+ mutableValues[9].update((int) LocalDate.parse("2001-01-01").toEpochDay());
+ mutableValues[10] = new MutableAny();
+ mutableValues[10].update(
+ InstantConverterUtils.toEpochMicro(
+ Timestamp.valueOf(LocalDateTime.parse("2031-01-01T00:00:00")).toInstant()));
+ mutableValues[11] = new MutableAny();
+ mutableValues[12] = new MutableAny();
+ mutableValues[12].update(
+ ArrayData.toArrayData(
+ new Object[] {
+ UTF8String.fromString("string1fsa"),
+ UTF8String.fromString("stringdsa2"),
+ UTF8String.fromString("strfdsaing3")
+ }));
+
+ mutableValues[13] = new MutableAny();
+ mutableValues[13].update(ArrayData.toArrayData(new Object[] {false, true, true}));
+
+ mutableValues[14] = new MutableAny();
+ mutableValues[14].update(
+ ArrayData.toArrayData(new Object[] {(byte) 6, (byte) 2, (byte) 1}));
+
+ mutableValues[15] = new MutableAny();
+ mutableValues[15].update(
+ ArrayData.toArrayData(new Object[] {(short) 7, (short) 8, (short) 9}));
+
+ mutableValues[16] = new MutableAny();
+ mutableValues[16].update(ArrayData.toArrayData(new Object[] {3, 77, 22}));
+
+ mutableValues[17] = new MutableAny();
+ mutableValues[17].update(ArrayData.toArrayData(new Object[] {143L, 642L, 533L}));
+
+ mutableValues[18] = new MutableAny();
+ mutableValues[18].update(ArrayData.toArrayData(new Object[] {24.1f, 54.2f, 1.3f}));
+
+ mutableValues[19] = new MutableAny();
+ mutableValues[19].update(ArrayData.toArrayData(new Object[] {431.11, 2422.22, 3243.33}));
+
+ mutableValues[20] = new MutableAny();
+ mutableValues[20].update(
+ ArrayBasedMapData.apply(
+ new Object[] {
+ UTF8String.fromString("kefdsay3"),
+ UTF8String.fromString("keyfs1"),
+ UTF8String.fromString("kedfasy2")
+ },
+ new Object[] {
+ UTF8String.fromString("vfdasalue3"),
+ UTF8String.fromString("valfdsue1"),
+ UTF8String.fromString("vafdslue2")
+ }));
+
+ SpecificInternalRow specificInternalRow = new SpecificInternalRow(mutableValues);
+
+ MutableValue[] mutableValues1 = new MutableValue[24];
+
+ mutableValues1[0] = new MutableByte();
+ mutableValues1[0].update(RowKind.INSERT.toByteValue());
+ mutableValues1[1] = new MutableAny();
+ mutableValues1[1].update(UTF8String.fromString("test.test.test1"));
+ mutableValues1[2] = new MutableInt();
+ mutableValues1[2].update(233);
+ mutableValues1[3] = new MutableAny();
+ mutableValues1[3].update(UTF8String.fromString("string3"));
+ mutableValues1[4] = new MutableBoolean();
+ mutableValues1[4].update(true);
+ mutableValues1[5] = new MutableFloat();
+ mutableValues1[5].update(231.1f);
+ mutableValues1[6] = new MutableDouble();
+ mutableValues1[6].update(3533.33);
+ mutableValues1[7] = new MutableByte();
+ mutableValues1[7].update((byte) 7);
+ mutableValues1[8] = new MutableShort();
+ mutableValues1[8].update((short) 2);
+ mutableValues1[9] = new MutableLong();
+ mutableValues1[9].update(Long.MAX_VALUE - 2);
+ mutableValues1[10] = new MutableAny();
+ mutableValues1[10].update(Decimal.apply(new BigDecimal("65.55")));
+ mutableValues1[11] = new MutableInt();
+ mutableValues1[11].update((int) LocalDate.parse("2001-01-01").toEpochDay());
+ mutableValues1[12] = new MutableAny();
+ mutableValues1[12].update(
+ InstantConverterUtils.toEpochMicro(
+ Timestamp.valueOf(LocalDateTime.parse("2031-01-01T00:00:00")).toInstant()));
+ mutableValues1[13] = new MutableAny();
+ mutableValues1[14] = new MutableAny();
+ mutableValues1[14].update(
+ ArrayData.toArrayData(
+ new Object[] {
+ UTF8String.fromString("string1fsa"),
+ UTF8String.fromString("stringdsa2"),
+ UTF8String.fromString("strfdsaing3")
+ }));
+
+ mutableValues1[15] = new MutableAny();
+ mutableValues1[15].update(ArrayData.toArrayData(new Object[] {false, true, true}));
+
+ mutableValues1[16] = new MutableAny();
+ mutableValues1[16].update(
+ ArrayData.toArrayData(new Object[] {(byte) 6, (byte) 2, (byte) 1}));
+
+ mutableValues1[17] = new MutableAny();
+ mutableValues1[17].update(
+ ArrayData.toArrayData(new Object[] {(short) 7, (short) 8, (short) 9}));
+
+ mutableValues1[18] = new MutableAny();
+ mutableValues1[18].update(ArrayData.toArrayData(new Object[] {3, 77, 22}));
+
+ mutableValues1[19] = new MutableAny();
+ mutableValues1[19].update(ArrayData.toArrayData(new Object[] {143L, 642L, 533L}));
+
+ mutableValues1[20] = new MutableAny();
+ mutableValues1[20].update(ArrayData.toArrayData(new Object[] {24.1f, 54.2f, 1.3f}));
+
+ mutableValues1[21] = new MutableAny();
+ mutableValues1[21].update(ArrayData.toArrayData(new Object[] {431.11, 2422.22, 3243.33}));
+
+ mutableValues1[22] = new MutableAny();
+ mutableValues1[22].update(
+ ArrayBasedMapData.apply(
+ new Object[] {
+ UTF8String.fromString("kefdsay3"),
+ UTF8String.fromString("keyfs1"),
+ UTF8String.fromString("kedfasy2")
+ },
+ new Object[] {
+ UTF8String.fromString("vfdasalue3"),
+ UTF8String.fromString("valfdsue1"),
+ UTF8String.fromString("vafdslue2")
+ }));
+
+ mutableValues1[23] = new MutableAny();
+ mutableValues1[23].update(specificInternalRow);
+
+ specificInternalRow1 = new SpecificInternalRow(mutableValues1);
+
+ MutableValue[] mutableValues2 = new MutableValue[27];
+
+ for (int i = 0; i < mutableValues1.length; i++) {
+ mutableValues2[i] = mutableValues1[i].copy();
+ }
+ mutableValues2[24] = new MutableAny();
+ mutableValues2[25] = new MutableAny();
+ mutableValues2[26] = new MutableAny();
+
+ specificInternalRow2 = new SpecificInternalRow(mutableValues2);
+
+ seaTunnelRow3 =
+ new SeaTunnelRow(
+ new Object[] {
+ 233,
+ "string3",
+ 231.1f,
+ 231.1f,
+ true,
+ true,
+ 3533.33,
+ (byte) 7,
+ (byte) 7,
+ Long.MAX_VALUE - 2,
+ (short) 2,
+ new BigDecimal("65.55"),
+ LocalDateTime.parse("2031-01-01T00:00:00"),
+ LocalDate.parse("2001-01-01"),
+ null
+ });
+ seaTunnelRow3.setRowKind(RowKind.INSERT);
+ seaTunnelRow3.setTableId("test.test.test3");
+
+ // [0, 1, 3, 22, 2, 23, 4, 5, 24, 7, 6, 8, 10, 9, 11]
+ MutableValue[] mutableValues3 = new MutableValue[27];
+ mutableValues3[0] = new MutableByte();
+ mutableValues3[0].update(RowKind.INSERT.toByteValue());
+ mutableValues3[1] = new MutableAny();
+ mutableValues3[1].update(UTF8String.fromString("test.test.test3"));
+
+ mutableValues3[2] = new MutableInt();
+ mutableValues3[2].update(233);
+
+ mutableValues3[3] = new MutableAny();
+ mutableValues3[3].update(UTF8String.fromString("string3"));
+
+ mutableValues3[5] = new MutableFloat();
+ mutableValues3[5].update(231.1f);
+
+ mutableValues3[24] = new MutableFloat();
+ mutableValues3[24].update(231.1f);
+
+ mutableValues3[4] = new MutableBoolean();
+ mutableValues3[4].update(true);
+
+ mutableValues3[25] = new MutableBoolean();
+ mutableValues3[25].update(true);
+
+ mutableValues3[6] = new MutableDouble();
+ mutableValues3[6].update(3533.33);
+
+ mutableValues3[7] = new MutableByte();
+ mutableValues3[7].update((byte) 7);
+
+ mutableValues3[26] = new MutableByte();
+ mutableValues3[26].update((byte) 7);
+
+ mutableValues3[9] = new MutableLong();
+ mutableValues3[9].update(Long.MAX_VALUE - 2);
+
+ mutableValues3[8] = new MutableShort();
+ mutableValues3[8].update((short) 2);
+
+ mutableValues3[10] = new MutableAny();
+ mutableValues3[10].update(Decimal.apply(new BigDecimal("65.55")));
+
+ mutableValues3[12] = new MutableLong();
+ mutableValues3[12].update(
+ InstantConverterUtils.toEpochMicro(
+ Timestamp.valueOf(LocalDateTime.parse("2031-01-01T00:00:00")).toInstant()));
+
+ mutableValues3[11] = new MutableInt();
+ mutableValues3[11].update((int) LocalDate.parse("2001-01-01").toEpochDay());
+
+ for (int i = 0; i < mutableValues3.length; i++) {
+ if (mutableValues3[i] == null) {
+ mutableValues3[i] = new MutableAny();
+ }
+ }
+ specificInternalRow3 = new SpecificInternalRow(mutableValues3);
+ }
+}