You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ar...@apache.org on 2025/01/16 18:45:26 UTC
(seatunnel-web) branch main updated: [Feature] Support multi-table sink (#265)
This is an automated email from the ASF dual-hosted git repository.
arshad pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/seatunnel-web.git
The following commit(s) were added to refs/heads/main by this push:
new 2c309aa6 [Feature] Support multi-table sink (#265)
2c309aa6 is described below
commit 2c309aa6bbae71ee97e6c8156f5406d6a2c16af9
Author: Jast <sh...@apache.org>
AuthorDate: Fri Jan 17 02:45:20 2025 +0800
[Feature] Support multi-table sink (#265)
---
.../app/service/impl/JobInstanceServiceImpl.java | 8 +++--
.../app/service/impl/JobTaskServiceImpl.java | 40 +++++++++++++++-------
2 files changed, 33 insertions(+), 15 deletions(-)
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java
index 8d0b7957..6b1e97af 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java
@@ -177,7 +177,11 @@ public class JobInstanceServiceImpl extends SeatunnelBaseServiceImpl
Map<String, List<Config>> sinkMap = new LinkedHashMap<>();
Map<String, JobLine> inputLines =
lines.stream()
- .collect(Collectors.toMap(JobLine::getInputPluginId, Function.identity()));
+ .collect(
+ Collectors.toMap(
+ JobLine::getInputPluginId,
+ Function.identity(),
+ (existing, replacement) -> existing));
Map<String, JobLine> targetLines =
lines.stream()
.collect(Collectors.toMap(JobLine::getTargetPluginId, Function.identity()));
@@ -484,7 +488,7 @@ public class JobInstanceServiceImpl extends SeatunnelBaseServiceImpl
private Config addTableName(String tableName, JobLine jobLine, Config config) {
return config.withValue(
- tableName, ConfigValueFactory.fromAnyRef("Table" + jobLine.getId()));
+ tableName, ConfigValueFactory.fromAnyRef("Table" + jobLine.getInputPluginId()));
}
private Config filterEmptyValue(Config config) {
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java
index c5cef317..26ac6bcb 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java
@@ -289,15 +289,25 @@ public class JobTaskServiceImpl extends SeatunnelBaseServiceImpl implements IJob
Map<String, PluginConfig> pluginMap =
taskInfo.getPlugins().stream()
.collect(Collectors.toMap(PluginConfig::getPluginId, Function.identity()));
- Map<String, String> edgeMap =
+ Map<String, List<String>> edgeMap =
taskInfo.getEdges().stream()
- .collect(Collectors.toMap(Edge::getInputPluginId, Edge::getTargetPluginId));
+ .collect(
+ Collectors.groupingBy(
+ Edge::getInputPluginId,
+ Collectors.mapping(
+ Edge::getTargetPluginId, Collectors.toList())));
for (PluginConfig config : source) {
- PluginConfig nextConfig = pluginMap.get(edgeMap.get(config.getPluginId()));
- JobTaskCheckRes res = checkNextTaskSchema(config, nextConfig, pluginMap, edgeMap);
- if (res != null) {
- return res;
+ List<String> nextConfigs = edgeMap.get(config.getPluginId());
+ if (nextConfigs != null) {
+ for (String nextConfigId : nextConfigs) {
+ PluginConfig nextConfig = pluginMap.get(nextConfigId);
+ JobTaskCheckRes res =
+ checkNextTaskSchema(config, nextConfig, pluginMap, edgeMap);
+ if (res != null) {
+ return res;
+ }
+ }
}
}
return null;
@@ -307,7 +317,7 @@ public class JobTaskServiceImpl extends SeatunnelBaseServiceImpl implements IJob
PluginConfig config,
PluginConfig nextConfig,
Map<String, PluginConfig> pluginMap,
- Map<String, String> edgeMap)
+ Map<String, List<String>> edgeMap)
throws IOException {
Map<String, Object> options = nextConfig.getTransformOptions();
if (options != null && !options.isEmpty()) {
@@ -384,12 +394,16 @@ public class JobTaskServiceImpl extends SeatunnelBaseServiceImpl implements IJob
}
}
}
- if (edgeMap.containsKey(nextConfig.getPluginId())) {
- return checkNextTaskSchema(
- nextConfig,
- pluginMap.get(edgeMap.get(nextConfig.getPluginId())),
- pluginMap,
- edgeMap);
+ List<String> nextConfigIds = edgeMap.get(nextConfig.getPluginId());
+ if (nextConfigIds != null) {
+ for (String nextConfigId : nextConfigIds) {
+ JobTaskCheckRes res =
+ checkNextTaskSchema(
+ nextConfig, pluginMap.get(nextConfigId), pluginMap, edgeMap);
+ if (res != null) {
+ return res;
+ }
+ }
}
return null;
}