You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ar...@apache.org on 2025/01/16 18:45:26 UTC

(seatunnel-web) branch main updated: [Feature] Support multi-table sink (#265)

This is an automated email from the ASF dual-hosted git repository.

arshad pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/seatunnel-web.git


The following commit(s) were added to refs/heads/main by this push:
     new 2c309aa6 [Feature] Support multi-table sink (#265)
2c309aa6 is described below

commit 2c309aa6bbae71ee97e6c8156f5406d6a2c16af9
Author: Jast <sh...@apache.org>
AuthorDate: Fri Jan 17 02:45:20 2025 +0800

    [Feature] Support multi-table sink (#265)
---
 .../app/service/impl/JobInstanceServiceImpl.java   |  8 +++--
 .../app/service/impl/JobTaskServiceImpl.java       | 40 +++++++++++++++-------
 2 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java
index 8d0b7957..6b1e97af 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java
@@ -177,7 +177,11 @@ public class JobInstanceServiceImpl extends SeatunnelBaseServiceImpl
         Map<String, List<Config>> sinkMap = new LinkedHashMap<>();
         Map<String, JobLine> inputLines =
                 lines.stream()
-                        .collect(Collectors.toMap(JobLine::getInputPluginId, Function.identity()));
+                        .collect(
+                                Collectors.toMap(
+                                        JobLine::getInputPluginId,
+                                        Function.identity(),
+                                        (existing, replacement) -> existing));
         Map<String, JobLine> targetLines =
                 lines.stream()
                         .collect(Collectors.toMap(JobLine::getTargetPluginId, Function.identity()));
@@ -484,7 +488,7 @@ public class JobInstanceServiceImpl extends SeatunnelBaseServiceImpl
 
     private Config addTableName(String tableName, JobLine jobLine, Config config) {
         return config.withValue(
-                tableName, ConfigValueFactory.fromAnyRef("Table" + jobLine.getId()));
+                tableName, ConfigValueFactory.fromAnyRef("Table" + jobLine.getInputPluginId()));
     }
 
     private Config filterEmptyValue(Config config) {
diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java
index c5cef317..26ac6bcb 100644
--- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java
+++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java
@@ -289,15 +289,25 @@ public class JobTaskServiceImpl extends SeatunnelBaseServiceImpl implements IJob
         Map<String, PluginConfig> pluginMap =
                 taskInfo.getPlugins().stream()
                         .collect(Collectors.toMap(PluginConfig::getPluginId, Function.identity()));
-        Map<String, String> edgeMap =
+        Map<String, List<String>> edgeMap =
                 taskInfo.getEdges().stream()
-                        .collect(Collectors.toMap(Edge::getInputPluginId, Edge::getTargetPluginId));
+                        .collect(
+                                Collectors.groupingBy(
+                                        Edge::getInputPluginId,
+                                        Collectors.mapping(
+                                                Edge::getTargetPluginId, Collectors.toList())));
 
         for (PluginConfig config : source) {
-            PluginConfig nextConfig = pluginMap.get(edgeMap.get(config.getPluginId()));
-            JobTaskCheckRes res = checkNextTaskSchema(config, nextConfig, pluginMap, edgeMap);
-            if (res != null) {
-                return res;
+            List<String> nextConfigs = edgeMap.get(config.getPluginId());
+            if (nextConfigs != null) {
+                for (String nextConfigId : nextConfigs) {
+                    PluginConfig nextConfig = pluginMap.get(nextConfigId);
+                    JobTaskCheckRes res =
+                            checkNextTaskSchema(config, nextConfig, pluginMap, edgeMap);
+                    if (res != null) {
+                        return res;
+                    }
+                }
             }
         }
         return null;
@@ -307,7 +317,7 @@ public class JobTaskServiceImpl extends SeatunnelBaseServiceImpl implements IJob
             PluginConfig config,
             PluginConfig nextConfig,
             Map<String, PluginConfig> pluginMap,
-            Map<String, String> edgeMap)
+            Map<String, List<String>> edgeMap)
             throws IOException {
         Map<String, Object> options = nextConfig.getTransformOptions();
         if (options != null && !options.isEmpty()) {
@@ -384,12 +394,16 @@ public class JobTaskServiceImpl extends SeatunnelBaseServiceImpl implements IJob
                 }
             }
         }
-        if (edgeMap.containsKey(nextConfig.getPluginId())) {
-            return checkNextTaskSchema(
-                    nextConfig,
-                    pluginMap.get(edgeMap.get(nextConfig.getPluginId())),
-                    pluginMap,
-                    edgeMap);
+        List<String> nextConfigIds = edgeMap.get(nextConfig.getPluginId());
+        if (nextConfigIds != null) {
+            for (String nextConfigId : nextConfigIds) {
+                JobTaskCheckRes res =
+                        checkNextTaskSchema(
+                                nextConfig, pluginMap.get(nextConfigId), pluginMap, edgeMap);
+                if (res != null) {
+                    return res;
+                }
+            }
         }
         return null;
     }