You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by fa...@apache.org on 2022/07/21 11:48:23 UTC

[incubator-seatunnel] branch dev updated: [Core][Starter] Fix connector v2 can't deserialize on spark (#2221)

This is an automated email from the ASF dual-hosted git repository.

fanjia pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git


The following commit(s) were added to refs/heads/dev by this push:
     new cf5327cd4 [Core][Starter] Fix connector v2 can't deserialize on spark (#2221)
cf5327cd4 is described below

commit cf5327cd47da8c95eae06b8a086cb119db44b5ea
Author: Hisoka <fa...@qq.com>
AuthorDate: Thu Jul 21 19:48:17 2022 +0800

    [Core][Starter] Fix connector v2 can't deserialize on spark (#2221)
    
    * [Core][Starter] Fix connector v2 can't deserialize on spark
    
    * [Core][Starter] Add SerializationUtils Unit Test
    
    * [Core][Starter] Add SerializationUtils Unit Test
    
    * [Core][Flink] Fixed FlinkEnvironment registerPlugin logic both old and new api
---
 .../apache/seatunnel/flink/FlinkEnvironment.java   |  40 +--
 .../common/utils/SerializationException.java       |  68 +++++
 .../seatunnel/common/utils/SerializationUtils.java |  38 ++-
 .../common/utils/SerializationUtilsTest.java       |  74 ++++++
 .../core/starter/flink/env/FlinkEnvironment.java   | 286 ---------------------
 .../seatunnel/core/starter/spark/SparkStarter.java |  21 +-
 6 files changed, 205 insertions(+), 322 deletions(-)

diff --git a/seatunnel-apis/seatunnel-api-flink/src/main/java/org/apache/seatunnel/flink/FlinkEnvironment.java b/seatunnel-apis/seatunnel-api-flink/src/main/java/org/apache/seatunnel/flink/FlinkEnvironment.java
index d6e050ec1..49c97beb6 100644
--- a/seatunnel-apis/seatunnel-api-flink/src/main/java/org/apache/seatunnel/flink/FlinkEnvironment.java
+++ b/seatunnel-apis/seatunnel-api-flink/src/main/java/org/apache/seatunnel/flink/FlinkEnvironment.java
@@ -123,31 +123,31 @@ public class FlinkEnvironment implements RuntimeEnv {
     @Override
     public void registerPlugin(List<URL> pluginPaths) {
         pluginPaths.forEach(url -> LOGGER.info("register plugins : {}", url));
-        Configuration configuration;
+        List<Configuration> configurations = new ArrayList<>();
         try {
-            if (isStreaming()) {
-                configuration =
-                        (Configuration) Objects.requireNonNull(ReflectionUtils.getDeclaredMethod(StreamExecutionEnvironment.class,
-                                "getConfiguration")).orElseThrow(() -> new RuntimeException("can't find " +
-                                "method: getConfiguration")).invoke(this.environment);
-            } else {
-                configuration = batchEnvironment.getConfiguration();
+            configurations.add((Configuration) Objects.requireNonNull(ReflectionUtils.getDeclaredMethod(StreamExecutionEnvironment.class,
+                    "getConfiguration")).orElseThrow(() -> new RuntimeException("can't find " +
+                    "method: getConfiguration")).invoke(this.environment));
+            if (!isStreaming()) {
+                configurations.add(batchEnvironment.getConfiguration());
             }
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
-        List<String> jars = configuration.get(PipelineOptions.JARS);
-        if (jars == null) {
-            jars = new ArrayList<>();
-        }
-        jars.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
-        configuration.set(PipelineOptions.JARS, jars.stream().distinct().collect(Collectors.toList()));
-        List<String> classpath = configuration.get(PipelineOptions.CLASSPATHS);
-        if (classpath == null) {
-            classpath = new ArrayList<>();
-        }
-        classpath.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
-        configuration.set(PipelineOptions.CLASSPATHS, classpath.stream().distinct().collect(Collectors.toList()));
+        configurations.forEach(configuration -> {
+            List<String> jars = configuration.get(PipelineOptions.JARS);
+            if (jars == null) {
+                jars = new ArrayList<>();
+            }
+            jars.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
+            configuration.set(PipelineOptions.JARS, jars.stream().distinct().collect(Collectors.toList()));
+            List<String> classpath = configuration.get(PipelineOptions.CLASSPATHS);
+            if (classpath == null) {
+                classpath = new ArrayList<>();
+            }
+            classpath.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
+            configuration.set(PipelineOptions.CLASSPATHS, classpath.stream().distinct().collect(Collectors.toList()));
+        });
     }
 
     public StreamExecutionEnvironment getStreamExecutionEnvironment() {
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationException.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationException.java
new file mode 100644
index 000000000..33ca29a6f
--- /dev/null
+++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationException.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.common.utils;
+
+public class SerializationException extends RuntimeException {
+
+    /**
+     * Required for serialization support.
+     *
+     * @see java.io.Serializable
+     */
+    private static final long serialVersionUID = 2263144814025689516L;
+
+    /**
+     * <p>Constructs a new {@code SerializationException} without specified
+     * detail message.</p>
+     */
+    public SerializationException() {
+    }
+
+    /**
+     * <p>Constructs a new {@code SerializationException} with specified
+     * detail message.</p>
+     *
+     * @param msg The error message.
+     */
+    public SerializationException(final String msg) {
+        super(msg);
+    }
+
+    /**
+     * <p>Constructs a new {@code SerializationException} with specified
+     * nested {@code Throwable}.</p>
+     *
+     * @param cause The {@code Exception} or {@code Error}
+     *              that caused this exception to be thrown.
+     */
+    public SerializationException(final Throwable cause) {
+        super(cause);
+    }
+
+    /**
+     * <p>Constructs a new {@code SerializationException} with specified
+     * detail message and nested {@code Throwable}.</p>
+     *
+     * @param msg   The error message.
+     * @param cause The {@code Exception} or {@code Error}
+     *              that caused this exception to be thrown.
+     */
+    public SerializationException(final String msg, final Throwable cause) {
+        super(msg, cause);
+    }
+}
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationUtils.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationUtils.java
index 43dead167..7e721617e 100644
--- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationUtils.java
+++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationUtils.java
@@ -20,30 +20,60 @@ package org.apache.seatunnel.common.utils;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang3.StringUtils;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.ObjectStreamClass;
 import java.io.Serializable;
 
 public class SerializationUtils {
 
     public static String objectToString(Serializable obj) {
         if (obj != null) {
-            return Base64.encodeBase64String(org.apache.commons.lang3.SerializationUtils.serialize(obj));
+            return Base64.encodeBase64String(serialize(obj));
         }
         return null;
     }
 
     public static <T extends Serializable> T stringToObject(String str) {
         if (StringUtils.isNotEmpty(str)) {
-            return org.apache.commons.lang3.SerializationUtils.deserialize(Base64.decodeBase64(str));
+            return deserialize(Base64.decodeBase64(str));
         }
         return null;
     }
 
+    @SuppressWarnings("checkstyle:MagicNumber")
     public static <T extends Serializable> byte[] serialize(T obj) {
-        return org.apache.commons.lang3.SerializationUtils.serialize(obj);
+        try (ByteArrayOutputStream b = new ByteArrayOutputStream(512);
+             ObjectOutputStream out = new ObjectOutputStream(b)) {
+            out.writeObject(obj);
+            return b.toByteArray();
+        } catch (final IOException ex) {
+            throw new SerializationException(ex);
+        }
+
     }
 
     public static <T extends Serializable> T deserialize(byte[] bytes) {
-        return org.apache.commons.lang3.SerializationUtils.deserialize(bytes);
+        try (ByteArrayInputStream s = new ByteArrayInputStream(bytes);
+             ObjectInputStream in = new ObjectInputStream(s) {
+                 @Override
+                 protected Class<?> resolveClass(ObjectStreamClass desc) throws IOException, ClassNotFoundException {
+                     // make sure use current thread classloader
+                     ClassLoader cl = Thread.currentThread().getContextClassLoader();
+                     if (cl == null) {
+                         return super.resolveClass(desc);
+                     }
+                     return Class.forName(desc.getName(), false, cl);
+                 }
+             }) {
+            @SuppressWarnings("unchecked") final T obj = (T) in.readObject();
+            return obj;
+        } catch (final ClassNotFoundException | IOException ex) {
+            throw new SerializationException(ex);
+        }
     }
 
 }
diff --git a/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/SerializationUtilsTest.java b/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/SerializationUtilsTest.java
new file mode 100644
index 000000000..ad0e80cea
--- /dev/null
+++ b/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/SerializationUtilsTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.common.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+@SuppressWarnings("checkstyle:RegexpSingleline")
+public class SerializationUtilsTest {
+
+    @Test
+    public void testObjectToString() {
+
+        HashMap<String, String> data = new HashMap<>();
+        data.put("key1", "value1");
+        data.put("seatunnelTest", "apache SeaTunnel");
+        data.put("中 文", "Apache Asia");
+        String configStr = SerializationUtils.objectToString(data);
+        Assert.assertNotNull(configStr);
+
+        HashMap<String, String> dataAfter = SerializationUtils.stringToObject(configStr);
+
+        Assert.assertEquals(dataAfter, data);
+
+        data.put("key2", "");
+        Assert.assertNotEquals(dataAfter, data);
+
+    }
+
+    @Test
+    public void testByteToObject() {
+
+        HashMap<String, String> data = new HashMap<>();
+        data.put("key1", "value1");
+        data.put("seatunnelTest", "apache SeaTunnel");
+        data.put("中 文", "Apache Asia");
+
+        ArrayList<HashMap<String, String>> array = new ArrayList<>();
+        array.add(data);
+        HashMap<String, String> data2 = new HashMap<>();
+        data2.put("Apache Asia", "中 文");
+        data2.put("value1", "key1");
+        data2.put("apache SeaTunnel", "seatunnelTest");
+        array.add(data2);
+
+        byte[] result = SerializationUtils.serialize(array);
+
+        ArrayList<HashMap<String, String>> array2 = SerializationUtils.deserialize(result);
+
+        Assert.assertEquals(array2, array);
+
+        Assert.assertThrows(SerializationException.class, () -> SerializationUtils.deserialize(new byte[]{1, 0, 1}));
+
+    }
+
+}
diff --git a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/env/FlinkEnvironment.java b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/env/FlinkEnvironment.java
deleted file mode 100644
index f9de514b0..000000000
--- a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/env/FlinkEnvironment.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.seatunnel.core.starter.flink.env;
-
-import org.apache.seatunnel.apis.base.env.RuntimeEnv;
-import org.apache.seatunnel.common.config.CheckResult;
-import org.apache.seatunnel.common.constants.JobMode;
-import org.apache.seatunnel.common.utils.ReflectionUtils;
-import org.apache.seatunnel.flink.util.ConfigKeyName;
-import org.apache.seatunnel.flink.util.EnvironmentUtil;
-
-import org.apache.seatunnel.shade.com.typesafe.config.Config;
-
-import org.apache.flink.api.common.RuntimeExecutionMode;
-import org.apache.flink.api.common.time.Time;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.PipelineOptions;
-import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
-import org.apache.flink.runtime.state.StateBackend;
-import org.apache.flink.runtime.state.filesystem.FsStateBackend;
-import org.apache.flink.streaming.api.CheckpointingMode;
-import org.apache.flink.streaming.api.TimeCharacteristic;
-import org.apache.flink.streaming.api.environment.CheckpointConfig;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.table.api.EnvironmentSettings;
-import org.apache.flink.table.api.TableConfig;
-import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-import org.apache.flink.util.TernaryBoolean;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-public class FlinkEnvironment implements RuntimeEnv {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(FlinkEnvironment.class);
-
-    private Config config;
-
-    private StreamExecutionEnvironment environment;
-
-    private StreamTableEnvironment tableEnvironment;
-
-    private JobMode jobMode;
-
-    private String jobName = "seatunnel";
-
-    @Override
-    public FlinkEnvironment setConfig(Config config) {
-        this.config = config;
-        return this;
-    }
-
-    @Override
-    public Config getConfig() {
-        return config;
-    }
-
-    @Override
-    public CheckResult checkConfig() {
-        return EnvironmentUtil.checkRestartStrategy(config);
-    }
-
-    @Override
-    public FlinkEnvironment prepare() {
-        // Batch/Streaming both use data stream api in SeaTunnel New API
-        createStreamEnvironment();
-        createStreamTableEnvironment();
-        if (config.hasPath("job.name")) {
-            jobName = config.getString("job.name");
-        }
-        return this;
-    }
-
-    public String getJobName() {
-        return jobName;
-    }
-
-    @Override
-    public FlinkEnvironment setJobMode(JobMode jobMode) {
-        this.jobMode = jobMode;
-        return this;
-    }
-
-    @Override
-    public JobMode getJobMode() {
-        return jobMode;
-    }
-
-    @Override
-    public void registerPlugin(List<URL> pluginPaths) {
-        pluginPaths.forEach(url -> LOGGER.info("register plugins : {}", url));
-        Configuration configuration;
-        try {
-            configuration =
-                (Configuration) Objects.requireNonNull(ReflectionUtils.getDeclaredMethod(StreamExecutionEnvironment.class,
-                    "getConfiguration")).orElseThrow(() -> new RuntimeException("can't find " +
-                    "method: getConfiguration")).invoke(this.environment);
-        } catch (Exception e) {
-            throw new RuntimeException("Get flink configuration from environment failed", e);
-        }
-        List<String> jars = configuration.get(PipelineOptions.JARS);
-        if (jars == null) {
-            jars = new ArrayList<>();
-        }
-        jars.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
-        configuration.set(PipelineOptions.JARS, jars);
-        List<String> classpath = configuration.get(PipelineOptions.CLASSPATHS);
-        if (classpath == null) {
-            classpath = new ArrayList<>();
-        }
-        classpath.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
-        configuration.set(PipelineOptions.CLASSPATHS, classpath);
-    }
-
-    public StreamExecutionEnvironment getStreamExecutionEnvironment() {
-        return environment;
-    }
-
-    public StreamTableEnvironment getStreamTableEnvironment() {
-        return tableEnvironment;
-    }
-
-    private void createStreamTableEnvironment() {
-        // use blink and streammode
-        EnvironmentSettings.Builder envBuilder = EnvironmentSettings.newInstance()
-                .inStreamingMode();
-        if (this.config.hasPath(ConfigKeyName.PLANNER) && "blink"
-                .equals(this.config.getString(ConfigKeyName.PLANNER))) {
-            envBuilder.useBlinkPlanner();
-        } else {
-            envBuilder.useOldPlanner();
-        }
-        EnvironmentSettings environmentSettings = envBuilder.build();
-
-        tableEnvironment = StreamTableEnvironment.create(getStreamExecutionEnvironment(), environmentSettings);
-        TableConfig config = tableEnvironment.getConfig();
-        if (this.config.hasPath(ConfigKeyName.MAX_STATE_RETENTION_TIME) && this.config
-                .hasPath(ConfigKeyName.MIN_STATE_RETENTION_TIME)) {
-            long max = this.config.getLong(ConfigKeyName.MAX_STATE_RETENTION_TIME);
-            long min = this.config.getLong(ConfigKeyName.MIN_STATE_RETENTION_TIME);
-            config.setIdleStateRetentionTime(Time.seconds(min), Time.seconds(max));
-        }
-    }
-
-    private void createStreamEnvironment() {
-        environment = StreamExecutionEnvironment.getExecutionEnvironment();
-        setTimeCharacteristic();
-
-        setCheckpoint();
-
-        EnvironmentUtil.setRestartStrategy(config, environment.getConfig());
-
-        if (config.hasPath(ConfigKeyName.BUFFER_TIMEOUT_MILLIS)) {
-            long timeout = config.getLong(ConfigKeyName.BUFFER_TIMEOUT_MILLIS);
-            environment.setBufferTimeout(timeout);
-        }
-
-        if (config.hasPath(ConfigKeyName.PARALLELISM)) {
-            int parallelism = config.getInt(ConfigKeyName.PARALLELISM);
-            environment.setParallelism(parallelism);
-        }
-
-        if (config.hasPath(ConfigKeyName.MAX_PARALLELISM)) {
-            int max = config.getInt(ConfigKeyName.MAX_PARALLELISM);
-            environment.setMaxParallelism(max);
-        }
-
-        if (this.jobMode.equals(JobMode.BATCH)) {
-            environment.setRuntimeMode(RuntimeExecutionMode.BATCH);
-        }
-    }
-
-    private void setTimeCharacteristic() {
-        if (config.hasPath(ConfigKeyName.TIME_CHARACTERISTIC)) {
-            String timeType = config.getString(ConfigKeyName.TIME_CHARACTERISTIC);
-            switch (timeType.toLowerCase()) {
-                case "event-time":
-                    environment.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
-                    break;
-                case "ingestion-time":
-                    environment.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
-                    break;
-                case "processing-time":
-                    environment.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
-                    break;
-                default:
-                    LOGGER.warn(
-                            "set time-characteristic failed, unknown time-characteristic [{}],only support event-time,ingestion-time,processing-time",
-                            timeType);
-                    break;
-            }
-        }
-    }
-
-    private void setCheckpoint() {
-        if (config.hasPath(ConfigKeyName.CHECKPOINT_INTERVAL)) {
-            CheckpointConfig checkpointConfig = environment.getCheckpointConfig();
-            long interval = config.getLong(ConfigKeyName.CHECKPOINT_INTERVAL);
-            environment.enableCheckpointing(interval);
-
-            if (config.hasPath(ConfigKeyName.CHECKPOINT_MODE)) {
-                String mode = config.getString(ConfigKeyName.CHECKPOINT_MODE);
-                switch (mode.toLowerCase()) {
-                    case "exactly-once":
-                        checkpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
-                        break;
-                    case "at-least-once":
-                        checkpointConfig.setCheckpointingMode(CheckpointingMode.AT_LEAST_ONCE);
-                        break;
-                    default:
-                        LOGGER.warn(
-                                "set checkpoint.mode failed, unknown checkpoint.mode [{}],only support exactly-once,at-least-once",
-                                mode);
-                        break;
-                }
-            }
-
-            if (config.hasPath(ConfigKeyName.CHECKPOINT_TIMEOUT)) {
-                long timeout = config.getLong(ConfigKeyName.CHECKPOINT_TIMEOUT);
-                checkpointConfig.setCheckpointTimeout(timeout);
-            }
-
-            if (config.hasPath(ConfigKeyName.CHECKPOINT_DATA_URI)) {
-                String uri = config.getString(ConfigKeyName.CHECKPOINT_DATA_URI);
-                StateBackend fsStateBackend = new FsStateBackend(uri);
-                if (config.hasPath(ConfigKeyName.STATE_BACKEND)) {
-                    String stateBackend = config.getString(ConfigKeyName.STATE_BACKEND);
-                    if ("rocksdb".equalsIgnoreCase(stateBackend)) {
-                        StateBackend rocksDBStateBackend = new RocksDBStateBackend(fsStateBackend, TernaryBoolean.TRUE);
-                        environment.setStateBackend(rocksDBStateBackend);
-                    }
-                } else {
-                    environment.setStateBackend(fsStateBackend);
-                }
-            }
-
-            if (config.hasPath(ConfigKeyName.MAX_CONCURRENT_CHECKPOINTS)) {
-                int max = config.getInt(ConfigKeyName.MAX_CONCURRENT_CHECKPOINTS);
-                checkpointConfig.setMaxConcurrentCheckpoints(max);
-            }
-
-            if (config.hasPath(ConfigKeyName.CHECKPOINT_CLEANUP_MODE)) {
-                boolean cleanup = config.getBoolean(ConfigKeyName.CHECKPOINT_CLEANUP_MODE);
-                if (cleanup) {
-                    checkpointConfig.enableExternalizedCheckpoints(
-                            CheckpointConfig.ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION);
-                } else {
-                    checkpointConfig.enableExternalizedCheckpoints(
-                            CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
-
-                }
-            }
-
-            if (config.hasPath(ConfigKeyName.MIN_PAUSE_BETWEEN_CHECKPOINTS)) {
-                long minPause = config.getLong(ConfigKeyName.MIN_PAUSE_BETWEEN_CHECKPOINTS);
-                checkpointConfig.setMinPauseBetweenCheckpoints(minPause);
-            }
-
-            if (config.hasPath(ConfigKeyName.FAIL_ON_CHECKPOINTING_ERRORS)) {
-                int failNum = config.getInt(ConfigKeyName.FAIL_ON_CHECKPOINTING_ERRORS);
-                checkpointConfig.setTolerableCheckpointFailureNumber(failNum);
-            }
-        }
-    }
-
-}
diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java
index 3a8d7043f..00261a80c 100644
--- a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java
+++ b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java
@@ -24,13 +24,12 @@ import org.apache.seatunnel.common.config.Common;
 import org.apache.seatunnel.common.config.DeployMode;
 import org.apache.seatunnel.core.starter.Starter;
 import org.apache.seatunnel.core.starter.config.ConfigBuilder;
-import org.apache.seatunnel.core.starter.config.EngineType;
 import org.apache.seatunnel.core.starter.config.PluginType;
 import org.apache.seatunnel.core.starter.spark.args.SparkCommandArgs;
 import org.apache.seatunnel.core.starter.utils.CompressionUtils;
 import org.apache.seatunnel.plugin.discovery.PluginIdentifier;
-import org.apache.seatunnel.plugin.discovery.spark.SparkSinkPluginDiscovery;
-import org.apache.seatunnel.plugin.discovery.spark.SparkSourcePluginDiscovery;
+import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelSinkPluginDiscovery;
+import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelSourcePluginDiscovery;
 
 import org.apache.seatunnel.shade.com.typesafe.config.Config;
 import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
@@ -222,16 +221,16 @@ public class SparkStarter implements Starter {
      * return connector's jars, which located in 'connectors/spark/*'.
      */
     private List<Path> getConnectorJarDependencies() {
-        Path pluginRootDir = Common.connectorJarDir("spark");
+        Path pluginRootDir = Common.connectorJarDir("seatunnel");
         if (!Files.exists(pluginRootDir) || !Files.isDirectory(pluginRootDir)) {
             return Collections.emptyList();
         }
         Config config = new ConfigBuilder(Paths.get(commandArgs.getConfigFile())).getConfig();
         Set<URL> pluginJars = new HashSet<>();
-        SparkSourcePluginDiscovery sparkSourcePluginDiscovery = new SparkSourcePluginDiscovery();
-        SparkSinkPluginDiscovery sparkSinkPluginDiscovery = new SparkSinkPluginDiscovery();
-        pluginJars.addAll(sparkSourcePluginDiscovery.getPluginJarPaths(getPluginIdentifiers(config, PluginType.SOURCE)));
-        pluginJars.addAll(sparkSinkPluginDiscovery.getPluginJarPaths(getPluginIdentifiers(config, PluginType.SINK)));
+        SeaTunnelSourcePluginDiscovery seaTunnelSourcePluginDiscovery = new SeaTunnelSourcePluginDiscovery();
+        SeaTunnelSinkPluginDiscovery seaTunnelSinkPluginDiscovery = new SeaTunnelSinkPluginDiscovery();
+        pluginJars.addAll(seaTunnelSourcePluginDiscovery.getPluginJarPaths(getPluginIdentifiers(config, PluginType.SOURCE)));
+        pluginJars.addAll(seaTunnelSinkPluginDiscovery.getPluginJarPaths(getPluginIdentifiers(config, PluginType.SINK)));
         return pluginJars.stream().map(url -> new File(url.getPath()).toPath()).collect(Collectors.toList());
     }
 
@@ -317,10 +316,8 @@ public class SparkStarter implements Starter {
         return Arrays.stream(pluginTypes).flatMap((Function<PluginType, Stream<PluginIdentifier>>) pluginType -> {
             List<? extends Config> configList = config.getConfigList(pluginType.getType());
             return configList.stream()
-                    .map(pluginConfig -> PluginIdentifier
-                            .of(EngineType.SPARK.getEngine(),
-                                    pluginType.getType(),
-                                    pluginConfig.getString("plugin_name")));
+                    .map(pluginConfig -> PluginIdentifier.of("seatunnel", pluginType.getType(),
+                            pluginConfig.getString("plugin_name")));
         }).collect(Collectors.toList());
     }