You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by fa...@apache.org on 2022/07/21 11:48:23 UTC
[incubator-seatunnel] branch dev updated: [Core][Starter] Fix connector v2 can't deserialize on spark (#2221)
This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new cf5327cd4 [Core][Starter] Fix connector v2 can't deserialize on spark (#2221)
cf5327cd4 is described below
commit cf5327cd47da8c95eae06b8a086cb119db44b5ea
Author: Hisoka <fa...@qq.com>
AuthorDate: Thu Jul 21 19:48:17 2022 +0800
[Core][Starter] Fix connector v2 can't deserialize on spark (#2221)
* [Core][Starter] Fix connector v2 can't deserialize on spark
* [Core][Starter] Add SerializationUtils Unit Test
* [Core][Starter] Add SerializationUtils Unit Test
* [Core][Flink] Fixed FlinkEnvironment registerPlugin logic both old and new api
---
.../apache/seatunnel/flink/FlinkEnvironment.java | 40 +--
.../common/utils/SerializationException.java | 68 +++++
.../seatunnel/common/utils/SerializationUtils.java | 38 ++-
.../common/utils/SerializationUtilsTest.java | 74 ++++++
.../core/starter/flink/env/FlinkEnvironment.java | 286 ---------------------
.../seatunnel/core/starter/spark/SparkStarter.java | 21 +-
6 files changed, 205 insertions(+), 322 deletions(-)
diff --git a/seatunnel-apis/seatunnel-api-flink/src/main/java/org/apache/seatunnel/flink/FlinkEnvironment.java b/seatunnel-apis/seatunnel-api-flink/src/main/java/org/apache/seatunnel/flink/FlinkEnvironment.java
index d6e050ec1..49c97beb6 100644
--- a/seatunnel-apis/seatunnel-api-flink/src/main/java/org/apache/seatunnel/flink/FlinkEnvironment.java
+++ b/seatunnel-apis/seatunnel-api-flink/src/main/java/org/apache/seatunnel/flink/FlinkEnvironment.java
@@ -123,31 +123,31 @@ public class FlinkEnvironment implements RuntimeEnv {
@Override
public void registerPlugin(List<URL> pluginPaths) {
pluginPaths.forEach(url -> LOGGER.info("register plugins : {}", url));
- Configuration configuration;
+ List<Configuration> configurations = new ArrayList<>();
try {
- if (isStreaming()) {
- configuration =
- (Configuration) Objects.requireNonNull(ReflectionUtils.getDeclaredMethod(StreamExecutionEnvironment.class,
- "getConfiguration")).orElseThrow(() -> new RuntimeException("can't find " +
- "method: getConfiguration")).invoke(this.environment);
- } else {
- configuration = batchEnvironment.getConfiguration();
+ configurations.add((Configuration) Objects.requireNonNull(ReflectionUtils.getDeclaredMethod(StreamExecutionEnvironment.class,
+ "getConfiguration")).orElseThrow(() -> new RuntimeException("can't find " +
+ "method: getConfiguration")).invoke(this.environment));
+ if (!isStreaming()) {
+ configurations.add(batchEnvironment.getConfiguration());
}
} catch (Exception e) {
throw new RuntimeException(e);
}
- List<String> jars = configuration.get(PipelineOptions.JARS);
- if (jars == null) {
- jars = new ArrayList<>();
- }
- jars.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
- configuration.set(PipelineOptions.JARS, jars.stream().distinct().collect(Collectors.toList()));
- List<String> classpath = configuration.get(PipelineOptions.CLASSPATHS);
- if (classpath == null) {
- classpath = new ArrayList<>();
- }
- classpath.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
- configuration.set(PipelineOptions.CLASSPATHS, classpath.stream().distinct().collect(Collectors.toList()));
+ configurations.forEach(configuration -> {
+ List<String> jars = configuration.get(PipelineOptions.JARS);
+ if (jars == null) {
+ jars = new ArrayList<>();
+ }
+ jars.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
+ configuration.set(PipelineOptions.JARS, jars.stream().distinct().collect(Collectors.toList()));
+ List<String> classpath = configuration.get(PipelineOptions.CLASSPATHS);
+ if (classpath == null) {
+ classpath = new ArrayList<>();
+ }
+ classpath.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
+ configuration.set(PipelineOptions.CLASSPATHS, classpath.stream().distinct().collect(Collectors.toList()));
+ });
}
public StreamExecutionEnvironment getStreamExecutionEnvironment() {
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationException.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationException.java
new file mode 100644
index 000000000..33ca29a6f
--- /dev/null
+++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationException.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.common.utils;
+
+public class SerializationException extends RuntimeException {
+
+ /**
+ * Required for serialization support.
+ *
+ * @see java.io.Serializable
+ */
+ private static final long serialVersionUID = 2263144814025689516L;
+
+ /**
+ * <p>Constructs a new {@code SerializationException} without specified
+ * detail message.</p>
+ */
+ public SerializationException() {
+ }
+
+ /**
+ * <p>Constructs a new {@code SerializationException} with specified
+ * detail message.</p>
+ *
+ * @param msg The error message.
+ */
+ public SerializationException(final String msg) {
+ super(msg);
+ }
+
+ /**
+ * <p>Constructs a new {@code SerializationException} with specified
+ * nested {@code Throwable}.</p>
+ *
+ * @param cause The {@code Exception} or {@code Error}
+ * that caused this exception to be thrown.
+ */
+ public SerializationException(final Throwable cause) {
+ super(cause);
+ }
+
+ /**
+ * <p>Constructs a new {@code SerializationException} with specified
+ * detail message and nested {@code Throwable}.</p>
+ *
+ * @param msg The error message.
+ * @param cause The {@code Exception} or {@code Error}
+ * that caused this exception to be thrown.
+ */
+ public SerializationException(final String msg, final Throwable cause) {
+ super(msg, cause);
+ }
+}
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationUtils.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationUtils.java
index 43dead167..7e721617e 100644
--- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationUtils.java
+++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationUtils.java
@@ -20,30 +20,60 @@ package org.apache.seatunnel.common.utils;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.ObjectStreamClass;
import java.io.Serializable;
public class SerializationUtils {
public static String objectToString(Serializable obj) {
if (obj != null) {
- return Base64.encodeBase64String(org.apache.commons.lang3.SerializationUtils.serialize(obj));
+ return Base64.encodeBase64String(serialize(obj));
}
return null;
}
public static <T extends Serializable> T stringToObject(String str) {
if (StringUtils.isNotEmpty(str)) {
- return org.apache.commons.lang3.SerializationUtils.deserialize(Base64.decodeBase64(str));
+ return deserialize(Base64.decodeBase64(str));
}
return null;
}
+ @SuppressWarnings("checkstyle:MagicNumber")
public static <T extends Serializable> byte[] serialize(T obj) {
- return org.apache.commons.lang3.SerializationUtils.serialize(obj);
+ try (ByteArrayOutputStream b = new ByteArrayOutputStream(512);
+ ObjectOutputStream out = new ObjectOutputStream(b)) {
+ out.writeObject(obj);
+ return b.toByteArray();
+ } catch (final IOException ex) {
+ throw new SerializationException(ex);
+ }
+
}
public static <T extends Serializable> T deserialize(byte[] bytes) {
- return org.apache.commons.lang3.SerializationUtils.deserialize(bytes);
+ try (ByteArrayInputStream s = new ByteArrayInputStream(bytes);
+ ObjectInputStream in = new ObjectInputStream(s) {
+ @Override
+ protected Class<?> resolveClass(ObjectStreamClass desc) throws IOException, ClassNotFoundException {
+ // make sure use current thread classloader
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ if (cl == null) {
+ return super.resolveClass(desc);
+ }
+ return Class.forName(desc.getName(), false, cl);
+ }
+ }) {
+ @SuppressWarnings("unchecked") final T obj = (T) in.readObject();
+ return obj;
+ } catch (final ClassNotFoundException | IOException ex) {
+ throw new SerializationException(ex);
+ }
}
}
diff --git a/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/SerializationUtilsTest.java b/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/SerializationUtilsTest.java
new file mode 100644
index 000000000..ad0e80cea
--- /dev/null
+++ b/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/SerializationUtilsTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.common.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+@SuppressWarnings("checkstyle:RegexpSingleline")
+public class SerializationUtilsTest {
+
+ @Test
+ public void testObjectToString() {
+
+ HashMap<String, String> data = new HashMap<>();
+ data.put("key1", "value1");
+ data.put("seatunnelTest", "apache SeaTunnel");
+ data.put("中 文", "Apache Asia");
+ String configStr = SerializationUtils.objectToString(data);
+ Assert.assertNotNull(configStr);
+
+ HashMap<String, String> dataAfter = SerializationUtils.stringToObject(configStr);
+
+ Assert.assertEquals(dataAfter, data);
+
+ data.put("key2", "");
+ Assert.assertNotEquals(dataAfter, data);
+
+ }
+
+ @Test
+ public void testByteToObject() {
+
+ HashMap<String, String> data = new HashMap<>();
+ data.put("key1", "value1");
+ data.put("seatunnelTest", "apache SeaTunnel");
+ data.put("中 文", "Apache Asia");
+
+ ArrayList<HashMap<String, String>> array = new ArrayList<>();
+ array.add(data);
+ HashMap<String, String> data2 = new HashMap<>();
+ data2.put("Apache Asia", "中 文");
+ data2.put("value1", "key1");
+ data2.put("apache SeaTunnel", "seatunnelTest");
+ array.add(data2);
+
+ byte[] result = SerializationUtils.serialize(array);
+
+ ArrayList<HashMap<String, String>> array2 = SerializationUtils.deserialize(result);
+
+ Assert.assertEquals(array2, array);
+
+ Assert.assertThrows(SerializationException.class, () -> SerializationUtils.deserialize(new byte[]{1, 0, 1}));
+
+ }
+
+}
diff --git a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/env/FlinkEnvironment.java b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/env/FlinkEnvironment.java
deleted file mode 100644
index f9de514b0..000000000
--- a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/env/FlinkEnvironment.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.seatunnel.core.starter.flink.env;
-
-import org.apache.seatunnel.apis.base.env.RuntimeEnv;
-import org.apache.seatunnel.common.config.CheckResult;
-import org.apache.seatunnel.common.constants.JobMode;
-import org.apache.seatunnel.common.utils.ReflectionUtils;
-import org.apache.seatunnel.flink.util.ConfigKeyName;
-import org.apache.seatunnel.flink.util.EnvironmentUtil;
-
-import org.apache.seatunnel.shade.com.typesafe.config.Config;
-
-import org.apache.flink.api.common.RuntimeExecutionMode;
-import org.apache.flink.api.common.time.Time;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.PipelineOptions;
-import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
-import org.apache.flink.runtime.state.StateBackend;
-import org.apache.flink.runtime.state.filesystem.FsStateBackend;
-import org.apache.flink.streaming.api.CheckpointingMode;
-import org.apache.flink.streaming.api.TimeCharacteristic;
-import org.apache.flink.streaming.api.environment.CheckpointConfig;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.table.api.EnvironmentSettings;
-import org.apache.flink.table.api.TableConfig;
-import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-import org.apache.flink.util.TernaryBoolean;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-public class FlinkEnvironment implements RuntimeEnv {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FlinkEnvironment.class);
-
- private Config config;
-
- private StreamExecutionEnvironment environment;
-
- private StreamTableEnvironment tableEnvironment;
-
- private JobMode jobMode;
-
- private String jobName = "seatunnel";
-
- @Override
- public FlinkEnvironment setConfig(Config config) {
- this.config = config;
- return this;
- }
-
- @Override
- public Config getConfig() {
- return config;
- }
-
- @Override
- public CheckResult checkConfig() {
- return EnvironmentUtil.checkRestartStrategy(config);
- }
-
- @Override
- public FlinkEnvironment prepare() {
- // Batch/Streaming both use data stream api in SeaTunnel New API
- createStreamEnvironment();
- createStreamTableEnvironment();
- if (config.hasPath("job.name")) {
- jobName = config.getString("job.name");
- }
- return this;
- }
-
- public String getJobName() {
- return jobName;
- }
-
- @Override
- public FlinkEnvironment setJobMode(JobMode jobMode) {
- this.jobMode = jobMode;
- return this;
- }
-
- @Override
- public JobMode getJobMode() {
- return jobMode;
- }
-
- @Override
- public void registerPlugin(List<URL> pluginPaths) {
- pluginPaths.forEach(url -> LOGGER.info("register plugins : {}", url));
- Configuration configuration;
- try {
- configuration =
- (Configuration) Objects.requireNonNull(ReflectionUtils.getDeclaredMethod(StreamExecutionEnvironment.class,
- "getConfiguration")).orElseThrow(() -> new RuntimeException("can't find " +
- "method: getConfiguration")).invoke(this.environment);
- } catch (Exception e) {
- throw new RuntimeException("Get flink configuration from environment failed", e);
- }
- List<String> jars = configuration.get(PipelineOptions.JARS);
- if (jars == null) {
- jars = new ArrayList<>();
- }
- jars.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
- configuration.set(PipelineOptions.JARS, jars);
- List<String> classpath = configuration.get(PipelineOptions.CLASSPATHS);
- if (classpath == null) {
- classpath = new ArrayList<>();
- }
- classpath.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
- configuration.set(PipelineOptions.CLASSPATHS, classpath);
- }
-
- public StreamExecutionEnvironment getStreamExecutionEnvironment() {
- return environment;
- }
-
- public StreamTableEnvironment getStreamTableEnvironment() {
- return tableEnvironment;
- }
-
- private void createStreamTableEnvironment() {
- // use blink and streammode
- EnvironmentSettings.Builder envBuilder = EnvironmentSettings.newInstance()
- .inStreamingMode();
- if (this.config.hasPath(ConfigKeyName.PLANNER) && "blink"
- .equals(this.config.getString(ConfigKeyName.PLANNER))) {
- envBuilder.useBlinkPlanner();
- } else {
- envBuilder.useOldPlanner();
- }
- EnvironmentSettings environmentSettings = envBuilder.build();
-
- tableEnvironment = StreamTableEnvironment.create(getStreamExecutionEnvironment(), environmentSettings);
- TableConfig config = tableEnvironment.getConfig();
- if (this.config.hasPath(ConfigKeyName.MAX_STATE_RETENTION_TIME) && this.config
- .hasPath(ConfigKeyName.MIN_STATE_RETENTION_TIME)) {
- long max = this.config.getLong(ConfigKeyName.MAX_STATE_RETENTION_TIME);
- long min = this.config.getLong(ConfigKeyName.MIN_STATE_RETENTION_TIME);
- config.setIdleStateRetentionTime(Time.seconds(min), Time.seconds(max));
- }
- }
-
- private void createStreamEnvironment() {
- environment = StreamExecutionEnvironment.getExecutionEnvironment();
- setTimeCharacteristic();
-
- setCheckpoint();
-
- EnvironmentUtil.setRestartStrategy(config, environment.getConfig());
-
- if (config.hasPath(ConfigKeyName.BUFFER_TIMEOUT_MILLIS)) {
- long timeout = config.getLong(ConfigKeyName.BUFFER_TIMEOUT_MILLIS);
- environment.setBufferTimeout(timeout);
- }
-
- if (config.hasPath(ConfigKeyName.PARALLELISM)) {
- int parallelism = config.getInt(ConfigKeyName.PARALLELISM);
- environment.setParallelism(parallelism);
- }
-
- if (config.hasPath(ConfigKeyName.MAX_PARALLELISM)) {
- int max = config.getInt(ConfigKeyName.MAX_PARALLELISM);
- environment.setMaxParallelism(max);
- }
-
- if (this.jobMode.equals(JobMode.BATCH)) {
- environment.setRuntimeMode(RuntimeExecutionMode.BATCH);
- }
- }
-
- private void setTimeCharacteristic() {
- if (config.hasPath(ConfigKeyName.TIME_CHARACTERISTIC)) {
- String timeType = config.getString(ConfigKeyName.TIME_CHARACTERISTIC);
- switch (timeType.toLowerCase()) {
- case "event-time":
- environment.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- break;
- case "ingestion-time":
- environment.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
- break;
- case "processing-time":
- environment.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
- break;
- default:
- LOGGER.warn(
- "set time-characteristic failed, unknown time-characteristic [{}],only support event-time,ingestion-time,processing-time",
- timeType);
- break;
- }
- }
- }
-
- private void setCheckpoint() {
- if (config.hasPath(ConfigKeyName.CHECKPOINT_INTERVAL)) {
- CheckpointConfig checkpointConfig = environment.getCheckpointConfig();
- long interval = config.getLong(ConfigKeyName.CHECKPOINT_INTERVAL);
- environment.enableCheckpointing(interval);
-
- if (config.hasPath(ConfigKeyName.CHECKPOINT_MODE)) {
- String mode = config.getString(ConfigKeyName.CHECKPOINT_MODE);
- switch (mode.toLowerCase()) {
- case "exactly-once":
- checkpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
- break;
- case "at-least-once":
- checkpointConfig.setCheckpointingMode(CheckpointingMode.AT_LEAST_ONCE);
- break;
- default:
- LOGGER.warn(
- "set checkpoint.mode failed, unknown checkpoint.mode [{}],only support exactly-once,at-least-once",
- mode);
- break;
- }
- }
-
- if (config.hasPath(ConfigKeyName.CHECKPOINT_TIMEOUT)) {
- long timeout = config.getLong(ConfigKeyName.CHECKPOINT_TIMEOUT);
- checkpointConfig.setCheckpointTimeout(timeout);
- }
-
- if (config.hasPath(ConfigKeyName.CHECKPOINT_DATA_URI)) {
- String uri = config.getString(ConfigKeyName.CHECKPOINT_DATA_URI);
- StateBackend fsStateBackend = new FsStateBackend(uri);
- if (config.hasPath(ConfigKeyName.STATE_BACKEND)) {
- String stateBackend = config.getString(ConfigKeyName.STATE_BACKEND);
- if ("rocksdb".equalsIgnoreCase(stateBackend)) {
- StateBackend rocksDBStateBackend = new RocksDBStateBackend(fsStateBackend, TernaryBoolean.TRUE);
- environment.setStateBackend(rocksDBStateBackend);
- }
- } else {
- environment.setStateBackend(fsStateBackend);
- }
- }
-
- if (config.hasPath(ConfigKeyName.MAX_CONCURRENT_CHECKPOINTS)) {
- int max = config.getInt(ConfigKeyName.MAX_CONCURRENT_CHECKPOINTS);
- checkpointConfig.setMaxConcurrentCheckpoints(max);
- }
-
- if (config.hasPath(ConfigKeyName.CHECKPOINT_CLEANUP_MODE)) {
- boolean cleanup = config.getBoolean(ConfigKeyName.CHECKPOINT_CLEANUP_MODE);
- if (cleanup) {
- checkpointConfig.enableExternalizedCheckpoints(
- CheckpointConfig.ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION);
- } else {
- checkpointConfig.enableExternalizedCheckpoints(
- CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
-
- }
- }
-
- if (config.hasPath(ConfigKeyName.MIN_PAUSE_BETWEEN_CHECKPOINTS)) {
- long minPause = config.getLong(ConfigKeyName.MIN_PAUSE_BETWEEN_CHECKPOINTS);
- checkpointConfig.setMinPauseBetweenCheckpoints(minPause);
- }
-
- if (config.hasPath(ConfigKeyName.FAIL_ON_CHECKPOINTING_ERRORS)) {
- int failNum = config.getInt(ConfigKeyName.FAIL_ON_CHECKPOINTING_ERRORS);
- checkpointConfig.setTolerableCheckpointFailureNumber(failNum);
- }
- }
- }
-
-}
diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java
index 3a8d7043f..00261a80c 100644
--- a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java
+++ b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java
@@ -24,13 +24,12 @@ import org.apache.seatunnel.common.config.Common;
import org.apache.seatunnel.common.config.DeployMode;
import org.apache.seatunnel.core.starter.Starter;
import org.apache.seatunnel.core.starter.config.ConfigBuilder;
-import org.apache.seatunnel.core.starter.config.EngineType;
import org.apache.seatunnel.core.starter.config.PluginType;
import org.apache.seatunnel.core.starter.spark.args.SparkCommandArgs;
import org.apache.seatunnel.core.starter.utils.CompressionUtils;
import org.apache.seatunnel.plugin.discovery.PluginIdentifier;
-import org.apache.seatunnel.plugin.discovery.spark.SparkSinkPluginDiscovery;
-import org.apache.seatunnel.plugin.discovery.spark.SparkSourcePluginDiscovery;
+import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelSinkPluginDiscovery;
+import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelSourcePluginDiscovery;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
@@ -222,16 +221,16 @@ public class SparkStarter implements Starter {
* return connector's jars, which located in 'connectors/spark/*'.
*/
private List<Path> getConnectorJarDependencies() {
- Path pluginRootDir = Common.connectorJarDir("spark");
+ Path pluginRootDir = Common.connectorJarDir("seatunnel");
if (!Files.exists(pluginRootDir) || !Files.isDirectory(pluginRootDir)) {
return Collections.emptyList();
}
Config config = new ConfigBuilder(Paths.get(commandArgs.getConfigFile())).getConfig();
Set<URL> pluginJars = new HashSet<>();
- SparkSourcePluginDiscovery sparkSourcePluginDiscovery = new SparkSourcePluginDiscovery();
- SparkSinkPluginDiscovery sparkSinkPluginDiscovery = new SparkSinkPluginDiscovery();
- pluginJars.addAll(sparkSourcePluginDiscovery.getPluginJarPaths(getPluginIdentifiers(config, PluginType.SOURCE)));
- pluginJars.addAll(sparkSinkPluginDiscovery.getPluginJarPaths(getPluginIdentifiers(config, PluginType.SINK)));
+ SeaTunnelSourcePluginDiscovery seaTunnelSourcePluginDiscovery = new SeaTunnelSourcePluginDiscovery();
+ SeaTunnelSinkPluginDiscovery seaTunnelSinkPluginDiscovery = new SeaTunnelSinkPluginDiscovery();
+ pluginJars.addAll(seaTunnelSourcePluginDiscovery.getPluginJarPaths(getPluginIdentifiers(config, PluginType.SOURCE)));
+ pluginJars.addAll(seaTunnelSinkPluginDiscovery.getPluginJarPaths(getPluginIdentifiers(config, PluginType.SINK)));
return pluginJars.stream().map(url -> new File(url.getPath()).toPath()).collect(Collectors.toList());
}
@@ -317,10 +316,8 @@ public class SparkStarter implements Starter {
return Arrays.stream(pluginTypes).flatMap((Function<PluginType, Stream<PluginIdentifier>>) pluginType -> {
List<? extends Config> configList = config.getConfigList(pluginType.getType());
return configList.stream()
- .map(pluginConfig -> PluginIdentifier
- .of(EngineType.SPARK.getEngine(),
- pluginType.getType(),
- pluginConfig.getString("plugin_name")));
+ .map(pluginConfig -> PluginIdentifier.of("seatunnel", pluginType.getType(),
+ pluginConfig.getString("plugin_name")));
}).collect(Collectors.toList());
}