You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ri...@apache.org on 2022/01/21 11:10:46 UTC
[incubator-seatunnel] branch dev updated: [SeaTunnel #1116] Optimize CheckConfigUtil and add UT (#1117)
This is an automated email from the ASF dual-hosted git repository.
rickyhuo pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 33de30e [SeaTunnel #1116] Optimize CheckConfigUtil and add UT (#1117)
33de30e is described below
commit 33de30ebde44fd10817065ef46d94da8a9059483
Author: Simon <36...@qq.com>
AuthorDate: Fri Jan 21 19:10:38 2022 +0800
[SeaTunnel #1116] Optimize CheckConfigUtil and add UT (#1117)
* Optimize CheckConfigUtil and add UT
* 1
---
.../seatunnel/common/config/CheckConfigUtil.java | 33 +++++++-----
.../common/config/CheckConfigUtilTest.java | 62 +++++++++++++++++-----
.../org/apache/seatunnel/flink/sink/DorisSink.java | 2 +-
.../org/apache/seatunnel/flink/sink/DruidSink.java | 2 +-
.../apache/seatunnel/flink/source/DruidSource.java | 2 +-
.../apache/seatunnel/flink/sink/Elasticsearch.java | 2 +-
.../org/apache/seatunnel/flink/sink/FileSink.java | 2 +-
.../apache/seatunnel/flink/source/FileSource.java | 2 +-
.../apache/seatunnel/flink/sink/InfluxDbSink.java | 2 +-
.../org/apache/seatunnel/flink/sink/JdbcSink.java | 2 +-
.../apache/seatunnel/flink/source/JdbcSource.java | 2 +-
.../apache/seatunnel/flink/sink/KafkaTable.java | 2 +-
.../seatunnel/flink/source/KafkaTableStream.java | 4 +-
.../org/apache/seatunnel/spark/sink/Doris.scala | 4 +-
.../seatunnel/spark/sink/Elasticsearch.scala | 4 +-
.../seatunnel/spark/source/Elasticsearch.scala | 4 +-
.../org/apache/seatunnel/spark/sink/Email.scala | 2 +-
.../apache/seatunnel/spark/source/FakeStream.scala | 4 +-
.../org/apache/seatunnel/spark/source/File.scala | 4 +-
.../org/apache/seatunnel/spark/sink/Hbase.scala | 4 +-
.../org/apache/seatunnel/spark/sink/Hive.scala | 2 +-
.../org/apache/seatunnel/spark/source/Hive.scala | 4 +-
.../org/apache/seatunnel/spark/sink/Hudi.scala | 4 +-
.../org/apache/seatunnel/spark/source/Hudi.scala | 4 +-
.../org/apache/seatunnel/spark/sink/Jdbc.scala | 4 +-
.../org/apache/seatunnel/spark/source/Jdbc.scala | 4 +-
.../seatunnel/spark/source/KafkaStream.scala | 6 +--
.../org/apache/seatunnel/spark/sink/Kudu.scala | 4 +-
.../org/apache/seatunnel/spark/source/Kudu.scala | 4 +-
.../org/apache/seatunnel/spark/source/Neo4j.scala | 15 +++---
.../org/apache/seatunnel/spark/sink/Phoenix.scala | 2 +-
.../apache/seatunnel/spark/source/Phoenix.scala | 2 +-
.../org/apache/seatunnel/spark/sink/Tidb.scala | 4 +-
.../org/apache/seatunnel/spark/source/Tidb.scala | 4 +-
.../flink/transform/DataStreamToTable.java | 2 +-
.../apache/seatunnel/flink/transform/Split.java | 2 +-
.../org/apache/seatunnel/flink/transform/Sql.java | 2 +-
.../flink/transform/TableToDataStream.java | 2 +-
.../apache/seatunnel/spark/transform/Split.scala | 4 +-
.../org/apache/seatunnel/spark/transform/Sql.scala | 4 +-
40 files changed, 134 insertions(+), 90 deletions(-)
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/CheckConfigUtil.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/CheckConfigUtil.java
index f409466..9241ec8 100644
--- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/CheckConfigUtil.java
+++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/CheckConfigUtil.java
@@ -26,17 +26,22 @@ import java.util.stream.Collectors;
public class CheckConfigUtil {
+ /**
+ * please using {@link #checkAllExists} instead, since 2.0.5
+ */
+ @Deprecated
public static CheckResult check(Config config, String... params) {
- StringBuilder missingParams = new StringBuilder();
- for (String param : params) {
- if (!config.hasPath(param) || config.getAnyRef(param) == null) {
- missingParams.append(param).append(",");
- }
- }
+ return checkAllExists(config, params);
+ }
+
+ public static CheckResult checkAllExists(Config config, String... params) {
+ List<String> missingParams = Arrays.stream(params)
+ .filter(param -> !config.hasPath(param) || config.getAnyRef(param) == null)
+ .collect(Collectors.toList());
- if (missingParams.length() > 0) {
+ if (missingParams.size() > 0) {
String errorMsg = String.format("please specify [%s] as non-empty",
- missingParams.deleteCharAt(missingParams.length() - 1));
+ String.join(",", missingParams));
return CheckResult.error(errorMsg);
} else {
return CheckResult.success();
@@ -46,7 +51,7 @@ public class CheckConfigUtil {
/**
* check config if there was at least one usable
*/
- public static CheckResult checkOne(Config config, String... params) {
+ public static CheckResult checkAtLeastOneExists(Config config, String... params) {
if (params.length == 0) {
return CheckResult.success();
}
@@ -60,7 +65,7 @@ public class CheckConfigUtil {
if (missingParams.size() == params.length) {
String errorMsg = String.format("please specify at least one config of [%s] as non-empty",
- missingParams.stream().collect(Collectors.joining(",")));
+ String.join(",", missingParams));
return CheckResult.error(errorMsg);
} else {
return CheckResult.success();
@@ -70,12 +75,14 @@ public class CheckConfigUtil {
/**
* merge all check result
*/
- public static CheckResult mergeCheckMessage(CheckResult... checkResults) {
- List<CheckResult> notPassConfig = Arrays.stream(checkResults).filter(item -> !item.isSuccess()).collect(Collectors.toList());
+ public static CheckResult mergeCheckResults(CheckResult... checkResults) {
+ List<CheckResult> notPassConfig = Arrays.stream(checkResults)
+ .filter(item -> !item.isSuccess()).collect(Collectors.toList());
if (notPassConfig.isEmpty()) {
return CheckResult.success();
} else {
- String errMessage = notPassConfig.stream().map(CheckResult::getMsg).collect(Collectors.joining(","));
+ String errMessage = notPassConfig.stream().map(CheckResult::getMsg)
+ .collect(Collectors.joining(","));
return CheckResult.error(errMessage);
}
diff --git a/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CheckConfigUtilTest.java b/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CheckConfigUtilTest.java
index 459c7af..cbdb86a 100644
--- a/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CheckConfigUtilTest.java
+++ b/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CheckConfigUtilTest.java
@@ -26,23 +26,61 @@ import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
+import static org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists;
+import static org.apache.seatunnel.common.config.CheckConfigUtil.checkAtLeastOneExists;
+import static org.apache.seatunnel.common.config.CheckConfigUtil.mergeCheckResults;
+
public class CheckConfigUtilTest {
@Test
- public void testCheck() {
- Map<String, String> configMap = new HashMap<>();
- configMap.put("k0", "v0");
- configMap.put("k1", "v1");
- Config config = ConfigFactory.parseMap(configMap);
-
- CheckResult check = CheckConfigUtil.check(config, "k0", "k1");
- Assert.assertTrue(check.isSuccess());
+ public void testCheckAllExists() {
+ Config config = getConfig();
+ CheckResult checkResult = checkAllExists(config, "k0", "k1");
+ Assert.assertTrue(checkResult.isSuccess());
String errorMsg = "please specify [%s] as non-empty";
- check = CheckConfigUtil.check(config, "k0", "k1", "k2");
- Assert.assertEquals(String.format(errorMsg, "k2"), check.getMsg());
+ checkResult = checkAllExists(config, "k0", "k1", "k2");
+ Assert.assertEquals(String.format(errorMsg, "k2"), checkResult.getMsg());
+
+ checkResult = checkAllExists(config, "k0", "k1", "k2", "k3", "k4");
+ Assert.assertEquals(String.format(errorMsg, "k2,k3,k4"), checkResult.getMsg());
+ }
+
+ @Test
+ public void testCheckAtLeastOneExists() {
+ Config config = getConfig();
+ CheckResult checkResult = checkAtLeastOneExists(config, "k0", "k3", "k4");
+ Assert.assertTrue(checkResult.isSuccess());
+
+ String errorMsg = "please specify at least one config of [%s] as non-empty";
+ checkResult = checkAtLeastOneExists(config, "k3", "k2");
+ Assert.assertEquals(String.format(errorMsg, "k3,k2"), checkResult.getMsg());
+ }
+
+ @Test
+ public void testMergeCheckResults() {
+ Config config = getConfig();
+ CheckResult checkResult1 = checkAllExists(config, "k0", "k1");
+ CheckResult checkResult2 = checkAtLeastOneExists(config, "k1", "k3");
+ CheckResult checkResult3 = checkAllExists(config, "k0", "k3");
+ CheckResult checkResult4 = checkAtLeastOneExists(config, "k2", "k3");
+
+ CheckResult finalResult = mergeCheckResults(checkResult1, checkResult2);
+ Assert.assertTrue(finalResult.isSuccess());
+
+ String errorMsg1 = "please specify [%s] as non-empty";
+ String errorMsg2 = "please specify at least one config of [%s] as non-empty";
+ finalResult = mergeCheckResults(checkResult3, checkResult2);
+ Assert.assertEquals(String.format(errorMsg1, "k3"), finalResult.getMsg());
- check = CheckConfigUtil.check(config, "k0", "k1", "k2", "k3", "k4");
- Assert.assertEquals(String.format(errorMsg, "k2,k3,k4"), check.getMsg());
+ finalResult = mergeCheckResults(checkResult3, checkResult4);
+ Assert.assertEquals(String.format(errorMsg1 + "," + errorMsg2, "k3", "k2,k3"), finalResult.getMsg());
+ }
+
+ public Config getConfig() {
+ Map<String, String> configMap = new HashMap<>();
+ configMap.put("k0", "v0");
+ configMap.put("k1", "v1");
+ return ConfigFactory.parseMap(configMap);
}
}
diff --git a/seatunnel-connectors/seatunnel-connector-flink-doris/src/main/java/org/apache/seatunnel/flink/sink/DorisSink.java b/seatunnel-connectors/seatunnel-connector-flink-doris/src/main/java/org/apache/seatunnel/flink/sink/DorisSink.java
index 9e98901..8fd1044 100644
--- a/seatunnel-connectors/seatunnel-connector-flink-doris/src/main/java/org/apache/seatunnel/flink/sink/DorisSink.java
+++ b/seatunnel-connectors/seatunnel-connector-flink-doris/src/main/java/org/apache/seatunnel/flink/sink/DorisSink.java
@@ -65,7 +65,7 @@ public class DorisSink implements FlinkStreamSink<Row, Row>, FlinkBatchSink<Row,
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, "fenodes", "user", "password", "table", "database");
+ return CheckConfigUtil.checkAllExists(config, "fenodes", "user", "password", "table", "database");
}
@Override
diff --git a/seatunnel-connectors/seatunnel-connector-flink-druid/src/main/java/org/apache/seatunnel/flink/sink/DruidSink.java b/seatunnel-connectors/seatunnel-connector-flink-druid/src/main/java/org/apache/seatunnel/flink/sink/DruidSink.java
index 269628e..aae484d 100644
--- a/seatunnel-connectors/seatunnel-connector-flink-druid/src/main/java/org/apache/seatunnel/flink/sink/DruidSink.java
+++ b/seatunnel-connectors/seatunnel-connector-flink-druid/src/main/java/org/apache/seatunnel/flink/sink/DruidSink.java
@@ -52,7 +52,7 @@ public class DruidSink implements FlinkBatchSink<Row, Row> {
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, COORDINATOR_URL, DATASOURCE);
+ return CheckConfigUtil.checkAllExists(config, COORDINATOR_URL, DATASOURCE);
}
@Override
diff --git a/seatunnel-connectors/seatunnel-connector-flink-druid/src/main/java/org/apache/seatunnel/flink/source/DruidSource.java b/seatunnel-connectors/seatunnel-connector-flink-druid/src/main/java/org/apache/seatunnel/flink/source/DruidSource.java
index cdacfab..7c45757 100644
--- a/seatunnel-connectors/seatunnel-connector-flink-druid/src/main/java/org/apache/seatunnel/flink/source/DruidSource.java
+++ b/seatunnel-connectors/seatunnel-connector-flink-druid/src/main/java/org/apache/seatunnel/flink/source/DruidSource.java
@@ -94,7 +94,7 @@ public class DruidSource implements FlinkBatchSource<Row> {
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, JDBC_URL, DATASOURCE);
+ return CheckConfigUtil.checkAllExists(config, JDBC_URL, DATASOURCE);
}
@Override
diff --git a/seatunnel-connectors/seatunnel-connector-flink-elasticsearch/src/main/java/org/apache/seatunnel/flink/sink/Elasticsearch.java b/seatunnel-connectors/seatunnel-connector-flink-elasticsearch/src/main/java/org/apache/seatunnel/flink/sink/Elasticsearch.java
index 71a504d..1638e1b 100644
--- a/seatunnel-connectors/seatunnel-connector-flink-elasticsearch/src/main/java/org/apache/seatunnel/flink/sink/Elasticsearch.java
+++ b/seatunnel-connectors/seatunnel-connector-flink-elasticsearch/src/main/java/org/apache/seatunnel/flink/sink/Elasticsearch.java
@@ -62,7 +62,7 @@ public class Elasticsearch implements FlinkStreamSink<Row, Row>, FlinkBatchSink<
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, "hosts");
+ return CheckConfigUtil.checkAllExists(config, "hosts");
}
@Override
diff --git a/seatunnel-connectors/seatunnel-connector-flink-file/src/main/java/org/apache/seatunnel/flink/sink/FileSink.java b/seatunnel-connectors/seatunnel-connector-flink-file/src/main/java/org/apache/seatunnel/flink/sink/FileSink.java
index 6e4c6a6..ec8705a 100644
--- a/seatunnel-connectors/seatunnel-connector-flink-file/src/main/java/org/apache/seatunnel/flink/sink/FileSink.java
+++ b/seatunnel-connectors/seatunnel-connector-flink-file/src/main/java/org/apache/seatunnel/flink/sink/FileSink.java
@@ -107,7 +107,7 @@ public class FileSink implements FlinkStreamSink<Row, Row>, FlinkBatchSink<Row,
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, PATH, FORMAT);
+ return CheckConfigUtil.checkAllExists(config, PATH, FORMAT);
}
@Override
diff --git a/seatunnel-connectors/seatunnel-connector-flink-file/src/main/java/org/apache/seatunnel/flink/source/FileSource.java b/seatunnel-connectors/seatunnel-connector-flink-file/src/main/java/org/apache/seatunnel/flink/source/FileSource.java
index d388283..fad7f6a 100644
--- a/seatunnel-connectors/seatunnel-connector-flink-file/src/main/java/org/apache/seatunnel/flink/source/FileSource.java
+++ b/seatunnel-connectors/seatunnel-connector-flink-file/src/main/java/org/apache/seatunnel/flink/source/FileSource.java
@@ -70,7 +70,7 @@ public class FileSource implements FlinkBatchSource<Row> {
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, PATH, SOURCE_FORMAT, SCHEMA);
+ return CheckConfigUtil.checkAllExists(config, PATH, SOURCE_FORMAT, SCHEMA);
}
@Override
diff --git a/seatunnel-connectors/seatunnel-connector-flink-influxdb/src/main/java/org/apache/seatunnel/flink/sink/InfluxDbSink.java b/seatunnel-connectors/seatunnel-connector-flink-influxdb/src/main/java/org/apache/seatunnel/flink/sink/InfluxDbSink.java
index c447ab9..9a6b18f 100644
--- a/seatunnel-connectors/seatunnel-connector-flink-influxdb/src/main/java/org/apache/seatunnel/flink/sink/InfluxDbSink.java
+++ b/seatunnel-connectors/seatunnel-connector-flink-influxdb/src/main/java/org/apache/seatunnel/flink/sink/InfluxDbSink.java
@@ -60,7 +60,7 @@ public class InfluxDbSink implements FlinkBatchSink<Row, Row> {
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, SERVER_URL, DATABASE, MEASUREMENT, TAGS, FIELDS);
+ return CheckConfigUtil.checkAllExists(config, SERVER_URL, DATABASE, MEASUREMENT, TAGS, FIELDS);
}
@Override
diff --git a/seatunnel-connectors/seatunnel-connector-flink-jdbc/src/main/java/org/apache/seatunnel/flink/sink/JdbcSink.java b/seatunnel-connectors/seatunnel-connector-flink-jdbc/src/main/java/org/apache/seatunnel/flink/sink/JdbcSink.java
index 149b044..473ffd9 100644
--- a/seatunnel-connectors/seatunnel-connector-flink-jdbc/src/main/java/org/apache/seatunnel/flink/sink/JdbcSink.java
+++ b/seatunnel-connectors/seatunnel-connector-flink-jdbc/src/main/java/org/apache/seatunnel/flink/sink/JdbcSink.java
@@ -59,7 +59,7 @@ public class JdbcSink implements FlinkStreamSink<Row, Row>, FlinkBatchSink<Row,
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, "driver", "url", "username", "query");
+ return CheckConfigUtil.checkAllExists(config, "driver", "url", "username", "query");
}
@Override
diff --git a/seatunnel-connectors/seatunnel-connector-flink-jdbc/src/main/java/org/apache/seatunnel/flink/source/JdbcSource.java b/seatunnel-connectors/seatunnel-connector-flink-jdbc/src/main/java/org/apache/seatunnel/flink/source/JdbcSource.java
index 85b8a0a..e6a664d 100644
--- a/seatunnel-connectors/seatunnel-connector-flink-jdbc/src/main/java/org/apache/seatunnel/flink/source/JdbcSource.java
+++ b/seatunnel-connectors/seatunnel-connector-flink-jdbc/src/main/java/org/apache/seatunnel/flink/source/JdbcSource.java
@@ -107,7 +107,7 @@ public class JdbcSource implements FlinkBatchSource<Row> {
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, "driver", "url", "username", "query");
+ return CheckConfigUtil.checkAllExists(config, "driver", "url", "username", "query");
}
@Override
diff --git a/seatunnel-connectors/seatunnel-connector-flink-kafka/src/main/java/org/apache/seatunnel/flink/sink/KafkaTable.java b/seatunnel-connectors/seatunnel-connector-flink-kafka/src/main/java/org/apache/seatunnel/flink/sink/KafkaTable.java
index c904be5..9b1aeac 100644
--- a/seatunnel-connectors/seatunnel-connector-flink-kafka/src/main/java/org/apache/seatunnel/flink/sink/KafkaTable.java
+++ b/seatunnel-connectors/seatunnel-connector-flink-kafka/src/main/java/org/apache/seatunnel/flink/sink/KafkaTable.java
@@ -98,7 +98,7 @@ public class KafkaTable implements FlinkStreamSink<Row, Row> {
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, "topics");
+ return CheckConfigUtil.checkAllExists(config, "topics");
}
@Override
diff --git a/seatunnel-connectors/seatunnel-connector-flink-kafka/src/main/java/org/apache/seatunnel/flink/source/KafkaTableStream.java b/seatunnel-connectors/seatunnel-connector-flink-kafka/src/main/java/org/apache/seatunnel/flink/source/KafkaTableStream.java
index 2ae1d51..942cb9e 100644
--- a/seatunnel-connectors/seatunnel-connector-flink-kafka/src/main/java/org/apache/seatunnel/flink/source/KafkaTableStream.java
+++ b/seatunnel-connectors/seatunnel-connector-flink-kafka/src/main/java/org/apache/seatunnel/flink/source/KafkaTableStream.java
@@ -77,11 +77,11 @@ public class KafkaTableStream implements FlinkStreamSource<Row> {
@Override
public CheckResult checkConfig() {
- CheckResult result = CheckConfigUtil.check(config, TOPICS, SCHEMA, SOURCE_FORMAT, RESULT_TABLE_NAME);
+ CheckResult result = CheckConfigUtil.checkAllExists(config, TOPICS, SCHEMA, SOURCE_FORMAT, RESULT_TABLE_NAME);
if (result.isSuccess()) {
Config consumerConfig = TypesafeConfigUtils.extractSubConfig(config, consumerPrefix, false);
- return CheckConfigUtil.check(consumerConfig, BOOTSTRAP_SERVERS, GROUP_ID);
+ return CheckConfigUtil.checkAllExists(consumerConfig, BOOTSTRAP_SERVERS, GROUP_ID);
}
return result;
diff --git a/seatunnel-connectors/seatunnel-connector-spark-doris/src/main/scala/org/apache/seatunnel/spark/sink/Doris.scala b/seatunnel-connectors/seatunnel-connector-spark-doris/src/main/scala/org/apache/seatunnel/spark/sink/Doris.scala
index 91a4bbb..618b220 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-doris/src/main/scala/org/apache/seatunnel/spark/sink/Doris.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-doris/src/main/scala/org/apache/seatunnel/spark/sink/Doris.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
import scala.collection.mutable.ListBuffer
import org.apache.seatunnel.common.config.{CheckResult, TypesafeConfigUtils}
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.spark.SparkEnvironment
import org.apache.seatunnel.spark.batch.SparkBatchSink
import org.apache.spark.sql.{Dataset, Row}
@@ -69,7 +69,7 @@ class Doris extends SparkBatchSink with Serializable {
override def checkConfig(): CheckResult = {
val checkResult =
- check(config, Config.HOST, Config.DATABASE, Config.TABLE_NAME, Config.USER, Config.PASSWORD)
+ checkAllExists(config, Config.HOST, Config.DATABASE, Config.TABLE_NAME, Config.USER, Config.PASSWORD)
if (!checkResult.isSuccess) {
checkResult
diff --git a/seatunnel-connectors/seatunnel-connector-spark-elasticsearch/src/main/scala/org/apache/seatunnel/spark/sink/Elasticsearch.scala b/seatunnel-connectors/seatunnel-connector-spark-elasticsearch/src/main/scala/org/apache/seatunnel/spark/sink/Elasticsearch.scala
index b6350cd..dc22f2b 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-elasticsearch/src/main/scala/org/apache/seatunnel/spark/sink/Elasticsearch.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-elasticsearch/src/main/scala/org/apache/seatunnel/spark/sink/Elasticsearch.scala
@@ -16,7 +16,7 @@
*/
package org.apache.seatunnel.spark.sink
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.common.config.CheckResult
import org.apache.seatunnel.common.utils.StringTemplate
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
@@ -42,7 +42,7 @@ class Elasticsearch extends SparkBatchSink {
}
override def checkConfig(): CheckResult = {
- check(config, "hosts")
+ checkAllExists(config, "hosts")
}
override def prepare(environment: SparkEnvironment): Unit = {
diff --git a/seatunnel-connectors/seatunnel-connector-spark-elasticsearch/src/main/scala/org/apache/seatunnel/spark/source/Elasticsearch.scala b/seatunnel-connectors/seatunnel-connector-spark-elasticsearch/src/main/scala/org/apache/seatunnel/spark/source/Elasticsearch.scala
index 841c25a..06fc6e9 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-elasticsearch/src/main/scala/org/apache/seatunnel/spark/source/Elasticsearch.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-elasticsearch/src/main/scala/org/apache/seatunnel/spark/source/Elasticsearch.scala
@@ -19,7 +19,7 @@ package org.apache.seatunnel.spark.source
import scala.collection.JavaConversions._
import org.apache.seatunnel.common.config.{CheckResult, TypesafeConfigUtils}
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.spark.SparkEnvironment
import org.apache.seatunnel.spark.batch.SparkBatchSource
import org.apache.spark.sql.{Dataset, Row}
@@ -63,7 +63,7 @@ class Elasticsearch extends SparkBatchSource {
}
override def checkConfig(): CheckResult = {
- check(config, "hosts", "index")
+ checkAllExists(config, "hosts", "index")
}
}
diff --git a/seatunnel-connectors/seatunnel-connector-spark-email/src/main/scala/org/apache/seatunnel/spark/sink/Email.scala b/seatunnel-connectors/seatunnel-connector-spark-email/src/main/scala/org/apache/seatunnel/spark/sink/Email.scala
index c8bd99c..ba1bcc4 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-email/src/main/scala/org/apache/seatunnel/spark/sink/Email.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-email/src/main/scala/org/apache/seatunnel/spark/sink/Email.scala
@@ -95,7 +95,7 @@ class Email extends SparkBatchSink {
}
override def checkConfig(): CheckResult = {
- CheckConfigUtil.check(config, "from", "to", "host", "port", "password")
+ CheckConfigUtil.checkAllExists(config, "from", "to", "host", "port", "password")
}
override def prepare(prepareEnv: SparkEnvironment): Unit = {}
diff --git a/seatunnel-connectors/seatunnel-connector-spark-fake/src/main/scala/org/apache/seatunnel/spark/source/FakeStream.scala b/seatunnel-connectors/seatunnel-connector-spark-fake/src/main/scala/org/apache/seatunnel/spark/source/FakeStream.scala
index 3117e5c..aa44f27 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-fake/src/main/scala/org/apache/seatunnel/spark/source/FakeStream.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-fake/src/main/scala/org/apache/seatunnel/spark/source/FakeStream.scala
@@ -28,7 +28,7 @@ import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.receiver.Receiver
import java.security.SecureRandom
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import scala.collection.JavaConversions._
@@ -56,7 +56,7 @@ class FakeStream extends SparkStreamingSource[String] {
}
override def checkConfig(): CheckResult = {
- check(config, "content")
+ checkAllExists(config, "content")
}
}
diff --git a/seatunnel-connectors/seatunnel-connector-spark-file/src/main/scala/org/apache/seatunnel/spark/source/File.scala b/seatunnel-connectors/seatunnel-connector-spark-file/src/main/scala/org/apache/seatunnel/spark/source/File.scala
index fa68c92..2f16187 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-file/src/main/scala/org/apache/seatunnel/spark/source/File.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-file/src/main/scala/org/apache/seatunnel/spark/source/File.scala
@@ -20,7 +20,7 @@ import scala.collection.JavaConversions._
import scala.util.{Failure, Success, Try}
import org.apache.seatunnel.common.config.{CheckResult, TypesafeConfigUtils}
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.spark.SparkEnvironment
import org.apache.seatunnel.spark.batch.SparkBatchSource
import org.apache.spark.sql.{Dataset, Row, SparkSession}
@@ -35,7 +35,7 @@ class File extends SparkBatchSource {
}
override def checkConfig(): CheckResult = {
- check(config, "path")
+ checkAllExists(config, "path")
}
protected def fileReader(spark: SparkSession, path: String): Dataset[Row] = {
diff --git a/seatunnel-connectors/seatunnel-connector-spark-hbase/src/main/scala/org/apache/seatunnel/spark/sink/Hbase.scala b/seatunnel-connectors/seatunnel-connector-spark-hbase/src/main/scala/org/apache/seatunnel/spark/sink/Hbase.scala
index c6fc08e..701faf6 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-hbase/src/main/scala/org/apache/seatunnel/spark/sink/Hbase.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-hbase/src/main/scala/org/apache/seatunnel/spark/sink/Hbase.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.hbase.spark.{ByteArrayWrapper, FamiliesQualifiersValues
import org.apache.hadoop.hbase.spark.datasources.HBaseTableCatalog
import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles
import org.apache.hadoop.hbase.util.Bytes
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.common.config.CheckResult
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
import org.apache.seatunnel.spark.SparkEnvironment
@@ -44,7 +44,7 @@ class Hbase extends SparkBatchSink with Logging {
var hbasePrefix = "hbase."
override def checkConfig(): CheckResult = {
- check(config, "hbase.zookeeper.quorum", "catalog", "staging_dir")
+ checkAllExists(config, "hbase.zookeeper.quorum", "catalog", "staging_dir")
}
override def prepare(env: SparkEnvironment): Unit = {
diff --git a/seatunnel-connectors/seatunnel-connector-spark-hive/src/main/scala/org/apache/seatunnel/spark/sink/Hive.scala b/seatunnel-connectors/seatunnel-connector-spark-hive/src/main/scala/org/apache/seatunnel/spark/sink/Hive.scala
index 2453a64..1cbb2f3 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-hive/src/main/scala/org/apache/seatunnel/spark/sink/Hive.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-hive/src/main/scala/org/apache/seatunnel/spark/sink/Hive.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.{DataFrameWriter, Dataset, Row}
class Hive extends SparkBatchSink with Logging {
override def checkConfig(): CheckResult = {
- CheckConfigUtil.check(config, "sql", "source_table_name", "result_table_name")
+ CheckConfigUtil.checkAllExists(config, "sql", "source_table_name", "result_table_name")
}
override def prepare(env: SparkEnvironment): Unit = {}
diff --git a/seatunnel-connectors/seatunnel-connector-spark-hive/src/main/scala/org/apache/seatunnel/spark/source/Hive.scala b/seatunnel-connectors/seatunnel-connector-spark-hive/src/main/scala/org/apache/seatunnel/spark/source/Hive.scala
index 267ed48..782e07d 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-hive/src/main/scala/org/apache/seatunnel/spark/source/Hive.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-hive/src/main/scala/org/apache/seatunnel/spark/source/Hive.scala
@@ -16,7 +16,7 @@
*/
package org.apache.seatunnel.spark.source
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.common.config.CheckResult
import org.apache.seatunnel.spark.SparkEnvironment
import org.apache.seatunnel.spark.batch.SparkBatchSource
@@ -27,7 +27,7 @@ class Hive extends SparkBatchSource {
override def prepare(env: SparkEnvironment): Unit = {}
override def checkConfig(): CheckResult = {
- check(config, "pre_sql")
+ checkAllExists(config, "pre_sql")
}
override def getData(env: SparkEnvironment): Dataset[Row] = {
diff --git a/seatunnel-connectors/seatunnel-connector-spark-hudi/src/main/scala/org/apache/seatunnel/spark/sink/Hudi.scala b/seatunnel-connectors/seatunnel-connector-spark-hudi/src/main/scala/org/apache/seatunnel/spark/sink/Hudi.scala
index f248317..e28573c 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-hudi/src/main/scala/org/apache/seatunnel/spark/sink/Hudi.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-hudi/src/main/scala/org/apache/seatunnel/spark/sink/Hudi.scala
@@ -18,7 +18,7 @@ package org.apache.seatunnel.spark.sink
import scala.collection.JavaConversions._
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.common.config.CheckResult
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
import org.apache.seatunnel.spark.SparkEnvironment
@@ -28,7 +28,7 @@ import org.apache.spark.sql.{Dataset, Row}
class Hudi extends SparkBatchSink {
override def checkConfig(): CheckResult = {
- check(config, "hoodie.base.path", "hoodie.table.name")
+ checkAllExists(config, "hoodie.base.path", "hoodie.table.name")
}
override def prepare(env: SparkEnvironment): Unit = {
diff --git a/seatunnel-connectors/seatunnel-connector-spark-hudi/src/main/scala/org/apache/seatunnel/spark/source/Hudi.scala b/seatunnel-connectors/seatunnel-connector-spark-hudi/src/main/scala/org/apache/seatunnel/spark/source/Hudi.scala
index 85851f1..18f64a2 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-hudi/src/main/scala/org/apache/seatunnel/spark/source/Hudi.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-hudi/src/main/scala/org/apache/seatunnel/spark/source/Hudi.scala
@@ -16,7 +16,7 @@
*/
package org.apache.seatunnel.spark.source
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import scala.collection.JavaConversions._
import org.apache.seatunnel.common.config.CheckResult
@@ -29,7 +29,7 @@ class Hudi extends SparkBatchSource {
override def prepare(env: SparkEnvironment): Unit = {}
override def checkConfig(): CheckResult = {
- check(config, "hoodie.datasource.read.paths")
+ checkAllExists(config, "hoodie.datasource.read.paths")
}
override def getData(env: SparkEnvironment): Dataset[Row] = {
diff --git a/seatunnel-connectors/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/sink/Jdbc.scala b/seatunnel-connectors/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/sink/Jdbc.scala
index bfa7f43..143e3d5 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/sink/Jdbc.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/sink/Jdbc.scala
@@ -18,7 +18,7 @@ package org.apache.seatunnel.spark.sink
import scala.collection.JavaConversions._
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.common.config.CheckResult
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
import org.apache.seatunnel.spark.SparkEnvironment
@@ -56,7 +56,7 @@ class Jdbc extends SparkBatchSink {
}
override def checkConfig(): CheckResult = {
- check(config, "driver", "url", "dbTable", "user", "password")
+ checkAllExists(config, "driver", "url", "dbTable", "user", "password")
}
override def prepare(prepareEnv: SparkEnvironment): Unit = {
diff --git a/seatunnel-connectors/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/source/Jdbc.scala b/seatunnel-connectors/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/source/Jdbc.scala
index d52e405..47db8ad 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/source/Jdbc.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/source/Jdbc.scala
@@ -20,7 +20,7 @@ import scala.collection.JavaConversions._
import scala.util.{Failure, Success, Try}
import org.apache.seatunnel.common.config.{CheckResult, TypesafeConfigUtils}
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.spark.SparkEnvironment
import org.apache.seatunnel.spark.batch.SparkBatchSource
import org.apache.spark.sql.{DataFrameReader, Dataset, Row, SparkSession}
@@ -34,7 +34,7 @@ class Jdbc extends SparkBatchSource {
}
override def checkConfig(): CheckResult = {
- check(config, "url", "table", "user", "password")
+ checkAllExists(config, "url", "table", "user", "password")
}
def jdbcReader(sparkSession: SparkSession, driver: String): DataFrameReader = {
diff --git a/seatunnel-connectors/seatunnel-connector-spark-kafka/src/main/scala/org/apache/seatunnel/spark/source/KafkaStream.scala b/seatunnel-connectors/seatunnel-connector-spark-kafka/src/main/scala/org/apache/seatunnel/spark/source/KafkaStream.scala
index cfba86b..8d656fd 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-kafka/src/main/scala/org/apache/seatunnel/spark/source/KafkaStream.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-kafka/src/main/scala/org/apache/seatunnel/spark/source/KafkaStream.scala
@@ -22,7 +22,7 @@ import scala.collection.JavaConversions._
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.seatunnel.common.config.{CheckResult, TypesafeConfigUtils}
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
import org.apache.seatunnel.spark.SparkEnvironment
import org.apache.seatunnel.spark.stream.SparkStreamingSource
@@ -103,10 +103,10 @@ class KafkaStream extends SparkStreamingSource[(String, String)] {
}
override def checkConfig(): CheckResult = {
- val checkResult = check(config, "topics")
+ val checkResult = checkAllExists(config, "topics")
if (checkResult.isSuccess) {
val consumerConfig = TypesafeConfigUtils.extractSubConfig(config, consumerPrefix, false)
- check(consumerConfig, "group.id")
+ checkAllExists(consumerConfig, "group.id")
} else {
checkResult
}
diff --git a/seatunnel-connectors/seatunnel-connector-spark-kudu/src/main/scala/org/apache/seatunnel/spark/sink/Kudu.scala b/seatunnel-connectors/seatunnel-connector-spark-kudu/src/main/scala/org/apache/seatunnel/spark/sink/Kudu.scala
index 9f7ef86..2ce7bba 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-kudu/src/main/scala/org/apache/seatunnel/spark/sink/Kudu.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-kudu/src/main/scala/org/apache/seatunnel/spark/sink/Kudu.scala
@@ -19,7 +19,7 @@ package org.apache.seatunnel.spark.sink
import scala.collection.JavaConversions._
import org.apache.kudu.spark.kudu._
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.common.config.CheckResult
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
import org.apache.seatunnel.spark.SparkEnvironment
@@ -36,7 +36,7 @@ class Kudu extends SparkBatchSink {
}
override def checkConfig(): CheckResult = {
- check(config, "kudu_master", "kudu_table")
+ checkAllExists(config, "kudu_master", "kudu_table")
}
override def output(df: Dataset[Row], environment: SparkEnvironment): Unit = {
diff --git a/seatunnel-connectors/seatunnel-connector-spark-kudu/src/main/scala/org/apache/seatunnel/spark/source/Kudu.scala b/seatunnel-connectors/seatunnel-connector-spark-kudu/src/main/scala/org/apache/seatunnel/spark/source/Kudu.scala
index 8334677..0dba749 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-kudu/src/main/scala/org/apache/seatunnel/spark/source/Kudu.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-kudu/src/main/scala/org/apache/seatunnel/spark/source/Kudu.scala
@@ -18,7 +18,7 @@
package org.apache.seatunnel.spark.source
import org.apache.kudu.spark.kudu._
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.common.config.CheckResult
import org.apache.seatunnel.spark.SparkEnvironment
import org.apache.seatunnel.spark.batch.SparkBatchSource
@@ -27,7 +27,7 @@ import org.apache.spark.sql.{Dataset, Row}
class Kudu extends SparkBatchSource {
override def checkConfig(): CheckResult = {
- check(config, "kudu_master", "kudu_table")
+ checkAllExists(config, "kudu_master", "kudu_table")
}
override def getData(env: SparkEnvironment): Dataset[Row] = {
diff --git a/seatunnel-connectors/seatunnel-connector-spark-neo4j/src/main/scala/org/apache/seatunnel/spark/source/Neo4j.scala b/seatunnel-connectors/seatunnel-connector-spark-neo4j/src/main/scala/org/apache/seatunnel/spark/source/Neo4j.scala
index 20b0797..2d47118 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-neo4j/src/main/scala/org/apache/seatunnel/spark/source/Neo4j.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-neo4j/src/main/scala/org/apache/seatunnel/spark/source/Neo4j.scala
@@ -17,20 +17,20 @@
package org.apache.seatunnel.spark.source
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
import org.apache.seatunnel.common.config.{CheckConfigUtil, CheckResult}
+import org.apache.seatunnel.common.config.CheckConfigUtil.{checkAllExists, checkAtLeastOneExists}
import org.apache.seatunnel.spark.SparkEnvironment
import org.apache.seatunnel.spark.batch.SparkBatchSource
import org.apache.spark.sql.{DataFrameReader, Dataset, Row}
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
class Neo4j extends SparkBatchSource {
val neo4jConf: mutable.Map[String, String] = mutable.Map()
val readFormatter: String = "org.neo4j.spark.DataSource"
-
override def getData(env: SparkEnvironment): Dataset[Row] = {
val read: DataFrameReader = env.getSparkSession.read
read.format(readFormatter)
@@ -42,10 +42,9 @@ class Neo4j extends SparkBatchSource {
}
override def checkConfig(): CheckResult = {
-
- val checkMustConfigOneOfParams: CheckResult = CheckConfigUtil.checkOne(config, "query", "labels", "relationship")
- val checkMustConfigAllParams: CheckResult = CheckConfigUtil.check(config, "result_table_name", "url")
- CheckConfigUtil.mergeCheckMessage(checkMustConfigAllParams, checkMustConfigOneOfParams)
+ val checkMustConfigOneOfParams = checkAtLeastOneExists(config, "query", "labels", "relationship")
+ val checkMustConfigAllParams = checkAllExists(config, "result_table_name", "url")
+ CheckConfigUtil.mergeCheckResults(checkMustConfigAllParams, checkMustConfigOneOfParams)
}
diff --git a/seatunnel-connectors/seatunnel-connector-spark-phoenix/src/main/scala/org/apache/seatunnel/spark/sink/Phoenix.scala b/seatunnel-connectors/seatunnel-connector-spark-phoenix/src/main/scala/org/apache/seatunnel/spark/sink/Phoenix.scala
index 9fa65a2..a6cbbc2 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-phoenix/src/main/scala/org/apache/seatunnel/spark/sink/Phoenix.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-phoenix/src/main/scala/org/apache/seatunnel/spark/sink/Phoenix.scala
@@ -55,7 +55,7 @@ class Phoenix extends SparkBatchSink with Logging {
}
override def checkConfig(): CheckResult = {
- val checkResult = CheckConfigUtil.check(config, "zk-connect", "table")
+ val checkResult = CheckConfigUtil.checkAllExists(config, "zk-connect", "table")
if (checkResult.isSuccess) {
checkZkConnect(config.getString("zk-connect"))
}
diff --git a/seatunnel-connectors/seatunnel-connector-spark-phoenix/src/main/scala/org/apache/seatunnel/spark/source/Phoenix.scala b/seatunnel-connectors/seatunnel-connector-spark-phoenix/src/main/scala/org/apache/seatunnel/spark/source/Phoenix.scala
index 78693ed..9fdbd2b 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-phoenix/src/main/scala/org/apache/seatunnel/spark/source/Phoenix.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-phoenix/src/main/scala/org/apache/seatunnel/spark/source/Phoenix.scala
@@ -53,7 +53,7 @@ class Phoenix extends SparkBatchSource with Logging {
}
override def checkConfig(): CheckResult = {
- val checkResult = CheckConfigUtil.check(config, "zk-connect", "table")
+ val checkResult = CheckConfigUtil.checkAllExists(config, "zk-connect", "table")
if (checkResult.isSuccess) {
checkZkConnect(config.getString("zk-connect"))
}
diff --git a/seatunnel-connectors/seatunnel-connector-spark-tidb/src/main/scala/org/apache/seatunnel/spark/sink/Tidb.scala b/seatunnel-connectors/seatunnel-connector-spark-tidb/src/main/scala/org/apache/seatunnel/spark/sink/Tidb.scala
index 342f602..b955b36 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-tidb/src/main/scala/org/apache/seatunnel/spark/sink/Tidb.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-tidb/src/main/scala/org/apache/seatunnel/spark/sink/Tidb.scala
@@ -17,7 +17,7 @@
package org.apache.seatunnel.spark.sink
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.common.config.{CheckResult, TypesafeConfigUtils}
import org.apache.seatunnel.spark.SparkEnvironment
import org.apache.seatunnel.spark.batch.SparkBatchSink
@@ -58,7 +58,7 @@ class Tidb extends SparkBatchSink {
}
override def checkConfig(): CheckResult = {
- check(config, "addr", "port", "database", "table", "user", "password")
+ checkAllExists(config, "addr", "port", "database", "table", "user", "password")
}
override def prepare(prepareEnv: SparkEnvironment): Unit = {
diff --git a/seatunnel-connectors/seatunnel-connector-spark-tidb/src/main/scala/org/apache/seatunnel/spark/source/Tidb.scala b/seatunnel-connectors/seatunnel-connector-spark-tidb/src/main/scala/org/apache/seatunnel/spark/source/Tidb.scala
index 5d067c7..352bf4f 100644
--- a/seatunnel-connectors/seatunnel-connector-spark-tidb/src/main/scala/org/apache/seatunnel/spark/source/Tidb.scala
+++ b/seatunnel-connectors/seatunnel-connector-spark-tidb/src/main/scala/org/apache/seatunnel/spark/source/Tidb.scala
@@ -17,7 +17,7 @@
package org.apache.seatunnel.spark.source
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.common.config.CheckResult
import org.apache.seatunnel.spark.SparkEnvironment
import org.apache.seatunnel.spark.batch.SparkBatchSource
@@ -28,7 +28,7 @@ class Tidb extends SparkBatchSource {
override def prepare(env: SparkEnvironment): Unit = {}
override def checkConfig(): CheckResult = {
- check(config, "pre_sql", "database")
+ checkAllExists(config, "pre_sql", "database")
}
override def getData(env: SparkEnvironment): Dataset[Row] = {
diff --git a/seatunnel-transforms/seatunnel-transform-flink-datastream2table/src/main/java/org/apache/seatunnel/flink/transform/DataStreamToTable.java b/seatunnel-transforms/seatunnel-transform-flink-datastream2table/src/main/java/org/apache/seatunnel/flink/transform/DataStreamToTable.java
index 7e20257..8ea836d 100644
--- a/seatunnel-transforms/seatunnel-transform-flink-datastream2table/src/main/java/org/apache/seatunnel/flink/transform/DataStreamToTable.java
+++ b/seatunnel-transforms/seatunnel-transform-flink-datastream2table/src/main/java/org/apache/seatunnel/flink/transform/DataStreamToTable.java
@@ -59,7 +59,7 @@ public class DataStreamToTable implements FlinkStreamTransform<Row, Row>, FlinkB
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, Plugin.RESULT_TABLE_NAME);
+ return CheckConfigUtil.checkAllExists(config, Plugin.RESULT_TABLE_NAME);
}
@Override
diff --git a/seatunnel-transforms/seatunnel-transform-flink-split/src/main/java/org/apache/seatunnel/flink/transform/Split.java b/seatunnel-transforms/seatunnel-transform-flink-split/src/main/java/org/apache/seatunnel/flink/transform/Split.java
index a9c0d50..30aebd0 100644
--- a/seatunnel-transforms/seatunnel-transform-flink-split/src/main/java/org/apache/seatunnel/flink/transform/Split.java
+++ b/seatunnel-transforms/seatunnel-transform-flink-split/src/main/java/org/apache/seatunnel/flink/transform/Split.java
@@ -82,7 +82,7 @@ public class Split implements FlinkStreamTransform<Row, Row>, FlinkBatchTransfor
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, FIELDS);
+ return CheckConfigUtil.checkAllExists(config, FIELDS);
}
@Override
diff --git a/seatunnel-transforms/seatunnel-transform-flink-sql/src/main/java/org/apache/seatunnel/flink/transform/Sql.java b/seatunnel-transforms/seatunnel-transform-flink-sql/src/main/java/org/apache/seatunnel/flink/transform/Sql.java
index c8f46d2..4c1355c 100644
--- a/seatunnel-transforms/seatunnel-transform-flink-sql/src/main/java/org/apache/seatunnel/flink/transform/Sql.java
+++ b/seatunnel-transforms/seatunnel-transform-flink-sql/src/main/java/org/apache/seatunnel/flink/transform/Sql.java
@@ -65,7 +65,7 @@ public class Sql implements FlinkStreamTransform<Row, Row>, FlinkBatchTransform<
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, SQL);
+ return CheckConfigUtil.checkAllExists(config, SQL);
}
@Override
diff --git a/seatunnel-transforms/seatunnel-transform-flink-table2datastream/src/main/java/org/apache/seatunnel/flink/transform/TableToDataStream.java b/seatunnel-transforms/seatunnel-transform-flink-table2datastream/src/main/java/org/apache/seatunnel/flink/transform/TableToDataStream.java
index 29f0e09..ecdb394 100644
--- a/seatunnel-transforms/seatunnel-transform-flink-table2datastream/src/main/java/org/apache/seatunnel/flink/transform/TableToDataStream.java
+++ b/seatunnel-transforms/seatunnel-transform-flink-table2datastream/src/main/java/org/apache/seatunnel/flink/transform/TableToDataStream.java
@@ -66,7 +66,7 @@ public class TableToDataStream implements FlinkStreamTransform<Row, Row>, FlinkB
@Override
public CheckResult checkConfig() {
- return CheckConfigUtil.check(config, Plugin.SOURCE_TABLE_NAME);
+ return CheckConfigUtil.checkAllExists(config, Plugin.SOURCE_TABLE_NAME);
}
@Override
diff --git a/seatunnel-transforms/seatunnel-transform-spark-split/src/main/scala/org/apache/seatunnel/spark/transform/Split.scala b/seatunnel-transforms/seatunnel-transform-spark-split/src/main/scala/org/apache/seatunnel/spark/transform/Split.scala
index 5bc326b..968b92e 100644
--- a/seatunnel-transforms/seatunnel-transform-spark-split/src/main/scala/org/apache/seatunnel/spark/transform/Split.scala
+++ b/seatunnel-transforms/seatunnel-transform-spark-split/src/main/scala/org/apache/seatunnel/spark/transform/Split.scala
@@ -19,7 +19,7 @@ package org.apache.seatunnel.spark.transform
import org.apache.seatunnel.common.Constants
import scala.collection.JavaConversions._
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.common.config.CheckResult
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
import org.apache.seatunnel.spark.{BaseSparkTransform, SparkEnvironment}
@@ -57,7 +57,7 @@ class Split extends BaseSparkTransform {
}
override def checkConfig(): CheckResult = {
- check(config, "fields")
+ checkAllExists(config, "fields")
}
override def prepare(env: SparkEnvironment): Unit = {
diff --git a/seatunnel-transforms/seatunnel-transform-spark-sql/src/main/scala/org/apache/seatunnel/spark/transform/Sql.scala b/seatunnel-transforms/seatunnel-transform-spark-sql/src/main/scala/org/apache/seatunnel/spark/transform/Sql.scala
index 0a7f6d2..917c117 100644
--- a/seatunnel-transforms/seatunnel-transform-spark-sql/src/main/scala/org/apache/seatunnel/spark/transform/Sql.scala
+++ b/seatunnel-transforms/seatunnel-transform-spark-sql/src/main/scala/org/apache/seatunnel/spark/transform/Sql.scala
@@ -16,7 +16,7 @@
*/
package org.apache.seatunnel.spark.transform
-import org.apache.seatunnel.common.config.CheckConfigUtil.check
+import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.common.config.CheckResult
import org.apache.seatunnel.spark.{BaseSparkTransform, SparkEnvironment}
import org.apache.spark.sql.{Dataset, Row}
@@ -28,7 +28,7 @@ class Sql extends BaseSparkTransform {
}
override def checkConfig(): CheckResult = {
- check(config, "sql")
+ checkAllExists(config, "sql")
}
override def prepare(env: SparkEnvironment): Unit = {}