You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by fa...@apache.org on 2022/07/26 06:49:52 UTC
[incubator-seatunnel] branch 2.1.3-prepare updated: [Release]2.1.3-Prepare (#2264)
This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a commit to branch 2.1.3-prepare
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/2.1.3-prepare by this push:
new aac09675d [Release]2.1.3-Prepare (#2264)
aac09675d is described below
commit aac09675d11c3b10b7598bd435c802352ab58946
Author: Kirs <ki...@apache.org>
AuthorDate: Tue Jul 26 14:49:48 2022 +0800
[Release]2.1.3-Prepare (#2264)
* [Bug] [seatunnel-core] Failed to get APP_DIR path bug fixed (#2165)
[Core][Starter] When use cluster mode, but starter app root dir also should same as client mode. (#2141)
[bug]fix commandArgs -t(--check) conflict with flink deployment t… (#2174)
[Core][Starter] Change jar connector load logic (#2193)
[Core]Add plugin discovery module (#1881)
Upgrade common-collecions4 to 4.4
Upgrade common-codec to 1.13
* add plugin discover module
* Add SeaTunnel API module
* fix license check error
* ignore 403 deadlink check
* move plugin-mapping.properties to root path
* Revert "move plugin-mapping.properties to root path"
This reverts commit 372968ee0e753d48b24c256fb8e379222b59daf0.
* add fake-stream config
Co-authored-by: Wenjun Ruan <we...@apache.org>
---
.dlc.json | 3 +-
pom.xml | 61 ++----
.../seatunnel-api-base => seatunnel-api}/pom.xml | 24 ++-
.../api/common/PluginIdentifierInterface.java | 21 +-
.../seatunnel/api/common/PrepareFailException.java | 30 ++-
.../seatunnel/api/common/SeaTunnelContext.java | 87 ++++++++
.../api/common/SeaTunnelPluginLifeCycle.java | 35 ++-
.../api/serialization/DefaultSerializer.java | 31 ++-
.../api/serialization/DeserializationSchema.java | 36 ++--
.../api/serialization/SerializationSchema.java | 21 +-
.../seatunnel/api/serialization/Serializer.java | 28 ++-
.../api/sink/DefaultSinkWriterContext.java | 21 +-
.../apache/seatunnel/api/sink/SeaTunnelSink.java | 122 +++++++++++
.../api/sink/SinkAggregatedCommitter.java | 66 ++++++
.../apache/seatunnel/api/sink/SinkCommitter.java | 50 +++++
.../org/apache/seatunnel/api/sink/SinkWriter.java | 84 ++++++++
.../apache/seatunnel/api/source/Boundedness.java | 22 +-
.../org/apache/seatunnel/api/source/Collector.java | 20 +-
.../api/source/SeaTunnelContextAware.java | 19 +-
.../seatunnel/api/source/SeaTunnelSource.java | 101 +++++++++
.../apache/seatunnel/api/source/SourceEvent.java | 18 +-
.../apache/seatunnel/api/source/SourceReader.java | 119 +++++++++++
.../apache/seatunnel/api/source/SourceSplit.java | 18 +-
.../api/source/SourceSplitEnumerator.java | 124 +++++++++++
.../seatunnel/api/source/SupportCoordinate.java | 18 +-
.../seatunnel/api/state/CheckpointListener.java | 18 +-
.../seatunnel/api/table/catalog/Catalog.java | 116 ++++++++++
.../seatunnel/api/table/catalog/CatalogTable.java | 93 ++++++++
.../apache/seatunnel/api/table/catalog/Column.java | 223 +++++++++++++++++++
.../api/table/catalog/TableIdentifier.java | 81 +++++++
.../seatunnel/api/table/catalog/TablePath.java | 86 ++++++++
.../seatunnel/api/table/catalog/TableSchema.java | 132 ++++++++++++
.../table/catalog/exception/CatalogException.java | 31 ++-
.../exception/DatabaseNotExistException.java | 32 +++
.../catalog/exception/TableNotExistException.java | 35 +++
.../api/table/connector/DeserializationFormat.java | 28 ++-
.../api/table/connector/SerializationFormat.java | 15 +-
.../table/connector/SupportReadingMetadata.java | 21 +-
.../seatunnel/api/table/connector/TableSink.java | 15 +-
.../seatunnel/api/table/connector/TableSource.java | 16 +-
.../api/table/factory/CatalogFactory.java | 19 +-
.../factory/DeserializationFormatFactory.java | 16 +-
.../seatunnel/api/table/factory/Factory.java | 21 +-
.../api/table/factory/FactoryException.java | 15 +-
.../seatunnel/api/table/factory/FactoryUtil.java | 156 ++++++++++++++
.../table/factory/SerializationFormatFactory.java | 16 +-
.../api/table/factory/SupportMultipleTable.java | 60 ++++++
.../api/table/factory/TableFactoryContext.java | 58 +++++
.../api/table/factory/TableSinkFactory.java | 20 +-
.../api/table/factory/TableSourceFactory.java | 31 +--
.../apache/seatunnel/api/table/type/ArrayType.java | 87 ++++++++
.../apache/seatunnel/api/table/type/BasicType.java | 79 +++++++
.../seatunnel/api/table/type/CompositeType.java | 15 +-
.../seatunnel/api/table/type/DecimalType.java | 65 ++++++
.../seatunnel/api/table/type/LocalTimeType.java | 72 +++++++
.../apache/seatunnel/api/table/type/MapType.java | 104 +++++++++
.../api/table/type/PrimitiveByteArrayType.java | 38 ++--
.../apache/seatunnel/api/table/type/RowKind.java | 117 ++++++++++
.../api/table/type/SeaTunnelDataType.java | 19 +-
.../seatunnel/api/table/type/SeaTunnelRow.java | 105 +++++++++
.../seatunnel/api/table/type/SeaTunnelRowType.java | 120 +++++++++++
.../apache/seatunnel/api/table/type/SqlType.java | 34 +--
seatunnel-apis/pom.xml | 4 +-
seatunnel-apis/seatunnel-api-base/pom.xml | 5 +-
.../apache/seatunnel/apis/base/env/RuntimeEnv.java | 4 +-
.../apache/seatunnel/apis/base/plugin/Plugin.java | 14 +-
seatunnel-apis/seatunnel-api-flink/pom.xml | 5 +-
.../apache/seatunnel/flink/FlinkEnvironment.java | 58 ++---
seatunnel-apis/seatunnel-api-spark/pom.xml | 5 +-
.../apache/seatunnel/spark/SparkEnvironment.java | 4 +-
seatunnel-common/pom.xml | 13 +-
.../apache/seatunnel/common/PropertiesUtil.java | 37 ++++
.../org/apache/seatunnel/common/config/Common.java | 37 ++--
.../common/constants/CollectionConstants.java | 15 +-
.../seatunnel/common/constants}/PluginType.java | 6 +-
.../seatunnel/common/utils/ReflectionUtils.java | 56 ++++-
.../common/utils/SerializationException.java | 68 ++++++
.../seatunnel/common/utils/SerializationUtils.java | 79 +++++++
.../apache/seatunnel/common/config/CommonTest.java | 2 +-
.../ReflectionUtilsTest.java} | 27 ++-
.../common/utils/SerializationUtilsTest.java | 74 +++++++
.../common/utils/VariablesSubstituteTest.java | 1 -
seatunnel-connectors/plugin-mapping.properties | 2 +
.../clickhouse/sink/ClickhouseOutputFormat.java | 2 +-
.../org/apache/seatunnel/flink/fake/Config.java | 2 +-
.../flink/fake/source/FakeSourceStream.java | 2 +-
.../apache/seatunnel/spark/jdbc/source/Jdbc.scala | 4 +-
.../spark/jdbc/source/util/HiveDialet.scala | 2 +-
seatunnel-core/README.md | 8 +
seatunnel-core/pom.xml | 4 +-
seatunnel-core/seatunnel-core-base/pom.xml | 11 +-
.../org/apache/seatunnel/core/base/Seatunnel.java | 3 +-
.../core/base/command/AbstractCommandArgs.java | 3 +-
.../core/base/command/BaseTaskExecuteCommand.java | 5 +-
.../seatunnel/core/base/command/Command.java | 3 +-
...nContext.java => AbstractExecutionContext.java} | 43 ++--
.../seatunnel/core/base/config/ConfigBuilder.java | 21 +-
.../seatunnel/core/base/config/ConfigChecker.java | 36 ++--
.../core/base/config/EnvironmentFactory.java | 2 +
.../core/base/config/ExecutionFactory.java | 4 +-
.../seatunnel/core/base/config/PluginFactory.java | 237 ---------------------
.../CommandException.java} | 16 +-
.../CommandExecuteException.java} | 16 +-
.../ConfigCheckException.java} | 16 +-
.../core/base/config/PluginFactoryTest.java | 65 ------
.../seatunnel/core/base/utils/FileUtilsTest.java | 1 +
seatunnel-core/seatunnel-core-flink-sql/pom.xml | 4 +-
.../apache/seatunnel/core/sql/FlinkSqlStarter.java | 7 +-
.../apache/seatunnel/core/sql/job/Executor.java | 16 +-
seatunnel-core/seatunnel-core-flink/pom.xml | 28 ++-
.../apache/seatunnel/core/flink/FlinkStarter.java | 5 +-
.../seatunnel/core/flink/SeatunnelFlink.java | 3 +-
.../core/flink/args/FlinkCommandArgs.java | 4 +-
...mmand.java => FlinkApiConfValidateCommand.java} | 14 +-
...ommand.java => FlinkApiTaskExecuteCommand.java} | 37 ++--
.../core/flink/command/FlinkCommandBuilder.java | 9 +-
.../core/flink/config/FlinkApiConfigChecker.java | 37 ++--
.../core/flink/config/FlinkExecutionContext.java | 106 +++++++++
.../core/flink/utils/CommandLineUtils.java | 17 +-
.../core/flink/args/FlinkCommandArgsTest.java | 2 +-
.../flink/command/FlinkTaskExecuteCommandTest.java | 2 +-
.../core/flink/utils/CommandLineUtilsTest.java | 27 +--
seatunnel-core/seatunnel-core-spark/pom.xml | 24 ++-
.../seatunnel/core/spark/SeatunnelSpark.java | 3 +-
.../apache/seatunnel/core/spark/SparkStarter.java | 57 ++---
.../core/spark/command/SparkCommandBuilder.java | 5 +-
.../spark/command/SparkConfValidateCommand.java | 8 +-
.../spark/command/SparkTaskExecuteCommand.java | 59 +----
.../core/spark/config/SparkApiConfigChecker.java | 37 ++--
.../core/spark/config/SparkExecutionContext.java | 105 +++++++++
seatunnel-dist/release-docs/LICENSE | 5 +-
.../seatunnel/example/flink/LocalFlinkExample.java | 3 +-
.../seatunnel/example/spark/LocalSparkExample.java | 3 +-
.../pom.xml | 33 ++-
.../plugin/discovery/AbstractPluginDiscovery.java | 211 ++++++++++++++++++
.../plugin/discovery/PluginDiscovery.java | 70 ++++++
.../plugin/discovery/PluginIdentifier.java | 88 ++++++++
.../flink/FlinkAbstractPluginDiscovery.java | 40 ++++
.../discovery/flink/FlinkSinkPluginDiscovery.java | 16 +-
.../flink/FlinkSourcePluginDiscovery.java | 17 +-
.../flink/FlinkTransformPluginDiscovery.java | 39 ++--
.../SeaTunnelFlinkTransformPluginDiscovery.java | 20 +-
.../seatunnel/SeaTunnelSinkPluginDiscovery.java | 37 ++--
.../seatunnel/SeaTunnelSourcePluginDiscovery.java | 37 ++--
.../SeaTunnelSparkTransformPluginDiscovery.java | 18 +-
.../discovery/spark/SparkSinkPluginDiscovery.java | 17 +-
.../spark/SparkSourcePluginDiscovery.java | 17 +-
.../spark/SparkTransformPluginDiscovery.java | 39 ++--
tools/checkstyle/checkStyle.xml | 9 +-
tools/dependencies/known-dependencies.txt | 6 +-
150 files changed, 4427 insertions(+), 1324 deletions(-)
diff --git a/.dlc.json b/.dlc.json
index db4f4d1f8..480fdf1f2 100644
--- a/.dlc.json
+++ b/.dlc.json
@@ -26,6 +26,7 @@
"aliveStatusCodes": [
0,
200,
- 401
+ 401,
+ 403
]
}
diff --git a/pom.xml b/pom.xml
index 25c155c60..d368a1c49 100644
--- a/pom.xml
+++ b/pom.xml
@@ -79,9 +79,11 @@
-->
<module>seatunnel-common</module>
<module>seatunnel-apis</module>
+ <module>seatunnel-api</module>
<module>seatunnel-core</module>
<module>seatunnel-transforms</module>
<module>seatunnel-connectors</module>
+ <module>seatunnel-plugin-discovery</module>
<module>seatunnel-dist</module>
<module>seatunnel-examples</module>
<module>seatunnel-e2e</module>
@@ -108,7 +110,6 @@
<lombok.version>1.18.0</lombok.version>
<mysql.version>8.0.16</mysql.version>
<postgresql.version>42.3.3</postgresql.version>
- <skip.pmd.check>false</skip.pmd.check>
<maven.deploy.skip>false</maven.deploy.skip>
<maven.javadoc.skip>false</maven.javadoc.skip>
<maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version>
@@ -121,7 +122,6 @@
<maven-deploy-plugin.version>2.8.2</maven-deploy-plugin.version>
<scala-maven-plugin.version>3.3.1</scala-maven-plugin.version>
<maven-compiler-plugin.version>2.0.2</maven-compiler-plugin.version>
- <maven-pmd-plugin.version>3.8</maven-pmd-plugin.version>
<spoiwo.version>1.8.0</spoiwo.version>
<play-mailer.version>7.0.2</play-mailer.version>
<phoenix-spark.version>5.0.0-HBase-2.0</phoenix-spark.version>
@@ -137,9 +137,11 @@
<mongo-spark.version>2.2.0</mongo-spark.version>
<spark-redis.version>2.6.0</spark-redis.version>
<commons-lang3.version>3.4</commons-lang3.version>
+ <commons-collections4.version>4.4</commons-collections4.version>
<maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version>
<spark.scope>provided</spark.scope>
<flink.scope>provided</flink.scope>
+ <codec.version>1.13</codec.version>
<httpclient.version>4.5.13</httpclient.version>
<httpcore.version>4.4.4</httpcore.version>
<httpcore-nio.version>4.4.4</httpcore-nio.version>
@@ -297,6 +299,12 @@
<version>${spoiwo.version}</version>
</dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>${codec.version}</version>
+ </dependency>
+
<dependency>
<groupId>com.typesafe.play</groupId>
<artifactId>play-mailer_${scala.binary.version}</artifactId>
@@ -422,6 +430,11 @@
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-collections4</artifactId>
+ <version>${commons-collections4.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.flink</groupId>
@@ -860,42 +873,7 @@
</execution>
</executions>
</plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-pmd-plugin</artifactId>
- <version>${maven-pmd-plugin.version}</version>
- <configuration>
- <rulesets>
- <ruleset>rulesets/java/ali-concurrent.xml</ruleset>
- <ruleset>rulesets/java/ali-exception.xml</ruleset>
- <ruleset>rulesets/java/ali-flowcontrol.xml</ruleset>
- <ruleset>rulesets/java/ali-naming.xml</ruleset>
- <ruleset>rulesets/java/ali-oop.xml</ruleset>
- <ruleset>rulesets/java/ali-orm.xml</ruleset>
- <ruleset>rulesets/java/ali-other.xml</ruleset>
- <ruleset>rulesets/java/ali-set.xml</ruleset>
- </rulesets>
- <printFailingErrors>true</printFailingErrors>
- <skip>${skip.pmd.check}</skip>
- </configuration>
- <executions>
- <execution>
- <id>validate</id>
- <phase>validate</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>com.alibaba.p3c</groupId>
- <artifactId>p3c-pmd</artifactId>
- <version>${p3c-pmd.version}</version>
- </dependency>
- </dependencies>
- </plugin>
+
<!-- checkstyle (End) -->
<plugin>
@@ -1041,12 +1019,7 @@
<groupId>org.scalastyle</groupId>
<artifactId>scalastyle-maven-plugin</artifactId>
</plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-pmd-plugin</artifactId>
- </plugin>
-
+
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>license-maven-plugin</artifactId>
diff --git a/seatunnel-apis/seatunnel-api-base/pom.xml b/seatunnel-api/pom.xml
similarity index 69%
copy from seatunnel-apis/seatunnel-api-base/pom.xml
copy to seatunnel-api/pom.xml
index a31906b20..8372d2648 100644
--- a/seatunnel-apis/seatunnel-api-base/pom.xml
+++ b/seatunnel-api/pom.xml
@@ -17,15 +17,20 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
+ <artifactId>seatunnel</artifactId>
<groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-apis</artifactId>
<version>2.1.3-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>seatunnel-api-base</artifactId>
+
+ <artifactId>seatunnel-api</artifactId>
+
+ <properties>
+ </properties>
<dependencies>
<dependency>
@@ -33,10 +38,13 @@
<artifactId>seatunnel-common</artifactId>
<version>${project.version}</version>
</dependency>
-
<dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-config-shade</artifactId>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
</dependency>
</dependencies>
-</project>
+</project>
\ No newline at end of file
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/PluginIdentifierInterface.java
similarity index 64%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/common/PluginIdentifierInterface.java
index 5cd80fc53..edc56d508 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/PluginIdentifierInterface.java
@@ -15,21 +15,18 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
-
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+package org.apache.seatunnel.api.common;
/**
- * Command interface.
- *
- * @param <T> args type
+ * todo: unified with Plugin
*/
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
-
+public interface PluginIdentifierInterface {
/**
- * Execute command
+ * Returns a unique identifier among same factory interfaces.
+ *
+ * <p>For consistency, an identifier should be declared as one lower case word (e.g. {@code
+ * kafka}). If multiple factories exist for different versions, a version should be appended
+ * using "-" (e.g. {@code elasticsearch-7}).
*/
- void execute();
-
+ String getPluginName();
}
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/PrepareFailException.java
similarity index 55%
copy from seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/common/PrepareFailException.java
index e9052bfe6..7cb75b98f 100644
--- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/PrepareFailException.java
@@ -15,28 +15,24 @@
* limitations under the License.
*/
-package org.apache.seatunnel.common;
+package org.apache.seatunnel.api.common;
-import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.common.constants.PluginType;
-import java.util.Properties;
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
-public final class PropertiesUtil {
+/**
+ * This exception will throw when {@link SeaTunnelPluginLifeCycle#prepare(Config)} failed.
+ */
+public class PrepareFailException extends RuntimeException {
- private PropertiesUtil() {
+ public PrepareFailException(String pluginName, PluginType type, String message) {
+ super(String.format("PluginName: %s, PluginType: %s, Message: %s", pluginName, type.getType(),
+ message));
}
- public static void setProperties(Config config, Properties properties, String prefix, boolean keepPrefix) {
- config.entrySet().forEach(entry -> {
- String key = entry.getKey();
- Object value = entry.getValue().unwrapped();
- if (key.startsWith(prefix)) {
- if (keepPrefix) {
- properties.put(key, value);
- } else {
- properties.put(key.substring(prefix.length()), value);
- }
- }
- });
+ public PrepareFailException(String pluginName, PluginType type, String message, Throwable cause) {
+ super(String.format("PluginName: %s, PluginType: %s, Message: %s", pluginName, type.getType(),
+ message), cause);
}
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/SeaTunnelContext.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/SeaTunnelContext.java
new file mode 100644
index 000000000..b6152e44f
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/SeaTunnelContext.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.common;
+
+import org.apache.seatunnel.api.table.catalog.TableSchema;
+import org.apache.seatunnel.common.config.Common;
+import org.apache.seatunnel.common.constants.JobMode;
+
+import java.io.Serializable;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * This class is used to store the context of the application. e.g. the table schema, catalog...etc.
+ */
+public final class SeaTunnelContext implements Serializable {
+
+ private static final long serialVersionUID = -1L;
+
+ private static final SeaTunnelContext INSTANCE = new SeaTunnelContext();
+
+ // tableName -> tableSchema
+ private final Map<String, TableSchema> tableSchemaMap = new ConcurrentHashMap<>(Common.COLLECTION_SIZE);
+
+ private JobMode jobMode;
+
+ private final String jobId;
+
+ public static SeaTunnelContext getContext() {
+ return INSTANCE;
+ }
+
+ /**
+ * Put table schema.
+ *
+ * @param tableName table name
+ * @param tableSchema table schema
+ */
+ public void addSchema(String tableName, TableSchema tableSchema) {
+ tableSchemaMap.put(tableName, tableSchema);
+ }
+
+ /**
+ * Get table schema.
+ *
+ * @param tableName table name.
+ * @return table schema.
+ */
+ public Optional<TableSchema> getSchema(String tableName) {
+ return Optional.ofNullable(tableSchemaMap.get(tableName));
+ }
+
+ public SeaTunnelContext setJobMode(JobMode jobMode) {
+ this.jobMode = jobMode;
+ return this;
+ }
+
+ public JobMode getJobMode() {
+ return jobMode;
+ }
+
+ public String getJobId() {
+ return this.jobId;
+ }
+
+ private SeaTunnelContext() {
+ this.jobId = UUID.randomUUID().toString().replace("-", "");
+ }
+
+}
diff --git a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/SeaTunnelPluginLifeCycle.java
similarity index 62%
copy from seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/common/SeaTunnelPluginLifeCycle.java
index f0cf54447..30d7c0f57 100644
--- a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/SeaTunnelPluginLifeCycle.java
@@ -15,33 +15,22 @@
* limitations under the License.
*/
-package org.apache.seatunnel.apis.base.env;
-
-import org.apache.seatunnel.common.config.CheckResult;
-import org.apache.seatunnel.common.constants.JobMode;
+package org.apache.seatunnel.api.common;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
-import java.net.URL;
-import java.util.Set;
-
/**
- * engine related runtime environment
+ * This interface is the life cycle of a plugin, after a plugin created,
+ * will execute prepare method to do some initialize operation.
*/
-public interface RuntimeEnv {
-
- RuntimeEnv setConfig(Config config);
-
- Config getConfig();
-
- CheckResult checkConfig();
-
- RuntimeEnv prepare();
-
- RuntimeEnv setJobMode(JobMode mode);
-
- JobMode getJobMode();
-
- void registerPlugin(Set<URL> pluginPaths);
+public interface SeaTunnelPluginLifeCycle {
+
+ /**
+ * Use the pluginConfig to do some initialize operation.
+ *
+ * @param pluginConfig plugin config.
+ * @throws PrepareFailException if plugin prepare failed, the {@link PrepareFailException} will throw.
+ */
+ void prepare(Config pluginConfig) throws PrepareFailException;
}
diff --git a/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/serialization/DefaultSerializer.java
similarity index 57%
copy from seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/serialization/DefaultSerializer.java
index ee715c102..2100b9529 100644
--- a/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/serialization/DefaultSerializer.java
@@ -15,27 +15,26 @@
* limitations under the License.
*/
-package org.apache.seatunnel.common.config;
+package org.apache.seatunnel.api.serialization;
-import static org.junit.Assert.assertEquals;
+import org.apache.seatunnel.common.utils.SerializationUtils;
-import org.junit.Test;
+import java.io.IOException;
+import java.io.Serializable;
-import java.io.File;
+public class DefaultSerializer<T extends Serializable> implements Serializer<T> {
-public class CommonTest {
-
- static {
- Common.setDeployMode("client");
- }
-
- @Test
- public void appLibDir() {
- assertEquals(Common.appRootDir().toString() + File.separator + "lib", Common.appLibDir().toString());
+ @Override
+ public byte[] serialize(T obj) throws IOException {
+ if (obj != null) {
+ return SerializationUtils.serialize((Serializable) obj);
+ } else {
+ return null;
+ }
}
- @Test
- public void pluginTarFile() {
- assertEquals(Common.appRootDir().toString() + File.separator + "plugins.tar.gz", Common.pluginTarball().toString());
+ @Override
+ public T deserialize(byte[] serialized) throws IOException {
+ return SerializationUtils.deserialize(serialized);
}
}
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/ReflectionUtils.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/serialization/DeserializationSchema.java
similarity index 50%
copy from seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/ReflectionUtils.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/serialization/DeserializationSchema.java
index 597a8df10..0260d733a 100644
--- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/ReflectionUtils.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/serialization/DeserializationSchema.java
@@ -15,28 +15,30 @@
* limitations under the License.
*/
-package org.apache.seatunnel.common.utils;
+package org.apache.seatunnel.api.serialization;
-import java.lang.reflect.Method;
-import java.util.Optional;
+import org.apache.seatunnel.api.source.Collector;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
-public class ReflectionUtils {
+import java.io.IOException;
+import java.io.Serializable;
- public static Optional<Method> getDeclaredMethod(Class<?> clazz, String methodName, Class<?>... parameterTypes) {
+public interface DeserializationSchema<T> extends Serializable {
- Optional<Method> method = Optional.empty();
- Method m;
- for (; clazz != Object.class; clazz = clazz.getSuperclass()) {
- try {
- m = clazz.getDeclaredMethod(methodName, parameterTypes);
- m.setAccessible(true);
- return Optional.of(m);
- } catch (NoSuchMethodException e) {
- // do nothing
- }
- }
+ /**
+ * Deserializes the byte message.
+ *
+ * @param message The message, as a byte array.
+ * @return The deserialized message as an SeaTunnel Row (null if the message cannot be deserialized).
+ */
+ T deserialize(byte[] message) throws IOException;
- return method;
+ default void deserialize(byte[] message, Collector<T> out) throws IOException {
+ T deserialize = deserialize(message);
+ if (deserialize != null) {
+ out.collect(deserialize);
+ }
}
+ SeaTunnelDataType<T> getProducedType();
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/serialization/SerializationSchema.java
similarity index 69%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/serialization/SerializationSchema.java
index 5cd80fc53..53f4c7293 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/serialization/SerializationSchema.java
@@ -15,21 +15,16 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
+package org.apache.seatunnel.api.serialization;
-import org.apache.seatunnel.apis.base.command.CommandArgs;
-
-/**
- * Command interface.
- *
- * @param <T> args type
- */
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+public interface SerializationSchema {
/**
- * Execute command
+ * Serializes the incoming element to a specified type.
+ *
+ * @param element The incoming element to be serialized
+ * @return The serialized element.
*/
- void execute();
-
+ byte[] serialize(SeaTunnelRow element);
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/serialization/Serializer.java
similarity index 56%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/serialization/Serializer.java
index 5cd80fc53..617034864 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/serialization/Serializer.java
@@ -15,21 +15,27 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
+package org.apache.seatunnel.api.serialization;
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+import java.io.IOException;
-/**
- * Command interface.
- *
- * @param <T> args type
- */
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
+public interface Serializer<T> {
/**
- * Execute command
+ * Serializes the given object.
+ *
+ * @param obj The object to serialize.
+ * @return The serialized data (bytes).
+ * @throws IOException Thrown, if the serialization fails.
*/
- void execute();
+ byte[] serialize(T obj) throws IOException;
+ /**
+ * De-serializes the given data (bytes).
+ *
+ * @param serialized The serialized data
+ * @return The deserialized object
+ * @throws IOException Thrown, if the deserialization fails.
+ */
+ T deserialize(byte[] serialized) throws IOException;
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/DefaultSinkWriterContext.java
similarity index 69%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/DefaultSinkWriterContext.java
index f5f308154..c8a3fd61f 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/DefaultSinkWriterContext.java
@@ -15,18 +15,21 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.api.sink;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
-
- private final String type;
+/**
+ * The default {@link SinkWriter.Context} implement class.
+ */
+public class DefaultSinkWriterContext implements SinkWriter.Context {
+ private final int subtask;
- PluginType(String type) {
- this.type = type;
+ public DefaultSinkWriterContext(int subtask) {
+ this.subtask = subtask;
}
- public String getType() {
- return type;
+ @Override
+ public int getIndexOfSubtask() {
+ return subtask;
}
+
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SeaTunnelSink.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SeaTunnelSink.java
new file mode 100644
index 000000000..59517e409
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SeaTunnelSink.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.sink;
+
+import org.apache.seatunnel.api.common.PluginIdentifierInterface;
+import org.apache.seatunnel.api.common.SeaTunnelPluginLifeCycle;
+import org.apache.seatunnel.api.serialization.Serializer;
+import org.apache.seatunnel.api.source.SeaTunnelContextAware;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * The SeaTunnel sink interface, developer should implement this class when create a sink connector.
+ *
+ * @param <IN> The data class by sink accept. Only support
+ * {@link org.apache.seatunnel.api.table.type.SeaTunnelRow} at now.
+ * @param <StateT> The state should be saved when job execute, this class should implement interface
+ * {@link Serializable}.
+ * @param <CommitInfoT> The commit message class return by {@link SinkWriter#prepareCommit()}, then
+ * {@link SinkCommitter} or {@link SinkAggregatedCommitter} and handle it, this class should implement interface
+ * {@link Serializable}.
+ * @param <AggregatedCommitInfoT> The aggregated commit message class, combine by {@link CommitInfoT}.
+ * {@link SinkAggregatedCommitter} handle it, this class should implement interface {@link Serializable}.
+ */
+public interface SeaTunnelSink<IN, StateT, CommitInfoT, AggregatedCommitInfoT>
+ extends Serializable, PluginIdentifierInterface, SeaTunnelPluginLifeCycle, SeaTunnelContextAware {
+
+ /**
+ * Set the row type info of sink row data. This method will be automatically called by translation.
+ *
+ * @param seaTunnelRowType The row type info of sink.
+ */
+ void setTypeInfo(SeaTunnelRowType seaTunnelRowType);
+
+ /**
+ * Get the data type of the records consumed by this sink.
+ *
+ * @return SeaTunnel data type.
+ */
+ SeaTunnelDataType<IN> getConsumedType();
+
+ /**
+ * This method will be called to creat {@link SinkWriter}
+ *
+ * @param context The sink context
+ * @return Return sink writer instance
+ * @throws IOException throws IOException when createWriter failed.
+ */
+ SinkWriter<IN, CommitInfoT, StateT> createWriter(SinkWriter.Context context) throws IOException;
+
+ default SinkWriter<IN, CommitInfoT, StateT> restoreWriter(SinkWriter.Context context,
+ List<StateT> states) throws IOException {
+ return createWriter(context);
+ }
+
+ /**
+ * Get {@link StateT} serializer. So that {@link StateT} can be transferred across processes
+ *
+ * @return Serializer of {@link StateT}
+ */
+ default Optional<Serializer<StateT>> getWriterStateSerializer() {
+ return Optional.empty();
+ }
+
+ /**
+ * This method will be called to create {@link SinkCommitter}
+ *
+ * @return Return sink committer instance
+ * @throws IOException throws IOException when createCommitter failed.
+ */
+ default Optional<SinkCommitter<CommitInfoT>> createCommitter() throws IOException {
+ return Optional.empty();
+ }
+
+ /**
+ * Get {@link CommitInfoT} serializer. So that {@link CommitInfoT} can be transferred across processes
+ *
+ * @return Serializer of {@link CommitInfoT}
+ */
+ default Optional<Serializer<CommitInfoT>> getCommitInfoSerializer() {
+ return Optional.empty();
+ }
+
+ /**
+ * This method will be called to create {@link SinkAggregatedCommitter}
+ *
+ * @return Return sink aggregated committer instance
+ * @throws IOException throws IOException when createAggregatedCommitter failed.
+ */
+ default Optional<SinkAggregatedCommitter<CommitInfoT, AggregatedCommitInfoT>> createAggregatedCommitter() throws IOException {
+ return Optional.empty();
+ }
+
+ /**
+ * Get {@link AggregatedCommitInfoT} serializer. So that {@link AggregatedCommitInfoT} can be transferred across processes
+ *
+ * @return Serializer of {@link AggregatedCommitInfoT}
+ */
+ default Optional<Serializer<AggregatedCommitInfoT>> getAggregatedCommitInfoSerializer() {
+ return Optional.empty();
+ }
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkAggregatedCommitter.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkAggregatedCommitter.java
new file mode 100644
index 000000000..5d0068a91
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkAggregatedCommitter.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.sink;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * The committer combine taskManager/Worker Commit message. Then commit it uses
+ * {@link SinkAggregatedCommitter#commit(List)}. This class will execute in single thread.
+ * <p>
+ * See Also {@link SinkCommitter}
+ *
+ * @param <CommitInfoT> The type of commit message.
+ * @param <AggregatedCommitInfoT> The type of commit message after combine.
+ */
+public interface SinkAggregatedCommitter<CommitInfoT, AggregatedCommitInfoT> extends Serializable {
+
+ /**
+ * Commit message to third party data receiver, The method need to achieve idempotency.
+ *
+ * @param aggregatedCommitInfo The list of combine commit message.
+ * @return The commit message which need retry.
+ * @throws IOException throw IOException when commit failed.
+ */
+ List<AggregatedCommitInfoT> commit(List<AggregatedCommitInfoT> aggregatedCommitInfo) throws IOException;
+
+ /**
+ * The logic about how to combine commit message.
+ *
+ * @param commitInfos The list of commit message.
+ * @return The commit message after combine.
+ */
+ AggregatedCommitInfoT combine(List<CommitInfoT> commitInfos);
+
+ /**
+ * If {@link #commit(List)} failed, this method will be called (**Only** on Spark engine at now).
+ *
+ * @param aggregatedCommitInfo The list of combine commit message.
+ * @throws Exception throw Exception when abort failed.
+ */
+ void abort(List<AggregatedCommitInfoT> aggregatedCommitInfo) throws Exception;
+
+ /**
+ * Close this resource.
+ *
+ * @throws IOException throw IOException when close failed.
+ */
+ void close() throws IOException;
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkCommitter.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkCommitter.java
new file mode 100644
index 000000000..f5dad9590
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkCommitter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.sink;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * The committer to commit message. We strongly recommend implementing {@link SinkAggregatedCommitter} first,
+ * as the current version of {@link SinkAggregatedCommitter} can provide more consistent behavior.
+ * <p>
+ * See Also {@link SinkAggregatedCommitter}
+ *
+ * @param <CommitInfoT> The type of commit message.
+ */
+public interface SinkCommitter<CommitInfoT> extends Serializable {
+
+ /**
+ * Commit message to third party data receiver, The method need to achieve idempotency.
+ *
+ * @param committables The list of commit message
+ * @return The commit message need retry.
+ * @throws IOException throw IOException when commit failed.
+ */
+ List<CommitInfoT> commit(List<CommitInfoT> committables) throws IOException;
+
+ /**
+ * Abort the transaction, this method will be called (**Only** on Spark engine) when the commit is failed.
+ *
+ * @param commitInfos The list of commit message, used to abort the commit.
+ * @throws IOException throw IOException when close failed.
+ */
+ void abort(List<CommitInfoT> commitInfos) throws IOException;
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkWriter.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkWriter.java
new file mode 100644
index 000000000..897e64b4c
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkWriter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.sink;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * The sink writer use to write data to third party data receiver. This class will run on taskManger/Worker.
+ *
+ * @param <T> The data class by sink accept. Only support
+ * {@link org.apache.seatunnel.api.table.type.SeaTunnelRow} at now.
+ * @param <CommitInfoT> The type of commit message.
+ * @param <StateT> The type of state.
+ */
+public interface SinkWriter<T, CommitInfoT, StateT> extends Serializable {
+
+ /**
+ * write data to third party data receiver.
+ *
+ * @param element the date need be written.
+ * @throws IOException throw IOException when write data failed.
+ */
+ void write(T element) throws IOException;
+
+ /**
+ * prepare the commit, will be called before {@link #snapshotState(long checkpointId)}.
+ * If you need to use 2pc, you can return the commit info in this method, and receive the commit info in {@link SinkCommitter#commit(List)}.
+ * If this method failed (by throw exception), **Only** Spark engine will call {@link #abortPrepare()}
+ *
+ * @return the commit info need to commit
+ */
+ Optional<CommitInfoT> prepareCommit() throws IOException;
+
+ /**
+ * @return The writer's state.
+ * @throws IOException if fail to snapshot writer's state.
+ */
+ default List<StateT> snapshotState(long checkpointId) throws IOException {
+ return Collections.emptyList();
+ }
+
+ /**
+ * Used to abort the {@link #prepareCommit()}, if the prepareCommit failed,
+ * there is no CommitInfoT, so the rollback work cannot be done by {@link SinkCommitter}. But we can
+ * use this method to rollback side effects of {@link #prepareCommit()}. Only use it in Spark engine at
+ * now.
+ */
+ void abortPrepare();
+
+ /**
+ * call it when SinkWriter close
+ *
+ * @throws IOException if close failed
+ */
+ void close() throws IOException;
+
+ interface Context extends Serializable{
+
+ /**
+ * @return The index of this subtask.
+ */
+ int getIndexOfSubtask();
+
+ }
+}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/Boundedness.java
similarity index 67%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/source/Boundedness.java
index 5cd80fc53..c7b3fcd95 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/Boundedness.java
@@ -15,21 +15,19 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
-
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+package org.apache.seatunnel.api.source;
/**
- * Command interface.
- *
- * @param <T> args type
+ * Used to define the boundedness of a source. In batch mode, the source is {@link Boundedness#BOUNDED}.
+ * In streaming mode, the source is {@link Boundedness#UNBOUNDED}.
*/
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
-
+public enum Boundedness {
/**
- * Execute command
+ * A BOUNDED stream is a stream with finite records.
*/
- void execute();
-
+ BOUNDED,
+ /**
+ * A UNBOUNDED stream is a stream with infinite records.
+ */
+ UNBOUNDED
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/Collector.java
similarity index 72%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/source/Collector.java
index 5cd80fc53..0b924bb57 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/Collector.java
@@ -15,21 +15,21 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
-
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+package org.apache.seatunnel.api.source;
/**
- * Command interface.
+ * A {@link Collector} is used to collect data from {@link SourceReader}.
*
- * @param <T> args type
+ * @param <T> data type.
*/
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
+public interface Collector<T> {
+
+ void collect(T record);
/**
- * Execute command
+ * Returns the checkpoint lock.
+ *
+ * @return The object to use as the lock
*/
- void execute();
-
+ Object getCheckpointLock();
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SeaTunnelContextAware.java
similarity index 71%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SeaTunnelContextAware.java
index 5cd80fc53..429f05155 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SeaTunnelContextAware.java
@@ -15,21 +15,16 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
+package org.apache.seatunnel.api.source;
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+import org.apache.seatunnel.api.common.SeaTunnelContext;
/**
- * Command interface.
- *
- * @param <T> args type
+ * This interface defines the runtime environment of the SeaTunnel application.
*/
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
-
- /**
- * Execute command
- */
- void execute();
+public interface SeaTunnelContextAware {
+ default void setSeaTunnelContext(SeaTunnelContext seaTunnelContext){
+ // nothing
+ }
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SeaTunnelSource.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SeaTunnelSource.java
new file mode 100644
index 000000000..f93f4d3bf
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SeaTunnelSource.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.source;
+
+import org.apache.seatunnel.api.common.PluginIdentifierInterface;
+import org.apache.seatunnel.api.common.SeaTunnelPluginLifeCycle;
+import org.apache.seatunnel.api.serialization.DefaultSerializer;
+import org.apache.seatunnel.api.serialization.Serializer;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+
+import java.io.Serializable;
+
+/**
+ * The interface for Source. It acts like a factory class that helps construct the {@link
+ * SourceSplitEnumerator} and {@link SourceReader} and corresponding serializers.
+ *
+ * @param <T> The type of records produced by the source.
+ * @param <SplitT> The type of splits handled by the source.
+ * @param <StateT> The type of checkpoint states.
+ */
+public interface SeaTunnelSource<T, SplitT extends SourceSplit, StateT extends Serializable>
+ extends Serializable, PluginIdentifierInterface, SeaTunnelPluginLifeCycle, SeaTunnelContextAware {
+
+ /**
+ * Get the boundedness of this source.
+ *
+ * @return the boundedness of this source.
+ */
+ Boundedness getBoundedness();
+
+ /**
+ * Get the data type of the records produced by this source.
+ *
+ * @return SeaTunnel data type.
+ */
+ SeaTunnelDataType<T> getProducedType();
+
+ /**
+ * Create source reader, used to produce data.
+ *
+ * @param readerContext reader context.
+ * @return source reader.
+ * @throws Exception when create reader failed.
+ */
+ SourceReader<T, SplitT> createReader(SourceReader.Context readerContext) throws Exception;
+
+ /**
+ * Create split serializer, use to serialize/deserialize split generated by {@link SourceSplitEnumerator}.
+ *
+ * @return split serializer.
+ */
+ default Serializer<SplitT> getSplitSerializer() {
+ return new DefaultSerializer<>();
+ }
+
+ /**
+ * Create source split enumerator, used to generate splits. This method will be called only once when start a source.
+ *
+ * @param enumeratorContext enumerator context.
+ * @return source split enumerator.
+ * @throws Exception when create enumerator failed.
+ */
+ SourceSplitEnumerator<SplitT, StateT> createEnumerator(SourceSplitEnumerator.Context<SplitT> enumeratorContext)
+ throws Exception;
+
+ /**
+ * Create source split enumerator, used to generate splits. This method will be called when restore from checkpoint.
+ *
+ * @param enumeratorContext enumerator context.
+ * @param checkpointState checkpoint state.
+ * @return source split enumerator.
+ * @throws Exception when create enumerator failed.
+ */
+ SourceSplitEnumerator<SplitT, StateT> restoreEnumerator(SourceSplitEnumerator.Context<SplitT> enumeratorContext,
+ StateT checkpointState) throws Exception;
+
+ /**
+ * Create enumerator state serializer, used to serialize/deserialize checkpoint state.
+ *
+ * @return enumerator state serializer.
+ */
+ default Serializer<StateT> getEnumeratorStateSerializer(){
+ return new DefaultSerializer<>();
+ }
+
+}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceEvent.java
similarity index 73%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceEvent.java
index f5f308154..2f4558fbc 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceEvent.java
@@ -15,18 +15,12 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.api.source;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import java.io.Serializable;
- private final String type;
-
- PluginType(String type) {
- this.type = type;
- }
-
- public String getType() {
- return type;
- }
+/**
+ * A base class for the events passed between the {@link SourceReader} and {@link SourceSplitEnumerator}.
+ */
+public interface SourceEvent extends Serializable {
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceReader.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceReader.java
new file mode 100644
index 000000000..72b3e7eb5
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceReader.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.source;
+
+import org.apache.seatunnel.api.state.CheckpointListener;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * The {@link SourceReader} is used to generate source record, and it will be running at worker.
+ *
+ * @param <T> record type.
+ * @param <SplitT> source split type.
+ */
+public interface SourceReader<T, SplitT extends SourceSplit> extends AutoCloseable, CheckpointListener {
+
+ /**
+ * Open the source reader.
+ */
+ void open() throws Exception;
+
+ /**
+ * Called to close the reader, in case it holds on to any resources, like threads or network
+ * connections.
+ */
+ @Override
+ void close() throws IOException;
+
+ /**
+ * Generate the next batch of records.
+ *
+ * @param output output collector.
+ * @throws Exception if error occurs.
+ */
+ void pollNext(Collector<T> output) throws Exception;
+
+ /**
+ * Get the current split checkpoint state by checkpointId.
+ *
+ * If the source is bounded, checkpoint is not triggered.
+ *
+ * @param checkpointId checkpoint Id.
+ * @return split checkpoint state.
+ * @throws Exception if error occurs.
+ */
+ List<SplitT> snapshotState(long checkpointId) throws Exception;
+
+ /**
+ * Add the split checkpoint state to reader.
+ *
+ * @param splits split checkpoint state.
+ */
+ void addSplits(List<SplitT> splits);
+
+ /**
+ * This method is called when the reader is notified that it will not receive any further
+ * splits.
+ *
+ * <p>It is triggered when the enumerator calls {@link
+ * SourceSplitEnumerator.Context#signalNoMoreSplits(int)} with the reader's parallel subtask.
+ */
+ void handleNoMoreSplits();
+
+ /**
+ * Handle the source event form {@link SourceSplitEnumerator}.
+ *
+ * @param sourceEvent source event.
+ */
+ default void handleSourceEvent(SourceEvent sourceEvent) {
+ }
+
+ interface Context {
+
+ /**
+ * @return The index of this subtask.
+ */
+ int getIndexOfSubtask();
+
+ /**
+ * @return boundedness of this reader.
+ */
+ Boundedness getBoundedness();
+
+ /**
+ * Indicator that the input has reached the end of data. Then will cancel this reader.
+ */
+ void signalNoMoreElement();
+
+ /**
+ * Sends a split request to the source's {@link SourceSplitEnumerator}. This will result in a call to
+ * the {@link SourceSplitEnumerator#handleSplitRequest(int)} method, with this reader's
+ * parallel subtask id and the hostname where this reader runs.
+ */
+ void sendSplitRequest();
+
+ /**
+ * Send a source event to the source coordinator.
+ *
+ * @param sourceEvent the source event to coordinator.
+ */
+ void sendSourceEventToEnumerator(SourceEvent sourceEvent);
+ }
+}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceSplit.java
similarity index 74%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceSplit.java
index 5cd80fc53..a4ccfdbfd 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceSplit.java
@@ -15,21 +15,19 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
+package org.apache.seatunnel.api.source;
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+import java.io.Serializable;
/**
- * Command interface.
- *
- * @param <T> args type
+ * An interface for all the Split types to extend.
*/
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
+public interface SourceSplit extends Serializable {
/**
- * Execute command
+ * Get the split id of this source split.
+ *
+ * @return id of this source split.
*/
- void execute();
-
+ String splitId();
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceSplitEnumerator.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceSplitEnumerator.java
new file mode 100644
index 000000000..3b1917bd0
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SourceSplitEnumerator.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.source;
+
+import org.apache.seatunnel.api.state.CheckpointListener;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * The {@link SourceSplitEnumerator} is responsible for enumerating the splits of a source. It will run at master.
+ *
+ * @param <SplitT> source split type
+ * @param <StateT>source split state type
+ */
+public interface SourceSplitEnumerator<SplitT extends SourceSplit, StateT> extends AutoCloseable, CheckpointListener {
+
+ void open();
+
+ /**
+ * The method is executed by the engine only once.
+ */
+ void run() throws Exception;
+
+ /**
+ * Called to close the enumerator, in case it holds on to any resources, like threads or network
+ * connections.
+ */
+ @Override
+ void close() throws IOException;
+
+ /**
+ * Add a split back to the split enumerator. It will only happen when a {@link SourceReader}
+ * fails and there are splits assigned to it after the last successful checkpoint.
+ *
+ * @param splits The split to add back to the enumerator for reassignment.
+ * @param subtaskId The id of the subtask to which the returned splits belong.
+ */
+ void addSplitsBack(List<SplitT> splits, int subtaskId);
+
+ int currentUnassignedSplitSize();
+
+ void handleSplitRequest(int subtaskId);
+
+ void registerReader(int subtaskId);
+
+ /**
+ * If the source is bounded, checkpoint is not triggered.
+ */
+ StateT snapshotState(long checkpointId) throws Exception;
+
+ /**
+ * Handle the source event from {@link SourceReader}.
+ *
+ * @param subtaskId The id of the subtask to which the source event from.
+ * @param sourceEvent source event.
+ */
+ default void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) {
+ }
+
+ interface Context<SplitT extends SourceSplit> {
+
+ int currentParallelism();
+
+ /**
+ * Get the currently registered readers. The mapping is from subtask id to the reader info.
+ *
+ * @return the currently registered readers.
+ */
+ Set<Integer> registeredReaders();
+
+ /**
+ * Assign the splits.
+ */
+ void assignSplit(int subtaskId, List<SplitT> splits);
+
+ /**
+ * Assigns a single split.
+ *
+ * <p>When assigning multiple splits, it is more efficient to assign all of them in a single
+ * call to the {@link #assignSplit} method.
+ *
+ * @param split The new split
+ * @param subtaskId The index of the operator's parallel subtask that shall receive the split.
+ */
+ default void assignSplit(int subtaskId, SplitT split) {
+ assignSplit(subtaskId, Collections.singletonList(split));
+ }
+
+ /**
+ * Signals a subtask that it will not receive any further split.
+ *
+ * @param subtask The index of the operator's parallel subtask that shall be signaled it will
+ * not receive any further split.
+ */
+ void signalNoMoreSplits(int subtask);
+
+ /**
+ * Send a source event to a source reader. The source reader is identified by its subtask id.
+ *
+ * @param subtaskId the subtask id of the source reader to send this event to.
+ * @param event the source event to send.
+ */
+ void sendEventToSourceReader(int subtaskId, SourceEvent event);
+ }
+
+}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SupportCoordinate.java
similarity index 73%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SupportCoordinate.java
index f5f308154..2de9a3af4 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/source/SupportCoordinate.java
@@ -15,18 +15,10 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.api.source;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
-
- private final String type;
-
- PluginType(String type) {
- this.type = type;
- }
-
- public String getType() {
- return type;
- }
+/**
+ * Used to mark whether the interface supports coordination.
+ */
+public interface SupportCoordinate {
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/state/CheckpointListener.java
similarity index 73%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/state/CheckpointListener.java
index 5cd80fc53..7f3a5dac6 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/state/CheckpointListener.java
@@ -15,21 +15,15 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
-
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+package org.apache.seatunnel.api.state;
/**
- * Command interface.
- *
- * @param <T> args type
+ * If the data flow is bounded, checkpoint is not triggered.
*/
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
+public interface CheckpointListener {
- /**
- * Execute command
- */
- void execute();
+ void notifyCheckpointComplete(long checkpointId) throws Exception;
+ default void notifyCheckpointAborted(long checkpointId) throws Exception {
+ }
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/Catalog.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/Catalog.java
new file mode 100644
index 000000000..e3f83d9ff
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/Catalog.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.catalog;
+
+import org.apache.seatunnel.api.table.catalog.exception.CatalogException;
+import org.apache.seatunnel.api.table.catalog.exception.DatabaseNotExistException;
+import org.apache.seatunnel.api.table.catalog.exception.TableNotExistException;
+import org.apache.seatunnel.api.table.factory.Factory;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Interface for reading and writing table metadata from SeaTunnel. Each connector need to contain
+ * the implementation of Catalog.
+ */
+public interface Catalog {
+
+ default Optional<Factory> getFactory() {
+ return Optional.empty();
+ }
+
+ /**
+ * Open the catalog. Used for any required preparation in initialization phase.
+ *
+ * @throws CatalogException in case of any runtime exception
+ */
+ void open() throws CatalogException;
+
+ /**
+ * Close the catalog when it is no longer needed and release any resource that it might be
+ * holding.
+ *
+ * @throws CatalogException in case of any runtime exception
+ */
+ void close() throws CatalogException;
+
+ // --------------------------------------------------------------------------------------------
+ // database
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Get the name of the default database for this catalog. The default database will be the
+ * current database for the catalog when user's session doesn't specify a current database. The
+ * value probably comes from configuration, will not change for the life time of the catalog
+ * instance.
+ *
+ * @return the name of the current database
+ * @throws CatalogException in case of any runtime exception
+ */
+ String getDefaultDatabase() throws CatalogException;
+
+ /**
+ * Check if a database exists in this catalog.
+ *
+ * @param databaseName Name of the database
+ * @return true if the given database exists in the catalog false otherwise
+ * @throws CatalogException in case of any runtime exception
+ */
+ boolean databaseExists(String databaseName) throws CatalogException;
+
+ /**
+ * Get the names of all databases in this catalog.
+ *
+ * @return a list of the names of all databases
+ * @throws CatalogException in case of any runtime exception
+ */
+ List<String> listDatabases() throws CatalogException;
+
+ // --------------------------------------------------------------------------------------------
+ // table
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Get names of all tables under this database. An empty list is returned if none
+ * exists.
+ *
+ * @return a list of the names of all tables in this database
+ * @throws CatalogException in case of any runtime exception
+ */
+ List<String> listTables(String databaseName) throws CatalogException, DatabaseNotExistException;
+
+ /**
+ * Check if a table exist in this catalog.
+ *
+ * @param tablePath Path of the table
+ * @return true if the given table exists in the catalog false otherwise
+ * @throws CatalogException in case of any runtime exception
+ */
+ boolean tableExists(TablePath tablePath) throws CatalogException;
+
+ /**
+ * Return a {@link CatalogTable} identified by the given {@link
+ * TablePath}. The framework will resolve the metadata objects when necessary.
+ *
+ * @param tablePath Path of the table
+ * @return The requested table
+ * @throws CatalogException in case of any runtime exception
+ */
+ CatalogTable getTable(TablePath tablePath) throws CatalogException, TableNotExistException;
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogTable.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogTable.java
new file mode 100644
index 000000000..1c8542300
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogTable.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.catalog;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represent the table metadata in SeaTunnel.
+ */
+public final class CatalogTable implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Used to identify the table.
+ */
+ private final TableIdentifier tableId;
+
+ /**
+ * The table schema metadata.
+ */
+ private final TableSchema tableSchema;
+
+ private final Map<String, String> options;
+
+ private final List<String> partitionKeys;
+
+ private final String comment;
+
+ public static CatalogTable of(
+ TableIdentifier tableId,
+ TableSchema tableSchema,
+ Map<String, String> options,
+ List<String> partitionKeys,
+ String comment) {
+ return new CatalogTable(
+ tableId,
+ tableSchema,
+ options,
+ partitionKeys,
+ comment);
+ }
+
+ private CatalogTable(
+ TableIdentifier tableId,
+ TableSchema tableSchema,
+ Map<String, String> options,
+ List<String> partitionKeys,
+ String comment) {
+ this.tableId = tableId;
+ this.tableSchema = tableSchema;
+ this.options = options;
+ this.partitionKeys = partitionKeys;
+ this.comment = comment;
+ }
+
+ public TableIdentifier getTableId() {
+ return tableId;
+ }
+
+ public TableSchema getTableSchema() {
+ return tableSchema;
+ }
+
+ public Map<String, String> getOptions() {
+ return options;
+ }
+
+ public List<String> getPartitionKeys() {
+ return partitionKeys;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/Column.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/Column.java
new file mode 100644
index 000000000..aea7fade2
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/Column.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.catalog;
+
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.Optional;
+
+@SuppressWarnings("PMD.AbstractClassShouldStartWithAbstractNamingRule")
+public abstract class Column implements Serializable {
+
+ private static final long serialVersionUID = -1L;
+
+ /**
+ * column name.
+ */
+ protected final String name;
+
+ /**
+ * Data type of the column.
+ */
+ protected final SeaTunnelDataType<?> dataType;
+
+ protected final String comment;
+
+ private Column(String name, SeaTunnelDataType<?> dataType, String comment) {
+ this.name = name;
+ this.dataType = dataType;
+ this.comment = comment;
+ }
+
+ /**
+ * Creates a regular table column that represents physical data.
+ */
+ public static PhysicalColumn physical(String name, SeaTunnelDataType<?> dataType) {
+ return new PhysicalColumn(name, dataType);
+ }
+
+ /**
+ * Creates a metadata column from metadata of the given column name or from metadata of the
+ * given key (if not null).
+ *
+ * <p>Allows to specify whether the column is virtual or not.
+ */
+ public static MetadataColumn metadata(
+ String name, SeaTunnelDataType<?> dataType, String metadataKey) {
+ return new MetadataColumn(name, dataType, metadataKey);
+ }
+
+ /**
+ * Add the comment to the column and return the new object.
+ */
+ public abstract Column withComment(String comment);
+
+ /**
+ * Returns whether the given column is a physical column of a table; neither computed nor
+ * metadata.
+ */
+ public abstract boolean isPhysical();
+
+ /**
+ * Returns the data type of this column.
+ */
+ public SeaTunnelDataType<?> getDataType() {
+ return this.dataType;
+ }
+
+ /**
+ * Returns the name of this column.
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Returns the comment of this column.
+ */
+ public Optional<String> getComment() {
+ return Optional.ofNullable(comment);
+ }
+
+ /**
+ * Returns a copy of the column with a replaced {@link SeaTunnelDataType}.
+ */
+ public abstract Column copy(SeaTunnelDataType<?> newType);
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Column that = (Column) o;
+ return Objects.equals(this.name, that.name)
+ && Objects.equals(this.dataType, that.dataType)
+ && Objects.equals(this.comment, that.comment);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(this.name, this.dataType);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Specific kinds of columns
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Representation of a physical column.
+ */
+ public static final class PhysicalColumn extends Column {
+
+ private PhysicalColumn(String name, SeaTunnelDataType<?> dataType) {
+ this(name, dataType, null);
+ }
+
+ private PhysicalColumn(String name, SeaTunnelDataType<?> dataType, String comment) {
+ super(name, dataType, comment);
+ }
+
+ @Override
+ public PhysicalColumn withComment(String comment) {
+ if (comment == null) {
+ return this;
+ }
+ return new PhysicalColumn(name, dataType, comment);
+ }
+
+ @Override
+ public boolean isPhysical() {
+ return true;
+ }
+
+ @Override
+ public Column copy(SeaTunnelDataType<?> newDataType) {
+ return new PhysicalColumn(name, newDataType, comment);
+ }
+ }
+
+ /**
+ * Representation of a metadata column.
+ */
+ public static final class MetadataColumn extends Column {
+
+ private final String metadataKey;
+
+ private MetadataColumn(
+ String name, SeaTunnelDataType<?> dataType, String metadataKey) {
+ this(name, dataType, metadataKey, null);
+ }
+
+ private MetadataColumn(
+ String name,
+ SeaTunnelDataType<?> dataType,
+ String metadataKey,
+ String comment) {
+ super(name, dataType, comment);
+ this.metadataKey = metadataKey;
+ }
+
+ public Optional<String> getMetadataKey() {
+ return Optional.ofNullable(metadataKey);
+ }
+
+ @Override
+ public MetadataColumn withComment(String comment) {
+ if (comment == null) {
+ return this;
+ }
+ return new MetadataColumn(name, dataType, metadataKey, comment);
+ }
+
+ @Override
+ public boolean isPhysical() {
+ return false;
+ }
+
+ @Override
+ public Column copy(SeaTunnelDataType<?> newDataType) {
+ return new MetadataColumn(name, newDataType, metadataKey, comment);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ MetadataColumn that = (MetadataColumn) o;
+ return Objects.equals(metadataKey, that.metadataKey);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), metadataKey);
+ }
+ }
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/TableIdentifier.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/TableIdentifier.java
new file mode 100644
index 000000000..e3c60ada1
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/TableIdentifier.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.catalog;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public final class TableIdentifier implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final String catalogName;
+
+ private final String databaseName;
+
+ private final String tableName;
+
+ private TableIdentifier(String catalogName, String databaseName, String tableName) {
+ this.catalogName = catalogName;
+ this.databaseName = databaseName;
+ this.tableName = tableName;
+ }
+
+ public static TableIdentifier of(String catalogName, String databaseName, String tableName) {
+ return new TableIdentifier(catalogName, databaseName, tableName);
+ }
+
+ public String getCatalogName() {
+ return catalogName;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public String gettableName() {
+ return tableName;
+ }
+
+ public TablePath toTablePath() {
+ return TablePath.of(databaseName, tableName);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ TableIdentifier that = (TableIdentifier) o;
+ return catalogName.equals(that.catalogName)
+ && databaseName.equals(that.databaseName)
+ && tableName.equals(that.tableName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(catalogName, databaseName, tableName);
+ }
+
+ @Override
+ public String toString() {
+ return String.join(".", catalogName, databaseName, tableName);
+ }
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/TablePath.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/TablePath.java
new file mode 100644
index 000000000..61f61d936
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/TablePath.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.catalog;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public final class TablePath implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private final String databaseName;
+ private final String tableName;
+
+ private TablePath(String databaseName, String tableName) {
+ this.databaseName = databaseName;
+ this.tableName = tableName;
+ }
+
+ public static TablePath of(String fullName) {
+ String[] paths = fullName.split("\\.");
+
+ if (paths.length != 2) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot get split '%s' to get databaseName and tableName", fullName));
+ }
+
+ return new TablePath(paths[0], paths[1]);
+ }
+
+ public static TablePath of(String databaseName, String tableName) {
+ return new TablePath(databaseName, tableName);
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String getFullName() {
+ return String.format("%s.%s", databaseName, tableName);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ TablePath that = (TablePath) o;
+
+ return Objects.equals(databaseName, that.databaseName)
+ && Objects.equals(tableName, that.tableName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(databaseName, tableName);
+ }
+
+ @Override
+ public String toString() {
+ return getFullName();
+ }
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/TableSchema.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/TableSchema.java
new file mode 100644
index 000000000..61d99ef52
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/TableSchema.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.catalog;
+
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Represent a physical table schema.
+ */
+public final class TableSchema implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private final List<Column> columns;
+
+ private final PrimaryKey primaryKey;
+
+ private TableSchema(List<Column> columns, PrimaryKey primaryKey) {
+ this.columns = columns;
+ this.primaryKey = primaryKey;
+ }
+
+ public static TableSchema.Builder builder() {
+ return new Builder();
+ }
+
+ /**
+ * Returns all {@link Column}s of this schema.
+ */
+ public List<Column> getColumns() {
+ return columns;
+ }
+
+ public static final class Builder {
+ private final List<Column> columns = new ArrayList<>();
+
+ private PrimaryKey primaryKey;
+
+ public Builder columns(List<Column> columns) {
+ this.columns.addAll(columns);
+ return this;
+ }
+
+ public Builder column(Column column) {
+ this.columns.add(column);
+ return this;
+ }
+
+ public Builder physicalColumn(String name, SeaTunnelDataType<?> dataType) {
+ this.columns.add(Column.physical(name, dataType));
+ return this;
+ }
+
+ public Builder primaryKey(PrimaryKey primaryKey) {
+ this.primaryKey = primaryKey;
+ return this;
+ }
+
+ public Builder primaryKey(String constraintName, List<String> columnNames) {
+ this.primaryKey = PrimaryKey.of(constraintName, columnNames);
+ return this;
+ }
+
+ public TableSchema build() {
+ return new TableSchema(columns, primaryKey);
+ }
+ }
+
+ public static final class PrimaryKey implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final String constraintName;
+ private final List<String> columnNames;
+
+ private PrimaryKey(String constraintName, List<String> columnNames) {
+ this.constraintName = constraintName;
+ this.columnNames = columnNames;
+ }
+
+ public static PrimaryKey of(String constraintName, List<String> columnNames) {
+ return new PrimaryKey(constraintName, columnNames);
+ }
+
+ public String getConstraintName() {
+ return constraintName;
+ }
+
+ public List<String> getColumnNames() {
+ return columnNames;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("CONSTRAINT %s PRIMARY KEY (%s) NOT ENFORCED", constraintName, String.join(", ", columnNames));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PrimaryKey that = (PrimaryKey) o;
+ return constraintName.equals(that.constraintName) && columnNames.equals(that.columnNames);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(constraintName, columnNames);
+ }
+ }
+}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/exception/CatalogException.java
similarity index 58%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/exception/CatalogException.java
index 5cd80fc53..d91e6670e 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/exception/CatalogException.java
@@ -15,21 +15,32 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
-
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+package org.apache.seatunnel.api.table.catalog.exception;
/**
- * Command interface.
- *
- * @param <T> args type
+ * A catalog-related, runtime exception.
*/
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
+public class CatalogException extends RuntimeException {
/**
- * Execute command
+ * @param message the detail message.
*/
- void execute();
+ public CatalogException(String message) {
+ super(message);
+ }
+ /**
+ * @param cause the cause.
+ */
+ public CatalogException(Throwable cause) {
+ super(cause);
+ }
+
+ /**
+ * @param message the detail message.
+ * @param cause the cause.
+ */
+ public CatalogException(String message, Throwable cause) {
+ super(message, cause);
+ }
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/exception/DatabaseNotExistException.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/exception/DatabaseNotExistException.java
new file mode 100644
index 000000000..5e81505f7
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/exception/DatabaseNotExistException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.catalog.exception;
+
+/** Exception for trying to operate on a database that doesn't exist. */
+public class DatabaseNotExistException extends Exception {
+ private static final String MSG = "Database %s does not exist in Catalog %s.";
+
+ public DatabaseNotExistException(String catalogName, String databaseName, Throwable cause) {
+ super(String.format(MSG, databaseName, catalogName), cause);
+ }
+
+ public DatabaseNotExistException(String catalogName, String databaseName) {
+ this(catalogName, databaseName, null);
+ }
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/exception/TableNotExistException.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/exception/TableNotExistException.java
new file mode 100644
index 000000000..d37a586eb
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/exception/TableNotExistException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.catalog.exception;
+
+import org.apache.seatunnel.api.table.catalog.TablePath;
+
+/** Exception for trying to operate on a table that doesn't exist. */
+public class TableNotExistException extends Exception {
+
+ private static final String MSG = "Table %s does not exist in Catalog %s.";
+
+ public TableNotExistException(String catalogName, TablePath tablePath) {
+ this(catalogName, tablePath, null);
+ }
+
+ public TableNotExistException(String catalogName, TablePath tablePath, Throwable cause) {
+ super(String.format(MSG, tablePath.getFullName(), catalogName), cause);
+ }
+}
diff --git a/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/DeserializationFormat.java
similarity index 54%
copy from seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/DeserializationFormat.java
index ee715c102..6a3cb0ad5 100644
--- a/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/DeserializationFormat.java
@@ -15,27 +15,25 @@
* limitations under the License.
*/
-package org.apache.seatunnel.common.config;
+package org.apache.seatunnel.api.table.connector;
-import static org.junit.Assert.assertEquals;
+import org.apache.seatunnel.api.serialization.DeserializationSchema;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
-import org.junit.Test;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
-import java.io.File;
+public interface DeserializationFormat {
-public class CommonTest {
+ DeserializationSchema createDeserializationSchema();
- static {
- Common.setDeployMode("client");
+ default Map<String, SeaTunnelDataType<?>> listReadableMetadata() {
+ return Collections.emptyMap();
}
- @Test
- public void appLibDir() {
- assertEquals(Common.appRootDir().toString() + File.separator + "lib", Common.appLibDir().toString());
- }
-
- @Test
- public void pluginTarFile() {
- assertEquals(Common.appRootDir().toString() + File.separator + "plugins.tar.gz", Common.pluginTarball().toString());
+ default void applyReadableMetadata(List<String> metadataKeys, SeaTunnelDataType<?> dataType) {
+ throw new UnsupportedOperationException(
+ "A decoding format must override this method to apply metadata keys.");
}
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/SerializationFormat.java
similarity index 74%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/SerializationFormat.java
index f5f308154..bc0c5c052 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/SerializationFormat.java
@@ -15,18 +15,11 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.api.table.connector;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import org.apache.seatunnel.api.serialization.SerializationSchema;
- private final String type;
+public interface SerializationFormat {
- PluginType(String type) {
- this.type = type;
- }
-
- public String getType() {
- return type;
- }
+ SerializationSchema createSerializationSchema();
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/SupportReadingMetadata.java
similarity index 60%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/SupportReadingMetadata.java
index 5cd80fc53..fa765cc09 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/SupportReadingMetadata.java
@@ -15,21 +15,20 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
+package org.apache.seatunnel.api.table.connector;
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+
+import java.util.List;
+import java.util.Map;
/**
- * Command interface.
- *
- * @param <T> args type
+ * Used for {@link TableSource} to support metadata columns.
*/
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
+public interface SupportReadingMetadata {
- /**
- * Execute command
- */
- void execute();
+ Map<String, SeaTunnelDataType<?>> listReadableMetadata(CatalogTable catalogTable);
+ void applyReadableMetadata(CatalogTable catalogTable, List<String> metadataKeys, SeaTunnelDataType<?> dataType);
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/TableSink.java
similarity index 74%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/TableSink.java
index f5f308154..8461e7dc1 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/TableSink.java
@@ -15,18 +15,11 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.api.table.connector;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import org.apache.seatunnel.api.sink.SeaTunnelSink;
- private final String type;
+public interface TableSink<IN, StateT, CommitInfoT, AggregatedCommitInfoT> {
- PluginType(String type) {
- this.type = type;
- }
-
- public String getType() {
- return type;
- }
+ SeaTunnelSink<IN, StateT, CommitInfoT, AggregatedCommitInfoT> createSink();
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/TableSource.java
similarity index 70%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/TableSource.java
index f5f308154..d9b6294c4 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/connector/TableSource.java
@@ -15,18 +15,14 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.api.table.connector;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import org.apache.seatunnel.api.source.SeaTunnelSource;
+import org.apache.seatunnel.api.source.SourceSplit;
- private final String type;
+import java.io.Serializable;
- PluginType(String type) {
- this.type = type;
- }
+public interface TableSource<T, SplitT extends SourceSplit, StateT extends Serializable> {
- public String getType() {
- return type;
- }
+ SeaTunnelSource<T, SplitT, StateT> createSource();
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/CatalogFactory.java
similarity index 73%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/CatalogFactory.java
index 5cd80fc53..2448c77e2 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/CatalogFactory.java
@@ -15,21 +15,16 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
+package org.apache.seatunnel.api.table.factory;
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+import org.apache.seatunnel.api.table.catalog.Catalog;
-/**
- * Command interface.
- *
- * @param <T> args type
- */
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
+import java.util.Map;
+
+public interface CatalogFactory extends Factory {
/**
- * Execute command
+ * Creates a {@link Catalog} using the options.
*/
- void execute();
-
+ Catalog createCatalog(String catalogName, Map<String, String> options);
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/DeserializationFormatFactory.java
similarity index 73%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/DeserializationFormatFactory.java
index f5f308154..29285f641 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/DeserializationFormatFactory.java
@@ -15,18 +15,10 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.api.table.factory;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import org.apache.seatunnel.api.table.connector.DeserializationFormat;
- private final String type;
-
- PluginType(String type) {
- this.type = type;
- }
-
- public String getType() {
- return type;
- }
+public interface DeserializationFormatFactory extends Factory {
+ DeserializationFormat createDeserializationFormat(TableFactoryContext context);
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/Factory.java
similarity index 63%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/Factory.java
index 5cd80fc53..098d3d291 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/Factory.java
@@ -15,21 +15,20 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
-
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+package org.apache.seatunnel.api.table.factory;
/**
- * Command interface.
- *
- * @param <T> args type
+ * todo: use PluginIdentifier.
+ * This is the SPI interface.
*/
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
+public interface Factory {
/**
- * Execute command
+ * Returns a unique identifier among same factory interfaces.
+ *
+ * <p>For consistency, an identifier should be declared as one lower case word (e.g. {@code
+ * kafka}). If multiple factories exist for different versions, a version should be appended
+ * using "-" (e.g. {@code elasticsearch-7}).
*/
- void execute();
-
+ String factoryIdentifier();
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/FactoryException.java
similarity index 75%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/FactoryException.java
index f5f308154..64e5214c7 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/FactoryException.java
@@ -15,18 +15,15 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.api.table.factory;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+public class FactoryException extends RuntimeException {
- private final String type;
-
- PluginType(String type) {
- this.type = type;
+ public FactoryException(String message, Throwable cause) {
+ super(message, cause);
}
- public String getType() {
- return type;
+ public FactoryException(String message) {
+ super(message);
}
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/FactoryUtil.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/FactoryUtil.java
new file mode 100644
index 000000000..288e35beb
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/FactoryUtil.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.factory;
+
+import org.apache.seatunnel.api.sink.SeaTunnelSink;
+import org.apache.seatunnel.api.source.SeaTunnelSource;
+import org.apache.seatunnel.api.source.SourceSplit;
+import org.apache.seatunnel.api.table.catalog.Catalog;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.connector.TableSource;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.ServiceConfigurationError;
+import java.util.ServiceLoader;
+import java.util.stream.Collectors;
+
+/**
+ * Use SPI to create {@link TableSourceFactory}, {@link TableSinkFactory} and {@link CatalogFactory}.
+ */
+public final class FactoryUtil {
+
+ private static final Logger LOG = LoggerFactory.getLogger(FactoryUtil.class);
+
+ public static <T, SplitT extends SourceSplit, StateT extends Serializable> List<SeaTunnelSource<T, SplitT, StateT>> createAndPrepareSource(
+ List<CatalogTable> multipleTables,
+ Map<String, String> options,
+ ClassLoader classLoader,
+ String factoryIdentifier) {
+
+ try {
+ final TableSourceFactory factory = discoverFactory(classLoader, TableSourceFactory.class, factoryIdentifier);
+ List<SeaTunnelSource<T, SplitT, StateT>> sources = new ArrayList<>(multipleTables.size());
+ if (factory instanceof SupportMultipleTable) {
+ TableFactoryContext context = new TableFactoryContext(multipleTables, options, classLoader);
+ SupportMultipleTable multipleTableSourceFactory = (SupportMultipleTable) factory;
+ // TODO: create all source
+ SupportMultipleTable.Result result = multipleTableSourceFactory.applyTables(context);
+ TableSource<T, SplitT, StateT> multipleTableSource = factory.createSource(
+ new TableFactoryContext(result.getAcceptedTables(), options, classLoader));
+ // TODO: handle reading metadata
+ SeaTunnelSource<T, SplitT, StateT> source = multipleTableSource.createSource();
+ sources.add(source);
+ }
+ return sources;
+ } catch (Throwable t) {
+ throw new FactoryException(
+ String.format(
+ "Unable to create a source for identifier '%s'.", factoryIdentifier),
+ t);
+ }
+ }
+
+ public static <IN, StateT, CommitInfoT, AggregatedCommitInfoT> SeaTunnelSink<IN, StateT, CommitInfoT, AggregatedCommitInfoT> createAndPrepareSink(
+ ClassLoader classLoader, String factoryIdentifier) {
+ // todo: do we need to set table?
+ TableSinkFactory<IN, StateT, CommitInfoT, AggregatedCommitInfoT> factory = discoverFactory(classLoader, TableSinkFactory.class, factoryIdentifier);
+ return factory.createSink(null).createSink();
+ }
+
+ public static Catalog createCatalog(String catalogName,
+ Map<String, String> options,
+ ClassLoader classLoader,
+ String factoryIdentifier) {
+ CatalogFactory catalogFactory = discoverFactory(classLoader, CatalogFactory.class, factoryIdentifier);
+ return catalogFactory.createCatalog(catalogName, options);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T extends Factory> T discoverFactory(
+ ClassLoader classLoader, Class<T> factoryClass, String factoryIdentifier) {
+ final List<Factory> factories = discoverFactories(classLoader);
+
+ final List<Factory> foundFactories =
+ factories.stream()
+ .filter(f -> factoryClass.isAssignableFrom(f.getClass()))
+ .collect(Collectors.toList());
+
+ if (foundFactories.isEmpty()) {
+ throw new FactoryException(
+ String.format(
+ "Could not find any factories that implement '%s' in the classpath.",
+ factoryClass.getName()));
+ }
+
+ final List<Factory> matchingFactories =
+ foundFactories.stream()
+ .filter(f -> f.factoryIdentifier().equals(factoryIdentifier))
+ .collect(Collectors.toList());
+
+ if (matchingFactories.isEmpty()) {
+ throw new FactoryException(
+ String.format(
+ "Could not find any factory for identifier '%s' that implements '%s' in the classpath.\n\n"
+ + "Available factory identifiers are:\n\n"
+ + "%s",
+ factoryIdentifier,
+ factoryClass.getName(),
+ foundFactories.stream()
+ .map(Factory::factoryIdentifier)
+ .distinct()
+ .sorted()
+ .collect(Collectors.joining("\n"))));
+ }
+
+ if (matchingFactories.size() > 1) {
+ throw new FactoryException(
+ String.format(
+ "Multiple factories for identifier '%s' that implement '%s' found in the classpath.\n\n"
+ + "Ambiguous factory classes are:\n\n"
+ + "%s",
+ factoryIdentifier,
+ factoryClass.getName(),
+ matchingFactories.stream()
+ .map(f -> f.getClass().getName())
+ .sorted()
+ .collect(Collectors.joining("\n"))));
+ }
+
+ return (T) matchingFactories.get(0);
+ }
+
+ private static List<Factory> discoverFactories(ClassLoader classLoader) {
+ try {
+ final List<Factory> result = new LinkedList<>();
+ ServiceLoader.load(Factory.class, classLoader)
+ .iterator()
+ .forEachRemaining(result::add);
+ return result;
+ } catch (ServiceConfigurationError e) {
+ LOG.error("Could not load service provider for factories.", e);
+ throw new FactoryException("Could not load service provider for factories.", e);
+ }
+ }
+}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/SerializationFormatFactory.java
similarity index 73%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/SerializationFormatFactory.java
index f5f308154..653bd03b3 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/SerializationFormatFactory.java
@@ -15,18 +15,10 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.api.table.factory;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import org.apache.seatunnel.api.table.connector.SerializationFormat;
- private final String type;
-
- PluginType(String type) {
- this.type = type;
- }
-
- public String getType() {
- return type;
- }
+public interface SerializationFormatFactory extends Factory {
+ SerializationFormat createSerializationFormat(TableFactoryContext context);
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/SupportMultipleTable.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/SupportMultipleTable.java
new file mode 100644
index 000000000..6b15786a1
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/SupportMultipleTable.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.factory;
+
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+
+import java.util.List;
+
+/**
+ * Used to declare that the connector can handle data from multiple tables.
+ * <p> The expansion of the {@link TableSourceFactory}.
+ */
+public interface SupportMultipleTable {
+
+ /**
+ * A connector can pick tables and return the accepted and remaining tables.
+ */
+ Result applyTables(TableFactoryContext context);
+
+ final class Result {
+ private final List<CatalogTable> acceptedTables;
+ private final List<CatalogTable> remainingTables;
+
+ private Result(
+ List<CatalogTable> acceptedTables,
+ List<CatalogTable> remainingTables) {
+ this.acceptedTables = acceptedTables;
+ this.remainingTables = remainingTables;
+ }
+
+ public static Result of(
+ List<CatalogTable> acceptedTables,
+ List<CatalogTable> remainingTables) {
+ return new Result(acceptedTables, remainingTables);
+ }
+
+ public List<CatalogTable> getAcceptedTables() {
+ return acceptedTables;
+ }
+
+ public List<CatalogTable> getRemainingTables() {
+ return remainingTables;
+ }
+ }
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/TableFactoryContext.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/TableFactoryContext.java
new file mode 100644
index 000000000..0c01436ce
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/TableFactoryContext.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.factory;
+
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+
+import java.util.List;
+import java.util.Map;
+
+public class TableFactoryContext {
+
+ private final List<CatalogTable> catalogTables;
+ private final Map<String, String> options;
+ private final ClassLoader classLoader;
+
+ public TableFactoryContext(
+ List<CatalogTable> catalogTables,
+ Map<String, String> options,
+ ClassLoader classLoader) {
+ this.catalogTables = catalogTables;
+ this.options = options;
+ this.classLoader = classLoader;
+ }
+
+ public ClassLoader getClassLoader() {
+ return this.classLoader;
+ }
+
+ /**
+ * Returns a list of tables that need to be processed.
+ *
+ * <p> By default, return only single table.
+ *
+ * <p> If you need multiple tables, implement {@link SupportMultipleTable}.
+ */
+ public List<CatalogTable> getCatalogTables() {
+ return catalogTables;
+ }
+
+ public Map<String, String> getOptions() {
+ return this.options;
+ }
+}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/TableSinkFactory.java
similarity index 51%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/TableSinkFactory.java
index 5cd80fc53..b46c66c21 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/TableSinkFactory.java
@@ -15,21 +15,21 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
+package org.apache.seatunnel.api.table.factory;
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+import org.apache.seatunnel.api.table.connector.TableSink;
/**
- * Command interface.
+ * This is an SPI interface, used to create {@link TableSink}. Each plugin need to have it own implementation.
+ * todo: now we have not use this interface, we directly use {@link org.apache.seatunnel.api.sink.SeaTunnelSink} as the SPI interface.
*
- * @param <T> args type
+ * @param <IN> row type
+ * @param <StateT> state type
+ * @param <CommitInfoT> commit info type
+ * @param <AggregatedCommitInfoT> aggregated commit info type
*/
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
+public interface TableSinkFactory<IN, StateT, CommitInfoT, AggregatedCommitInfoT> extends Factory {
- /**
- * Execute command
- */
- void execute();
+ TableSink<IN, StateT, CommitInfoT, AggregatedCommitInfoT> createSink(TableFactoryContext context);
}
diff --git a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/TableSourceFactory.java
similarity index 56%
copy from seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/TableSourceFactory.java
index f0cf54447..d637cc970 100644
--- a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/factory/TableSourceFactory.java
@@ -15,33 +15,18 @@
* limitations under the License.
*/
-package org.apache.seatunnel.apis.base.env;
+package org.apache.seatunnel.api.table.factory;
-import org.apache.seatunnel.common.config.CheckResult;
-import org.apache.seatunnel.common.constants.JobMode;
+import org.apache.seatunnel.api.source.SourceSplit;
+import org.apache.seatunnel.api.table.connector.TableSource;
-import org.apache.seatunnel.shade.com.typesafe.config.Config;
-
-import java.net.URL;
-import java.util.Set;
+import java.io.Serializable;
/**
- * engine related runtime environment
+ * This is an SPI interface, used to create {@link TableSource}. Each plugin need to have it own implementation.
+ * todo: now we have not use this interface, we directly use {@link org.apache.seatunnel.api.source.SeaTunnelSource} as the SPI interface
*/
-public interface RuntimeEnv {
-
- RuntimeEnv setConfig(Config config);
-
- Config getConfig();
-
- CheckResult checkConfig();
-
- RuntimeEnv prepare();
-
- RuntimeEnv setJobMode(JobMode mode);
-
- JobMode getJobMode();
-
- void registerPlugin(Set<URL> pluginPaths);
+public interface TableSourceFactory extends Factory {
+ <T, SplitT extends SourceSplit, StateT extends Serializable> TableSource<T, SplitT, StateT> createSource(TableFactoryContext context);
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/ArrayType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/ArrayType.java
new file mode 100644
index 000000000..d357e036a
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/ArrayType.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.type;
+
+import java.util.Objects;
+
+public class ArrayType<T, E> implements SeaTunnelDataType<T> {
+ private static final long serialVersionUID = 2L;
+
+ public static final ArrayType<String[], String> STRING_ARRAY_TYPE =
+ new ArrayType<>(String[].class, BasicType.STRING_TYPE);
+ public static final ArrayType<Boolean[], Boolean> BOOLEAN_ARRAY_TYPE =
+ new ArrayType<>(Boolean[].class, BasicType.BOOLEAN_TYPE);
+ public static final ArrayType<Byte[], Byte> BYTE_ARRAY_TYPE =
+ new ArrayType<>(Byte[].class, BasicType.BYTE_TYPE);
+ public static final ArrayType<Short[], Short> SHORT_ARRAY_TYPE =
+ new ArrayType<>(Short[].class, BasicType.SHORT_TYPE);
+ public static final ArrayType<Integer[], Integer> INT_ARRAY_TYPE =
+ new ArrayType<>(Integer[].class, BasicType.INT_TYPE);
+ public static final ArrayType<Long[], Long> LONG_ARRAY_TYPE =
+ new ArrayType<>(Long[].class, BasicType.LONG_TYPE);
+ public static final ArrayType<Float[], Float> FLOAT_ARRAY_TYPE =
+ new ArrayType<>(Float[].class, BasicType.FLOAT_TYPE);
+ public static final ArrayType<Double[], Double> DOUBLE_ARRAY_TYPE =
+ new ArrayType<>(Double[].class, BasicType.DOUBLE_TYPE);
+
+ // --------------------------------------------------------------------------------------------
+
+ private final Class<T> arrayClass;
+ private final BasicType<E> elementType;
+
+ private ArrayType(Class<T> arrayClass, BasicType<E> elementType) {
+ this.arrayClass = arrayClass;
+ this.elementType = elementType;
+ }
+
+ public BasicType<E> getElementType() {
+ return elementType;
+ }
+
+ @Override
+ public Class<T> getTypeClass() {
+ return arrayClass;
+ }
+
+ @Override
+ public SqlType getSqlType() {
+ return SqlType.ARRAY;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(arrayClass, elementType);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof ArrayType)) {
+ return false;
+ }
+ ArrayType<?, ?> that = (ArrayType<?, ?>) obj;
+ return Objects.equals(arrayClass, that.arrayClass) && Objects.equals(elementType, that.elementType);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("ARRAY<%s>", elementType);
+ }
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/BasicType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/BasicType.java
new file mode 100644
index 000000000..70769101d
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/BasicType.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.type;
+
+import java.util.Objects;
+
+public class BasicType<T> implements SeaTunnelDataType<T> {
+ private static final long serialVersionUID = 2L;
+
+ public static final BasicType<String> STRING_TYPE = new BasicType<>(String.class, SqlType.STRING);
+ public static final BasicType<Boolean> BOOLEAN_TYPE = new BasicType<>(Boolean.class, SqlType.BOOLEAN);
+ public static final BasicType<Byte> BYTE_TYPE = new BasicType<>(Byte.class, SqlType.TINYINT);
+ public static final BasicType<Short> SHORT_TYPE = new BasicType<>(Short.class, SqlType.SMALLINT);
+ public static final BasicType<Integer> INT_TYPE = new BasicType<>(Integer.class, SqlType.INT);
+ public static final BasicType<Long> LONG_TYPE = new BasicType<>(Long.class, SqlType.BIGINT);
+ public static final BasicType<Float> FLOAT_TYPE = new BasicType<>(Float.class, SqlType.FLOAT);
+ public static final BasicType<Double> DOUBLE_TYPE = new BasicType<>(Double.class, SqlType.DOUBLE);
+ public static final BasicType<Void> VOID_TYPE = new BasicType<>(Void.class, SqlType.NULL);
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * The physical type class.
+ */
+ private final Class<T> typeClass;
+ private final SqlType sqlType;
+
+ protected BasicType(Class<T> typeClass, SqlType sqlType) {
+ this.typeClass = typeClass;
+ this.sqlType = sqlType;
+ }
+
+ @Override
+ public Class<T> getTypeClass() {
+ return this.typeClass;
+ }
+
+ @Override
+ public SqlType getSqlType() {
+ return this.sqlType;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof BasicType)) {
+ return false;
+ }
+ BasicType<?> that = (BasicType<?>) obj;
+ return Objects.equals(typeClass, that.typeClass) && Objects.equals(sqlType, that.sqlType);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(typeClass, sqlType);
+ }
+
+ @Override
+ public String toString() {
+ return sqlType.toString();
+ }
+}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/CompositeType.java
similarity index 74%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/CompositeType.java
index f5f308154..0abf7a41c 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/CompositeType.java
@@ -15,18 +15,11 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.api.table.type;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import java.util.List;
- private final String type;
+public interface CompositeType<T> extends SeaTunnelDataType<T> {
- PluginType(String type) {
- this.type = type;
- }
-
- public String getType() {
- return type;
- }
+ List<SeaTunnelDataType<?>> getChildren();
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/DecimalType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/DecimalType.java
new file mode 100644
index 000000000..2f85ad5bd
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/DecimalType.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.type;
+
+import java.math.BigDecimal;
+import java.util.Objects;
+
+public final class DecimalType extends BasicType<BigDecimal> {
+ private static final long serialVersionUID = 1L;
+
+ private final int precision;
+
+ private final int scale;
+
+ public DecimalType(int precision, int scale) {
+ super(BigDecimal.class, SqlType.DECIMAL);
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof DecimalType)) {
+ return false;
+ }
+ DecimalType that = (DecimalType) o;
+ return this.precision == that.precision && this.scale == that.scale;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(precision, scale);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Decimal(%d, %d)", precision, scale);
+ }
+
+ public int getPrecision() {
+ return precision;
+ }
+
+ public int getScale() {
+ return scale;
+ }
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/LocalTimeType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/LocalTimeType.java
new file mode 100644
index 000000000..f6b897260
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/LocalTimeType.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.type;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.temporal.Temporal;
+import java.util.Objects;
+
+public class LocalTimeType<T extends Temporal> implements SeaTunnelDataType<T> {
+ private static final long serialVersionUID = 2L;
+
+ public static final LocalTimeType<LocalDate> LOCAL_DATE_TYPE = new LocalTimeType<>(LocalDate.class, SqlType.DATE);
+ public static final LocalTimeType<LocalTime> LOCAL_TIME_TYPE = new LocalTimeType<>(LocalTime.class, SqlType.TIME);
+ public static final LocalTimeType<LocalDateTime> LOCAL_DATE_TIME_TYPE = new LocalTimeType<>(LocalDateTime.class, SqlType.TIMESTAMP);
+
+ private final Class<T> typeClass;
+ private final SqlType sqlType;
+
+ private LocalTimeType(Class<T> typeClass, SqlType sqlType) {
+ this.typeClass = typeClass;
+ this.sqlType = sqlType;
+ }
+
+ @Override
+ public Class<T> getTypeClass() {
+ return typeClass;
+ }
+
+ @Override
+ public SqlType getSqlType() {
+ return this.sqlType;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(typeClass);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof LocalTimeType)) {
+ return false;
+ }
+ LocalTimeType<?> that = (LocalTimeType<?>) obj;
+ return Objects.equals(typeClass, that.typeClass);
+ }
+
+ @Override
+ public String toString() {
+ return sqlType.toString();
+ }
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/MapType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/MapType.java
new file mode 100644
index 000000000..af46f483e
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/MapType.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.type;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.collect.Lists;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class MapType<K, V> implements CompositeType<Map<K, V>> {
+
+ private static final List<SqlType> SUPPORTED_KEY_TYPES = Arrays.asList(
+ SqlType.NULL,
+ SqlType.BOOLEAN,
+ SqlType.TINYINT,
+ SqlType.SMALLINT,
+ SqlType.INT,
+ SqlType.BIGINT,
+ SqlType.DATE,
+ SqlType.TIME,
+ SqlType.TIMESTAMP,
+ SqlType.FLOAT,
+ SqlType.DOUBLE,
+ SqlType.STRING,
+ SqlType.DECIMAL
+ );
+
+ private final SeaTunnelDataType<K> keyType;
+ private final SeaTunnelDataType<V> valueType;
+
+ public MapType(SeaTunnelDataType<K> keyType, SeaTunnelDataType<V> valueType) {
+ checkNotNull(keyType, "The key type is required.");
+ checkNotNull(valueType, "The value type is required.");
+ checkArgument(SUPPORTED_KEY_TYPES.contains(keyType.getSqlType()), "Unsupported key types: %s", keyType);
+ this.keyType = keyType;
+ this.valueType = valueType;
+ }
+
+ public SeaTunnelDataType<K> getKeyType() {
+ return keyType;
+ }
+
+ public SeaTunnelDataType<V> getValueType() {
+ return valueType;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Class<Map<K, V>> getTypeClass() {
+ return (Class<Map<K, V>>) (Class<?>) Map.class;
+ }
+
+ @Override
+ public SqlType getSqlType() {
+ return SqlType.MAP;
+ }
+
+ @Override
+ public List<SeaTunnelDataType<?>> getChildren() {
+ return Lists.newArrayList(this.keyType, this.valueType);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof MapType)) {
+ return false;
+ }
+ MapType<?, ?> that = (MapType<?, ?>) obj;
+ return Objects.equals(keyType, that.keyType) && Objects.equals(valueType, that.valueType);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(keyType, valueType);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Map<%s, %s>", keyType, valueType);
+ }
+}
diff --git a/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/PrimitiveByteArrayType.java
similarity index 51%
copy from seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/PrimitiveByteArrayType.java
index ee715c102..66ac125dc 100644
--- a/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/PrimitiveByteArrayType.java
@@ -15,27 +15,39 @@
* limitations under the License.
*/
-package org.apache.seatunnel.common.config;
+package org.apache.seatunnel.api.table.type;
-import static org.junit.Assert.assertEquals;
+public class PrimitiveByteArrayType implements SeaTunnelDataType<byte[]> {
+ public static final PrimitiveByteArrayType INSTANCE = new PrimitiveByteArrayType();
-import org.junit.Test;
+ private PrimitiveByteArrayType() {
+ }
-import java.io.File;
+ @Override
+ public Class<byte[]> getTypeClass() {
+ return byte[].class;
+ }
-public class CommonTest {
+ @Override
+ public SqlType getSqlType() {
+ return SqlType.BYTES;
+ }
- static {
- Common.setDeployMode("client");
+ @Override
+ public int hashCode() {
+ return byte[].class.hashCode();
}
- @Test
- public void appLibDir() {
- assertEquals(Common.appRootDir().toString() + File.separator + "lib", Common.appLibDir().toString());
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ return obj instanceof PrimitiveByteArrayType;
}
- @Test
- public void pluginTarFile() {
- assertEquals(Common.appRootDir().toString() + File.separator + "plugins.tar.gz", Common.pluginTarball().toString());
+ @Override
+ public String toString() {
+ return SqlType.BYTES.toString();
}
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/RowKind.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/RowKind.java
new file mode 100644
index 000000000..528ce01ee
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/RowKind.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.type;
+
+/** Lists all kinds of changes that a row can describe in a changelog. */
+public enum RowKind {
+ // Note: Enums have no stable hash code across different JVMs, use toByteValue() for
+ // this purpose.
+
+ /** Insertion operation. */
+ INSERT("+I", (byte) 0),
+
+ /**
+ * Update operation with the previous content of the updated row.
+ *
+ * <p>This kind SHOULD occur together with {@link #UPDATE_AFTER} for modelling an update that
+ * needs to retract the previous row first. It is useful in cases of a non-idempotent update,
+ * i.e., an update of a row that is not uniquely identifiable by a key.
+ */
+ UPDATE_BEFORE("-U", (byte) 1),
+
+ /**
+ * Update operation with new content of the updated row.
+ *
+ * <p>This kind CAN occur together with {@link #UPDATE_BEFORE} for modelling an update that
+ * needs to retract the previous row first. OR it describes an idempotent update, i.e., an
+ * update of a row that is uniquely identifiable by a key.
+ */
+ UPDATE_AFTER("+U", (byte) 2),
+
+ /** Deletion operation. */
+ DELETE("-D", (byte) 3);
+
+ private final String shortString;
+
+ private final byte value;
+
+ /**
+ * Creates a {@link RowKind} enum with the given short string and byte value representation of
+ * the {@link RowKind}.
+ */
+ RowKind(String shortString, byte value) {
+ this.shortString = shortString;
+ this.value = value;
+ }
+
+ /**
+ * Returns a short string representation of this {@link RowKind}.
+ *
+ * <p>
+ *
+ * <ul>
+ * <li>"+I" represents {@link #INSERT}.
+ * <li>"-U" represents {@link #UPDATE_BEFORE}.
+ * <li>"+U" represents {@link #UPDATE_AFTER}.
+ * <li>"-D" represents {@link #DELETE}.
+ * </ul>
+ */
+ public String shortString() {
+ return shortString;
+ }
+
+ /**
+ * Returns the byte value representation of this {@link RowKind}. The byte value is used for
+ * serialization and deserialization.
+ *
+ * <p>
+ *
+ * <ul>
+ * <li>"0" represents {@link #INSERT}.
+ * <li>"1" represents {@link #UPDATE_BEFORE}.
+ * <li>"2" represents {@link #UPDATE_AFTER}.
+ * <li>"3" represents {@link #DELETE}.
+ * </ul>
+ */
+ public byte toByteValue() {
+ return value;
+ }
+
+ /**
+ * Creates a {@link RowKind} from the given byte value. Each {@link RowKind} has a byte value
+ * representation.
+ *
+ * @see #toByteValue() for mapping of byte value and {@link RowKind}.
+ */
+ @SuppressWarnings("MagicNumber")
+ public static RowKind fromByteValue(byte value) {
+ switch (value) {
+ case 0:
+ return INSERT;
+ case 1:
+ return UPDATE_BEFORE;
+ case 2:
+ return UPDATE_AFTER;
+ case 3:
+ return DELETE;
+ default:
+ throw new UnsupportedOperationException(
+ "Unsupported byte value '" + value + "' for row kind.");
+ }
+ }
+}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SeaTunnelDataType.java
similarity index 68%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SeaTunnelDataType.java
index 5cd80fc53..946f4a728 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SeaTunnelDataType.java
@@ -15,21 +15,22 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
+package org.apache.seatunnel.api.table.type;
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+import java.io.Serializable;
/**
- * Command interface.
- *
- * @param <T> args type
+ * Logic data type of column in SeaTunnel.
*/
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
+public interface SeaTunnelDataType<T> extends Serializable {
/**
- * Execute command
+ * Gets the class of the type represented by this data type.
*/
- void execute();
+ Class<T> getTypeClass();
+ /**
+ * Gets the SQL standard type represented by this data type.
+ */
+ SqlType getSqlType();
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SeaTunnelRow.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SeaTunnelRow.java
new file mode 100644
index 000000000..aa5923ec1
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SeaTunnelRow.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.type;
+
+import org.apache.seatunnel.api.table.factory.SupportMultipleTable;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * SeaTunnel row type.
+ */
+public final class SeaTunnelRow implements Serializable {
+ private static final long serialVersionUID = -1L;
+ /** Table identifier, used for the source connector that {@link SupportMultipleTable}. */
+ private int tableId = -1;
+ /** The kind of change that a row describes in a changelog. */
+ private RowKind kind = RowKind.INSERT;
+ /** The array to store the actual internal format values. */
+ private final Object[] fields;
+
+ public SeaTunnelRow(int arity) {
+ this.fields = new Object[arity];
+ }
+
+ public SeaTunnelRow(Object[] fields) {
+ this.fields = fields;
+ }
+
+ public void setField(int pos, Object value) {
+ this.fields[pos] = value;
+ }
+
+ public void setTableId(int tableId) {
+ this.tableId = tableId;
+ }
+
+ public void setRowKind(RowKind kind) {
+ this.kind = kind;
+ }
+
+ public int getArity() {
+ return fields.length;
+ }
+
+ public int getTableId() {
+ return tableId;
+ }
+
+ public RowKind getRowKind() {
+ return this.kind;
+ }
+
+ public Object[] getFields() {
+ return fields;
+ }
+
+ public Object getField(int pos) {
+ return this.fields[pos];
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof SeaTunnelRow)) {
+ return false;
+ }
+ SeaTunnelRow that = (SeaTunnelRow) o;
+ return tableId == that.tableId && kind == that.kind && Arrays.deepEquals(fields, that.fields);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hash(tableId, kind);
+ result = 31 * result + Arrays.deepHashCode(fields);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "SeaTunnelRow{" +
+ "tableId=" + tableId +
+ ", kind=" + kind.shortString() +
+ ", fields=" + Arrays.toString(fields) +
+ '}';
+ }
+}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SeaTunnelRowType.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SeaTunnelRowType.java
new file mode 100644
index 000000000..cb94bba00
--- /dev/null
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SeaTunnelRowType.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.api.table.type;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class SeaTunnelRowType implements CompositeType<SeaTunnelRow> {
+ private static final long serialVersionUID = 2L;
+
+ /**
+ * The field name of the {@link SeaTunnelRow}.
+ */
+ private final String[] fieldNames;
+ /**
+ * The type of the field.
+ */
+ private final SeaTunnelDataType<?>[] fieldTypes;
+
+ public SeaTunnelRowType(String[] fieldNames, SeaTunnelDataType<?>[] fieldTypes) {
+ checkArgument(fieldNames.length == fieldTypes.length,
+ "The number of field names must be the same as the number of field types.");
+ this.fieldNames = fieldNames;
+ this.fieldTypes = fieldTypes;
+ }
+
+ @Override
+ public Class<SeaTunnelRow> getTypeClass() {
+ return SeaTunnelRow.class;
+ }
+
+ @Override
+ public SqlType getSqlType() {
+ return SqlType.ROW;
+ }
+
+ public String[] getFieldNames() {
+ return fieldNames;
+ }
+
+ public SeaTunnelDataType<?>[] getFieldTypes() {
+ return fieldTypes;
+ }
+
+ @Override
+ public List<SeaTunnelDataType<?>> getChildren() {
+ return Arrays.asList(fieldTypes);
+ }
+
+ public int getTotalFields() {
+ return fieldTypes.length;
+ }
+
+ public String getFieldName(int index) {
+ return fieldNames[index];
+ }
+
+ public SeaTunnelDataType<?> getFieldType(int index) {
+ return fieldTypes[index];
+ }
+
+ public int indexOf(String fieldName) {
+ for (int i = 0; i < fieldNames.length; i++) {
+ if (fieldNames[i].equals(fieldName)) {
+ return i;
+ }
+ }
+ throw new IllegalArgumentException(String.format("can't find field %s", fieldName));
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof SeaTunnelRowType)) {
+ return false;
+ }
+ SeaTunnelRowType that = (SeaTunnelRowType) obj;
+ return Arrays.equals(fieldNames, that.fieldNames) && Arrays.equals(fieldTypes, that.fieldTypes);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Arrays.hashCode(fieldNames);
+ result = 31 * result + Arrays.hashCode(fieldTypes);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder("ROW<");
+ for (int i = 0; i < fieldNames.length; i++) {
+ if (i > 0) {
+ builder.append(",");
+ }
+ builder.append(fieldNames[i])
+ .append(" ")
+ .append(fieldTypes[i]);
+ }
+ return builder.append(">").toString();
+ }
+}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SqlType.java
similarity index 72%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
copy to seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SqlType.java
index 5cd80fc53..ad8b3c347 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/SqlType.java
@@ -15,21 +15,27 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.command;
-
-import org.apache.seatunnel.apis.base.command.CommandArgs;
+package org.apache.seatunnel.api.table.type;
/**
- * Command interface.
- *
- * @param <T> args type
+ * The sql type of {@link SeaTunnelDataType}.
*/
-@FunctionalInterface
-public interface Command<T extends CommandArgs> {
-
- /**
- * Execute command
- */
- void execute();
-
+public enum SqlType {
+ ARRAY,
+ MAP,
+ STRING,
+ BOOLEAN,
+ TINYINT,
+ SMALLINT,
+ INT,
+ BIGINT,
+ FLOAT,
+ DOUBLE,
+ DECIMAL,
+ NULL,
+ BYTES,
+ DATE,
+ TIME,
+ TIMESTAMP,
+ ROW;
}
diff --git a/seatunnel-apis/pom.xml b/seatunnel-apis/pom.xml
index 7ffd22572..ee043b0c3 100644
--- a/seatunnel-apis/pom.xml
+++ b/seatunnel-apis/pom.xml
@@ -17,7 +17,9 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel</artifactId>
diff --git a/seatunnel-apis/seatunnel-api-base/pom.xml b/seatunnel-apis/seatunnel-api-base/pom.xml
index a31906b20..1e40b3ee3 100644
--- a/seatunnel-apis/seatunnel-api-base/pom.xml
+++ b/seatunnel-apis/seatunnel-api-base/pom.xml
@@ -17,7 +17,9 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-apis</artifactId>
@@ -39,4 +41,5 @@
<artifactId>seatunnel-config-shade</artifactId>
</dependency>
</dependencies>
+
</project>
diff --git a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java b/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
index f0cf54447..e8532a208 100644
--- a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
+++ b/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
@@ -23,7 +23,7 @@ import org.apache.seatunnel.common.constants.JobMode;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
import java.net.URL;
-import java.util.Set;
+import java.util.List;
/**
* engine related runtime environment
@@ -42,6 +42,6 @@ public interface RuntimeEnv {
JobMode getJobMode();
- void registerPlugin(Set<URL> pluginPaths);
+ void registerPlugin(List<URL> pluginPaths);
}
diff --git a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/plugin/Plugin.java b/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/plugin/Plugin.java
index 63370c44c..6cf048c50 100644
--- a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/plugin/Plugin.java
+++ b/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/plugin/Plugin.java
@@ -28,13 +28,13 @@ import java.io.Serializable;
* A base interface indicates belonging to SeaTunnel.
* Plugin will be used as follows:
* <pre>{@code
- * Plugin<?> plugin = new PluginA<>();
- * plugin.setConfig(Config);
- * CheckResult checkResult = plugin.checkConfig();
- * if (checkResult.getSuccess()) {
- * plugin.prepare();
- * // plugin execute code
- * plugin.close();
+ * try(Plugin<?> plugin = new PluginA<>()) {
+ * plugin.setConfig(Config);
+ * CheckResult checkResult = plugin.checkConfig();
+ * if (checkResult.getSuccess()) {
+ * plugin.prepare();
+ * // plugin execute code
+ * }
* }
*
* }</pre>
diff --git a/seatunnel-apis/seatunnel-api-flink/pom.xml b/seatunnel-apis/seatunnel-api-flink/pom.xml
index d32f642ea..cb2e1ef57 100644
--- a/seatunnel-apis/seatunnel-api-flink/pom.xml
+++ b/seatunnel-apis/seatunnel-api-flink/pom.xml
@@ -17,7 +17,9 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-apis</artifactId>
@@ -92,4 +94,5 @@
</dependency>
</dependencies>
+
</project>
diff --git a/seatunnel-apis/seatunnel-api-flink/src/main/java/org/apache/seatunnel/flink/FlinkEnvironment.java b/seatunnel-apis/seatunnel-api-flink/src/main/java/org/apache/seatunnel/flink/FlinkEnvironment.java
index b9160e4f1..49c97beb6 100644
--- a/seatunnel-apis/seatunnel-api-flink/src/main/java/org/apache/seatunnel/flink/FlinkEnvironment.java
+++ b/seatunnel-apis/seatunnel-api-flink/src/main/java/org/apache/seatunnel/flink/FlinkEnvironment.java
@@ -26,6 +26,7 @@ import org.apache.seatunnel.flink.util.EnvironmentUtil;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.configuration.Configuration;
@@ -49,7 +50,6 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
-import java.util.Set;
import java.util.stream.Collectors;
public class FlinkEnvironment implements RuntimeEnv {
@@ -88,10 +88,10 @@ public class FlinkEnvironment implements RuntimeEnv {
@Override
public FlinkEnvironment prepare() {
- if (isStreaming()) {
- createStreamEnvironment();
- createStreamTableEnvironment();
- } else {
+ // Batch/Streaming both use data stream api in SeaTunnel New API
+ createStreamEnvironment();
+ createStreamTableEnvironment();
+ if (!isStreaming()) {
createExecutionEnvironment();
createBatchTableEnvironment();
}
@@ -121,33 +121,33 @@ public class FlinkEnvironment implements RuntimeEnv {
}
@Override
- public void registerPlugin(Set<URL> pluginPaths) {
- LOGGER.info("register plugins :" + pluginPaths);
- Configuration configuration;
+ public void registerPlugin(List<URL> pluginPaths) {
+ pluginPaths.forEach(url -> LOGGER.info("register plugins : {}", url));
+ List<Configuration> configurations = new ArrayList<>();
try {
- if (isStreaming()) {
- configuration =
- (Configuration) Objects.requireNonNull(ReflectionUtils.getDeclaredMethod(StreamExecutionEnvironment.class,
- "getConfiguration")).orElseThrow(() -> new RuntimeException("can't find " +
- "method: getConfiguration")).invoke(this.environment);
- } else {
- configuration = batchEnvironment.getConfiguration();
+ configurations.add((Configuration) Objects.requireNonNull(ReflectionUtils.getDeclaredMethod(StreamExecutionEnvironment.class,
+ "getConfiguration")).orElseThrow(() -> new RuntimeException("can't find " +
+ "method: getConfiguration")).invoke(this.environment));
+ if (!isStreaming()) {
+ configurations.add(batchEnvironment.getConfiguration());
}
} catch (Exception e) {
throw new RuntimeException(e);
}
- List<String> jars = configuration.get(PipelineOptions.JARS);
- if (jars == null) {
- jars = new ArrayList<>();
- }
- jars.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
- configuration.set(PipelineOptions.JARS, jars.stream().distinct().collect(Collectors.toList()));
- List<String> classpath = configuration.get(PipelineOptions.CLASSPATHS);
- if (classpath == null) {
- classpath = new ArrayList<>();
- }
- classpath.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
- configuration.set(PipelineOptions.CLASSPATHS, classpath.stream().distinct().collect(Collectors.toList()));
+ configurations.forEach(configuration -> {
+ List<String> jars = configuration.get(PipelineOptions.JARS);
+ if (jars == null) {
+ jars = new ArrayList<>();
+ }
+ jars.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
+ configuration.set(PipelineOptions.JARS, jars.stream().distinct().collect(Collectors.toList()));
+ List<String> classpath = configuration.get(PipelineOptions.CLASSPATHS);
+ if (classpath == null) {
+ classpath = new ArrayList<>();
+ }
+ classpath.addAll(pluginPaths.stream().map(URL::toString).collect(Collectors.toList()));
+ configuration.set(PipelineOptions.CLASSPATHS, classpath.stream().distinct().collect(Collectors.toList()));
+ });
}
public StreamExecutionEnvironment getStreamExecutionEnvironment() {
@@ -202,6 +202,10 @@ public class FlinkEnvironment implements RuntimeEnv {
int max = config.getInt(ConfigKeyName.MAX_PARALLELISM);
environment.setMaxParallelism(max);
}
+
+ if (this.jobMode.equals(JobMode.BATCH)) {
+ environment.setRuntimeMode(RuntimeExecutionMode.BATCH);
+ }
}
public ExecutionEnvironment getBatchEnvironment() {
diff --git a/seatunnel-apis/seatunnel-api-spark/pom.xml b/seatunnel-apis/seatunnel-api-spark/pom.xml
index 965d646f6..27099c5fa 100644
--- a/seatunnel-apis/seatunnel-api-spark/pom.xml
+++ b/seatunnel-apis/seatunnel-api-spark/pom.xml
@@ -17,7 +17,9 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-apis</artifactId>
@@ -57,4 +59,5 @@
<artifactId>lz4</artifactId>
</dependency>
</dependencies>
+
</project>
diff --git a/seatunnel-apis/seatunnel-api-spark/src/main/java/org/apache/seatunnel/spark/SparkEnvironment.java b/seatunnel-apis/seatunnel-api-spark/src/main/java/org/apache/seatunnel/spark/SparkEnvironment.java
index bb0bdd9b0..3e2d5e91b 100644
--- a/seatunnel-apis/seatunnel-api-spark/src/main/java/org/apache/seatunnel/spark/SparkEnvironment.java
+++ b/seatunnel-apis/seatunnel-api-spark/src/main/java/org/apache/seatunnel/spark/SparkEnvironment.java
@@ -38,7 +38,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.URL;
-import java.util.Set;
+import java.util.List;
public class SparkEnvironment implements RuntimeEnv {
@@ -89,7 +89,7 @@ public class SparkEnvironment implements RuntimeEnv {
}
@Override
- public void registerPlugin(Set<URL> pluginPaths) {
+ public void registerPlugin(List<URL> pluginPaths) {
LOGGER.info("register plugins :" + pluginPaths);
// TODO we use --jar parameter to support submit multi-jar in spark cluster at now. Refactor it to
// support submit multi-jar in code or remove this logic.
diff --git a/seatunnel-common/pom.xml b/seatunnel-common/pom.xml
index 9756705d2..210d24baf 100644
--- a/seatunnel-common/pom.xml
+++ b/seatunnel-common/pom.xml
@@ -17,7 +17,9 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel</artifactId>
@@ -36,6 +38,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-collections4</artifactId>
+ </dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
@@ -44,6 +50,11 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java
index e9052bfe6..7d49900e5 100644
--- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java
+++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java
@@ -19,7 +19,11 @@ package org.apache.seatunnel.common;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.commons.lang3.StringUtils;
+
import java.util.Properties;
+import java.util.function.Consumer;
+import java.util.function.Function;
public final class PropertiesUtil {
@@ -39,4 +43,37 @@ public final class PropertiesUtil {
}
});
}
+
+ public static <E extends Enum<E>> E getEnum(final Config conf, final String key, final Class<E> enumClass, final E defaultEnum) {
+ if (!conf.hasPath(key)) {
+ return defaultEnum;
+ }
+ final String value = conf.getString(key);
+ if (StringUtils.isBlank(value)) {
+ return defaultEnum;
+ }
+ return Enum.valueOf(enumClass, value.toUpperCase());
+ }
+
+ public static <T> void setOption(Config config, String optionName, T defaultValue, Function<String, T> getter, Consumer<T> setter) {
+ T value;
+ if (config.hasPath(optionName)) {
+ value = getter.apply(optionName);
+ } else {
+ value = defaultValue;
+ }
+ if (value != null) {
+ setter.accept(value);
+ }
+ }
+
+ public static <T> void setOption(Config config, String optionName, Function<String, T> getter, Consumer<T> setter) {
+ T value = null;
+ if (config.hasPath(optionName)) {
+ value = getter.apply(optionName);
+ }
+ if (value != null) {
+ setter.accept(value);
+ }
+ }
}
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/Common.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/Common.java
index 5280250f5..67034db04 100644
--- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/Common.java
+++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/Common.java
@@ -21,10 +21,6 @@ import java.io.File;
import java.net.URISyntaxException;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
public class Common {
@@ -32,28 +28,27 @@ public class Common {
throw new IllegalStateException("Utility class");
}
- private static final List<String> ALLOWED_MODES = Arrays.stream(DeployMode.values())
- .map(DeployMode::getName).collect(Collectors.toList());
+ /**
+ * Used to set the size when create a new collection(just to pass the checkstyle).
+ */
+ public static final int COLLECTION_SIZE = 16;
- private static Optional<String> MODE = Optional.empty();
+ private static DeployMode MODE;
- public static boolean isModeAllowed(String mode) {
- return ALLOWED_MODES.contains(mode.toLowerCase());
- }
+ private static boolean STARTER = false;
/**
* Set mode. return false in case of failure
*/
- public static Boolean setDeployMode(String m) {
- if (isModeAllowed(m)) {
- MODE = Optional.of(m);
- return true;
- } else {
- return false;
- }
+ public static void setDeployMode(DeployMode mode) {
+ MODE = mode;
+ }
+
+ public static void setStarter(boolean inStarter) {
+ STARTER = inStarter;
}
- public static Optional<String> getDeployMode() {
+ public static DeployMode getDeployMode() {
return MODE;
}
@@ -65,7 +60,7 @@ public class Common {
* When running seatunnel in --master yarn or --master mesos, you can put plugins related files in plugins dir.
*/
public static Path appRootDir() {
- if (MODE.equals(Optional.of(DeployMode.CLIENT.getName()))) {
+ if (DeployMode.CLIENT == MODE || STARTER) {
try {
String path = Common.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath();
path = new File(path).getPath();
@@ -73,10 +68,10 @@ public class Common {
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
- } else if (MODE.equals(Optional.of(DeployMode.CLUSTER.getName()))) {
+ } else if (DeployMode.CLUSTER == MODE) {
return Paths.get("");
} else {
- throw new IllegalStateException("MODE not support : " + MODE.orElse("null"));
+ throw new IllegalStateException("deploy mode not support : " + MODE);
}
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/constants/CollectionConstants.java
similarity index 73%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-common/src/main/java/org/apache/seatunnel/common/constants/CollectionConstants.java
index f5f308154..28f5dc4d7 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/constants/CollectionConstants.java
@@ -15,18 +15,9 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.common.constants;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+public class CollectionConstants {
- private final String type;
-
- PluginType(String type) {
- this.type = type;
- }
-
- public String getType() {
- return type;
- }
+ public static final int MAP_SIZE = 6;
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/constants/PluginType.java
similarity index 92%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-common/src/main/java/org/apache/seatunnel/common/constants/PluginType.java
index f5f308154..d02e39859 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/constants/PluginType.java
@@ -15,9 +15,13 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.common.constants;
+/**
+ * The type of SeaTunnel plugin.
+ */
public enum PluginType {
+
SOURCE("source"), TRANSFORM("transform"), SINK("sink");
private final String type;
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/ReflectionUtils.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/ReflectionUtils.java
index 597a8df10..f186f0486 100644
--- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/ReflectionUtils.java
+++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/ReflectionUtils.java
@@ -17,6 +17,8 @@
package org.apache.seatunnel.common.utils;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Optional;
@@ -26,7 +28,7 @@ public class ReflectionUtils {
Optional<Method> method = Optional.empty();
Method m;
- for (; clazz != Object.class; clazz = clazz.getSuperclass()) {
+ for (; clazz != null; clazz = clazz.getSuperclass()) {
try {
m = clazz.getDeclaredMethod(methodName, parameterTypes);
m.setAccessible(true);
@@ -39,4 +41,56 @@ public class ReflectionUtils {
return method;
}
+ public static Optional<Object> getField(Object object, Class<?> clazz, String fieldName) {
+ try {
+ Field field = clazz.getDeclaredField(fieldName);
+ field.setAccessible(true);
+ return Optional.of(field.get(object));
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ return Optional.empty();
+ }
+ }
+
+ public static Optional<Object> getField(Object object, String fieldName) {
+ return getField(object, object.getClass(), fieldName);
+ }
+
+ public static void setField(Object object, Class<?> clazz, String fieldName, Object value) {
+ try {
+ Field field = clazz.getDeclaredField(fieldName);
+ field.setAccessible(true);
+ field.set(object, value);
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ throw new RuntimeException("Incompatible KafkaProducer version", e);
+ }
+ }
+
+ public static void setField(Object object, String fieldName, Object value) {
+ setField(object, object.getClass(), fieldName, value);
+ }
+
+ public static Object invoke(Object object, String methodName, Object... args) {
+ Class<?>[] argTypes = new Class[args.length];
+ for (int i = 0; i < args.length; i++) {
+ argTypes[i] = args[i].getClass();
+ }
+ return invoke(object, methodName, argTypes, args);
+ }
+
+ public static Object invoke(
+ Object object, String methodName, Class<?>[] argTypes, Object[] args) {
+ try {
+ Optional<Method> method = getDeclaredMethod(object.getClass(), methodName, argTypes);
+ if (method.isPresent()) {
+ method.get().setAccessible(true);
+ return method.get().invoke(object, args);
+ } else {
+ throw new NoSuchMethodException(String.format("method invoke failed, no such method '%s' in '%s'",
+ methodName, object.getClass()));
+ }
+ } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
+ throw new RuntimeException("method invoke failed", e);
+ }
+ }
+
}
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationException.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationException.java
new file mode 100644
index 000000000..33ca29a6f
--- /dev/null
+++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationException.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.common.utils;
+
+public class SerializationException extends RuntimeException {
+
+ /**
+ * Required for serialization support.
+ *
+ * @see java.io.Serializable
+ */
+ private static final long serialVersionUID = 2263144814025689516L;
+
+ /**
+ * <p>Constructs a new {@code SerializationException} without specified
+ * detail message.</p>
+ */
+ public SerializationException() {
+ }
+
+ /**
+ * <p>Constructs a new {@code SerializationException} with specified
+ * detail message.</p>
+ *
+ * @param msg The error message.
+ */
+ public SerializationException(final String msg) {
+ super(msg);
+ }
+
+ /**
+ * <p>Constructs a new {@code SerializationException} with specified
+ * nested {@code Throwable}.</p>
+ *
+ * @param cause The {@code Exception} or {@code Error}
+ * that caused this exception to be thrown.
+ */
+ public SerializationException(final Throwable cause) {
+ super(cause);
+ }
+
+ /**
+ * <p>Constructs a new {@code SerializationException} with specified
+ * detail message and nested {@code Throwable}.</p>
+ *
+ * @param msg The error message.
+ * @param cause The {@code Exception} or {@code Error}
+ * that caused this exception to be thrown.
+ */
+ public SerializationException(final String msg, final Throwable cause) {
+ super(msg, cause);
+ }
+}
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationUtils.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationUtils.java
new file mode 100644
index 000000000..7e721617e
--- /dev/null
+++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/SerializationUtils.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.common.utils;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.ObjectStreamClass;
+import java.io.Serializable;
+
+public class SerializationUtils {
+
+ public static String objectToString(Serializable obj) {
+ if (obj != null) {
+ return Base64.encodeBase64String(serialize(obj));
+ }
+ return null;
+ }
+
+ public static <T extends Serializable> T stringToObject(String str) {
+ if (StringUtils.isNotEmpty(str)) {
+ return deserialize(Base64.decodeBase64(str));
+ }
+ return null;
+ }
+
+ @SuppressWarnings("checkstyle:MagicNumber")
+ public static <T extends Serializable> byte[] serialize(T obj) {
+ try (ByteArrayOutputStream b = new ByteArrayOutputStream(512);
+ ObjectOutputStream out = new ObjectOutputStream(b)) {
+ out.writeObject(obj);
+ return b.toByteArray();
+ } catch (final IOException ex) {
+ throw new SerializationException(ex);
+ }
+
+ }
+
+ public static <T extends Serializable> T deserialize(byte[] bytes) {
+ try (ByteArrayInputStream s = new ByteArrayInputStream(bytes);
+ ObjectInputStream in = new ObjectInputStream(s) {
+ @Override
+ protected Class<?> resolveClass(ObjectStreamClass desc) throws IOException, ClassNotFoundException {
+ // make sure use current thread classloader
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ if (cl == null) {
+ return super.resolveClass(desc);
+ }
+ return Class.forName(desc.getName(), false, cl);
+ }
+ }) {
+ @SuppressWarnings("unchecked") final T obj = (T) in.readObject();
+ return obj;
+ } catch (final ClassNotFoundException | IOException ex) {
+ throw new SerializationException(ex);
+ }
+ }
+
+}
diff --git a/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java b/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java
index ee715c102..fed8b8f64 100644
--- a/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java
+++ b/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java
@@ -26,7 +26,7 @@ import java.io.File;
public class CommonTest {
static {
- Common.setDeployMode("client");
+ Common.setDeployMode(DeployMode.CLIENT);
}
@Test
diff --git a/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java b/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/ReflectionUtilsTest.java
similarity index 58%
copy from seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java
copy to seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/ReflectionUtilsTest.java
index ee715c102..422c70eef 100644
--- a/seatunnel-common/src/test/java/org/apache/seatunnel/common/config/CommonTest.java
+++ b/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/ReflectionUtilsTest.java
@@ -15,27 +15,24 @@
* limitations under the License.
*/
-package org.apache.seatunnel.common.config;
-
-import static org.junit.Assert.assertEquals;
+package org.apache.seatunnel.common.utils;
+import org.junit.Assert;
import org.junit.Test;
-import java.io.File;
-
-public class CommonTest {
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
- static {
- Common.setDeployMode("client");
- }
+public class ReflectionUtilsTest {
@Test
- public void appLibDir() {
- assertEquals(Common.appRootDir().toString() + File.separator + "lib", Common.appLibDir().toString());
- }
+ public void testInvoke() throws MalformedURLException {
+ ReflectionUtils.invoke(new String[]{}, "toString");
- @Test
- public void pluginTarFile() {
- assertEquals(Common.appRootDir().toString() + File.separator + "plugins.tar.gz", Common.pluginTarball().toString());
+ URLClassLoader classLoader = new URLClassLoader(new URL[]{}, Thread.currentThread().getContextClassLoader());
+ ReflectionUtils.invoke(classLoader, "addURL", new URL("file:///test"));
+ Assert.assertArrayEquals(classLoader.getURLs(), new URL[]{new URL("file:///test")});
}
+
}
diff --git a/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/SerializationUtilsTest.java b/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/SerializationUtilsTest.java
new file mode 100644
index 000000000..ad0e80cea
--- /dev/null
+++ b/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/SerializationUtilsTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.common.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+@SuppressWarnings("checkstyle:RegexpSingleline")
+public class SerializationUtilsTest {
+
+ @Test
+ public void testObjectToString() {
+
+ HashMap<String, String> data = new HashMap<>();
+ data.put("key1", "value1");
+ data.put("seatunnelTest", "apache SeaTunnel");
+ data.put("ä¸ æ–‡", "Apache Asia");
+ String configStr = SerializationUtils.objectToString(data);
+ Assert.assertNotNull(configStr);
+
+ HashMap<String, String> dataAfter = SerializationUtils.stringToObject(configStr);
+
+ Assert.assertEquals(dataAfter, data);
+
+ data.put("key2", "");
+ Assert.assertNotEquals(dataAfter, data);
+
+ }
+
+ @Test
+ public void testByteToObject() {
+
+ HashMap<String, String> data = new HashMap<>();
+ data.put("key1", "value1");
+ data.put("seatunnelTest", "apache SeaTunnel");
+ data.put("ä¸ æ–‡", "Apache Asia");
+
+ ArrayList<HashMap<String, String>> array = new ArrayList<>();
+ array.add(data);
+ HashMap<String, String> data2 = new HashMap<>();
+ data2.put("Apache Asia", "ä¸ æ–‡");
+ data2.put("value1", "key1");
+ data2.put("apache SeaTunnel", "seatunnelTest");
+ array.add(data2);
+
+ byte[] result = SerializationUtils.serialize(array);
+
+ ArrayList<HashMap<String, String>> array2 = SerializationUtils.deserialize(result);
+
+ Assert.assertEquals(array2, array);
+
+ Assert.assertThrows(SerializationException.class, () -> SerializationUtils.deserialize(new byte[]{1, 0, 1}));
+
+ }
+
+}
diff --git a/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/VariablesSubstituteTest.java b/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/VariablesSubstituteTest.java
index 33d721ae8..09d044a53 100644
--- a/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/VariablesSubstituteTest.java
+++ b/seatunnel-common/src/test/java/org/apache/seatunnel/common/utils/VariablesSubstituteTest.java
@@ -26,7 +26,6 @@ import java.util.HashMap;
public class VariablesSubstituteTest {
- @SuppressWarnings("checkstyle:MagicNumber")
@Test
public void testSubstitute() {
String timeFormat = "yyyyMMddHHmmss";
diff --git a/seatunnel-connectors/plugin-mapping.properties b/seatunnel-connectors/plugin-mapping.properties
index f4bfbf954..c2dfffc3b 100644
--- a/seatunnel-connectors/plugin-mapping.properties
+++ b/seatunnel-connectors/plugin-mapping.properties
@@ -18,6 +18,7 @@
# This mapping is used to resolve the Jar package name without version (or call artifactId)
# corresponding to the module in the user Config, helping SeaTunnel to load the correct Jar package.
+
# Flink Source
flink.source.DruidSource = seatunnel-connector-flink-druid
flink.source.FakeSource = seatunnel-connector-flink-fake
@@ -47,6 +48,7 @@ flink.sink.AssertSink = seatunnel-connector-flink-assert
spark.source.ElasticSearch = seatunnel-connector-spark-elasticsearch
spark.source.Fake = seatunnel-connector-spark-fake
+spark.source.FakeStream = seatunnel-connector-spark-fake
spark.source.FeishuSheet = seatunnel-connector-spark-feishu
spark.source.File = seatunnel-connector-spark-file
spark.source.Hbase = seatunnel-connector-spark-hbase
diff --git a/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-clickhouse/src/main/java/org/apache/seatunnel/flink/clickhouse/sink/ClickhouseOutputFormat.java b/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-clickhouse/src/main/java/org/apache/seatunnel/flink/clickhouse/sink/ClickhouseOutputFormat.java
index cb392b9a0..efd4eb7d5 100644
--- a/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-clickhouse/src/main/java/org/apache/seatunnel/flink/clickhouse/sink/ClickhouseOutputFormat.java
+++ b/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-clickhouse/src/main/java/org/apache/seatunnel/flink/clickhouse/sink/ClickhouseOutputFormat.java
@@ -224,7 +224,7 @@ public class ClickhouseOutputFormat extends RichOutputFormat<Row> {
break;
}
}
- result.put(field, function);
+ result.put(fieldType, function);
}
return result;
}
diff --git a/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-fake/src/main/java/org/apache/seatunnel/flink/fake/Config.java b/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-fake/src/main/java/org/apache/seatunnel/flink/fake/Config.java
index 79a55e192..50316d5b4 100644
--- a/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-fake/src/main/java/org/apache/seatunnel/flink/fake/Config.java
+++ b/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-fake/src/main/java/org/apache/seatunnel/flink/fake/Config.java
@@ -143,7 +143,7 @@ public final class Config {
/**
* mock_data_size Default value.
*/
- public static final int MOCK_DATA_SIZE_DEFAULT_VALUE = 300;
+ public static final int MOCK_DATA_SIZE_DEFAULT_VALUE = 10;
/**
* Create data interval, unit is second. Default is 1 second.
diff --git a/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-fake/src/main/java/org/apache/seatunnel/flink/fake/source/FakeSourceStream.java b/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-fake/src/main/java/org/apache/seatunnel/flink/fake/source/FakeSourceStream.java
index be99fcb50..dfec9da49 100644
--- a/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-fake/src/main/java/org/apache/seatunnel/flink/fake/source/FakeSourceStream.java
+++ b/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-fake/src/main/java/org/apache/seatunnel/flink/fake/source/FakeSourceStream.java
@@ -84,7 +84,7 @@ public class FakeSourceStream extends RichParallelSourceFunction<Row> implements
while (running){
Row rowData = MockSchema.mockRowData(mockDataSchema);
ctx.collect(rowData);
- TimeUnit.MILLISECONDS.sleep(mockDataInterval);
+ TimeUnit.SECONDS.sleep(mockDataInterval);
}
}
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/jdbc/source/Jdbc.scala b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/jdbc/source/Jdbc.scala
index df3d8c983..6a31beefd 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/jdbc/source/Jdbc.scala
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/jdbc/source/Jdbc.scala
@@ -22,7 +22,7 @@ import org.apache.seatunnel.common.config.{CheckResult, TypesafeConfigUtils}
import org.apache.seatunnel.common.config.CheckConfigUtil.checkAllExists
import org.apache.seatunnel.spark.SparkEnvironment
import org.apache.seatunnel.spark.batch.SparkBatchSource
-import org.apache.seatunnel.spark.jdbc.source.util.HiveDialet
+import org.apache.seatunnel.spark.jdbc.source.util.HiveDialect
import org.apache.spark.sql.jdbc.JdbcDialects
import org.apache.spark.sql.{DataFrameReader, Dataset, Row, SparkSession}
@@ -60,7 +60,7 @@ class Jdbc extends SparkBatchSource {
}
if (config.getString("url").startsWith("jdbc:hive2")) {
- JdbcDialects.registerDialect(new HiveDialet)
+ JdbcDialects.registerDialect(new HiveDialect)
}
reader
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/jdbc/source/util/HiveDialet.scala b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/jdbc/source/util/HiveDialet.scala
index c686cdfba..81eb48bf6 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/jdbc/source/util/HiveDialet.scala
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-jdbc/src/main/scala/org/apache/seatunnel/spark/jdbc/source/util/HiveDialet.scala
@@ -18,7 +18,7 @@ package org.apache.seatunnel.spark.jdbc.source.util
import org.apache.spark.sql.jdbc.JdbcDialect
-class HiveDialet extends JdbcDialect {
+class HiveDialect extends JdbcDialect {
override def canHandle(url: String): Boolean = {
url.startsWith("jdbc:hive2")
}
diff --git a/seatunnel-core/README.md b/seatunnel-core/README.md
new file mode 100644
index 000000000..c4b48e4cb
--- /dev/null
+++ b/seatunnel-core/README.md
@@ -0,0 +1,8 @@
+# Introduction
+
+This module is the seatunnel job entrypoint. Seatunnel jobs are started by the below process.
+
+
+- seatunnel-core-flink: The flink job starter.
+- seatunnel-core-flink-sql: The flink sql job starter.
+- seatunnel-core-spark: The spark job starter.
\ No newline at end of file
diff --git a/seatunnel-core/pom.xml b/seatunnel-core/pom.xml
index 26247952d..0e634a460 100644
--- a/seatunnel-core/pom.xml
+++ b/seatunnel-core/pom.xml
@@ -17,7 +17,9 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel</artifactId>
diff --git a/seatunnel-core/seatunnel-core-base/pom.xml b/seatunnel-core/seatunnel-core-base/pom.xml
index c58536a8f..40bddc281 100644
--- a/seatunnel-core/seatunnel-core-base/pom.xml
+++ b/seatunnel-core/seatunnel-core-base/pom.xml
@@ -17,7 +17,9 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-core</artifactId>
@@ -48,6 +50,12 @@
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-plugin-discovery</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
@@ -58,4 +66,5 @@
<artifactId>junit</artifactId>
</dependency>
</dependencies>
+
</project>
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/Seatunnel.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/Seatunnel.java
index 644df716b..e45872848 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/Seatunnel.java
+++ b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/Seatunnel.java
@@ -20,6 +20,7 @@ package org.apache.seatunnel.core.base;
import org.apache.seatunnel.apis.base.command.CommandArgs;
import org.apache.seatunnel.common.config.ConfigRuntimeException;
import org.apache.seatunnel.core.base.command.Command;
+import org.apache.seatunnel.core.base.exception.CommandException;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.Logger;
@@ -34,7 +35,7 @@ public class Seatunnel {
* @param command commandArgs
* @param <T> commandType
*/
- public static <T extends CommandArgs> void run(Command<T> command) {
+ public static <T extends CommandArgs> void run(Command<T> command) throws CommandException {
try {
command.execute();
} catch (ConfigRuntimeException e) {
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/AbstractCommandArgs.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/AbstractCommandArgs.java
index 7f46fb033..9042b2c14 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/AbstractCommandArgs.java
+++ b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/AbstractCommandArgs.java
@@ -37,7 +37,8 @@ public abstract class AbstractCommandArgs implements CommandArgs {
description = "variable substitution, such as -i city=beijing, or -i date=20190318")
private List<String> variables = Collections.emptyList();
- @Parameter(names = {"-t", "--check"},
+ // todo: use command type enum
+ @Parameter(names = {"-ck", "--check"},
description = "check config")
private boolean checkConfig = false;
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/BaseTaskExecuteCommand.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/BaseTaskExecuteCommand.java
index bab835e2b..b14b6fa96 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/BaseTaskExecuteCommand.java
+++ b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/BaseTaskExecuteCommand.java
@@ -33,7 +33,6 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.List;
import java.util.Objects;
-import java.util.Optional;
/**
* Base task execute command.
@@ -126,8 +125,8 @@ public abstract class BaseTaskExecuteCommand<T extends AbstractCommandArgs, E ex
}
private void deployModeCheck() {
- final Optional<String> mode = Common.getDeployMode();
- if (mode.isPresent() && DeployMode.CLUSTER.getName().equals(mode.get())) {
+ final DeployMode mode = Common.getDeployMode();
+ if (DeployMode.CLUSTER == mode) {
LOGGER.info("preparing cluster mode work dir files...");
File workDir = new File(".");
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
index 5cd80fc53..cca16c30f 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
+++ b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/command/Command.java
@@ -18,6 +18,7 @@
package org.apache.seatunnel.core.base.command;
import org.apache.seatunnel.apis.base.command.CommandArgs;
+import org.apache.seatunnel.core.base.exception.CommandException;
/**
* Command interface.
@@ -30,6 +31,6 @@ public interface Command<T extends CommandArgs> {
/**
* Execute command
*/
- void execute();
+ void execute() throws CommandException;
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ExecutionContext.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/AbstractExecutionContext.java
similarity index 60%
rename from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ExecutionContext.java
rename to seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/AbstractExecutionContext.java
index 7e91a3689..92d4232c3 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ExecutionContext.java
+++ b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/AbstractExecutionContext.java
@@ -22,37 +22,36 @@ import org.apache.seatunnel.apis.base.api.BaseSource;
import org.apache.seatunnel.apis.base.api.BaseTransform;
import org.apache.seatunnel.apis.base.env.RuntimeEnv;
import org.apache.seatunnel.common.constants.JobMode;
+import org.apache.seatunnel.common.constants.PluginType;
+import org.apache.seatunnel.plugin.discovery.PluginIdentifier;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import java.net.URL;
+import java.util.Arrays;
import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
/**
* The ExecutionContext contains all configuration needed to run the job.
*
* @param <ENVIRONMENT> environment type.
*/
-public class ExecutionContext<ENVIRONMENT extends RuntimeEnv> {
+public abstract class AbstractExecutionContext<ENVIRONMENT extends RuntimeEnv> {
private final Config config;
private final EngineType engine;
private final ENVIRONMENT environment;
private final JobMode jobMode;
- private final List<BaseSource<ENVIRONMENT>> sources;
- private final List<BaseTransform<ENVIRONMENT>> transforms;
- private final List<BaseSink<ENVIRONMENT>> sinks;
- public ExecutionContext(Config config, EngineType engine) {
+ public AbstractExecutionContext(Config config, EngineType engine) {
this.config = config;
this.engine = engine;
this.environment = new EnvironmentFactory<ENVIRONMENT>(config, engine).getEnvironment();
this.jobMode = environment.getJobMode();
- PluginFactory<ENVIRONMENT> pluginFactory = new PluginFactory<>(config, engine);
- this.environment.registerPlugin(pluginFactory.getPluginJarPaths());
- this.sources = pluginFactory.createPlugins(PluginType.SOURCE);
- this.transforms = pluginFactory.createPlugins(PluginType.TRANSFORM);
- this.sinks = pluginFactory.createPlugins(PluginType.SINK);
}
public Config getRootConfig() {
@@ -71,15 +70,23 @@ public class ExecutionContext<ENVIRONMENT extends RuntimeEnv> {
return jobMode;
}
- public List<BaseSource<ENVIRONMENT>> getSources() {
- return sources;
- }
+ public abstract List<BaseSource<ENVIRONMENT>> getSources();
- public List<BaseTransform<ENVIRONMENT>> getTransforms() {
- return transforms;
- }
+ public abstract List<BaseTransform<ENVIRONMENT>> getTransforms();
+
+ public abstract List<BaseSink<ENVIRONMENT>> getSinks();
+
+ public abstract List<URL> getPluginJars();
- public List<BaseSink<ENVIRONMENT>> getSinks() {
- return sinks;
+ @SuppressWarnings("checkstyle:Indentation")
+ protected List<PluginIdentifier> getPluginIdentifiers(PluginType... pluginTypes) {
+ return Arrays.stream(pluginTypes).flatMap((Function<PluginType, Stream<PluginIdentifier>>) pluginType -> {
+ List<? extends Config> configList = config.getConfigList(pluginType.getType());
+ return configList.stream()
+ .map(pluginConfig -> PluginIdentifier
+ .of(engine.getEngine(),
+ pluginType.getType(),
+ pluginConfig.getString("plugin_name")));
+ }).collect(Collectors.toList());
}
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ConfigBuilder.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ConfigBuilder.java
index 605613b07..5f80ae407 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ConfigBuilder.java
+++ b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ConfigBuilder.java
@@ -17,7 +17,6 @@
package org.apache.seatunnel.core.base.config;
-import org.apache.seatunnel.apis.base.env.RuntimeEnv;
import org.apache.seatunnel.common.config.ConfigRuntimeException;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
@@ -33,20 +32,16 @@ import java.nio.file.Path;
/**
* Used to build the {@link Config} from file.
*
- * @param <ENVIRONMENT> environment type.
*/
-public class ConfigBuilder<ENVIRONMENT extends RuntimeEnv> {
+public class ConfigBuilder {
private static final Logger LOGGER = LoggerFactory.getLogger(ConfigBuilder.class);
- private static final String PLUGIN_NAME_KEY = "plugin_name";
private final Path configFile;
- private final EngineType engine;
private final Config config;
- public ConfigBuilder(Path configFile, EngineType engine) {
+ public ConfigBuilder(Path configFile) {
this.configFile = configFile;
- this.engine = engine;
this.config = load();
}
@@ -75,16 +70,4 @@ public class ConfigBuilder<ENVIRONMENT extends RuntimeEnv> {
return config;
}
- /**
- * check if config is valid.
- **/
- public void checkConfig() {
- // check environment
- ENVIRONMENT environment = new EnvironmentFactory<ENVIRONMENT>(config, engine).getEnvironment();
- // check plugins
- PluginFactory<ENVIRONMENT> pluginFactory = new PluginFactory<>(config, engine);
- pluginFactory.createPlugins(PluginType.SOURCE);
- pluginFactory.createPlugins(PluginType.TRANSFORM);
- pluginFactory.createPlugins(PluginType.SINK);
- }
}
diff --git a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ConfigChecker.java
similarity index 63%
copy from seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
copy to seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ConfigChecker.java
index f0cf54447..30e4243e7 100644
--- a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
+++ b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ConfigChecker.java
@@ -15,33 +15,25 @@
* limitations under the License.
*/
-package org.apache.seatunnel.apis.base.env;
+package org.apache.seatunnel.core.base.config;
-import org.apache.seatunnel.common.config.CheckResult;
-import org.apache.seatunnel.common.constants.JobMode;
+import org.apache.seatunnel.apis.base.env.RuntimeEnv;
+import org.apache.seatunnel.core.base.exception.ConfigCheckException;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
-import java.net.URL;
-import java.util.Set;
-
/**
- * engine related runtime environment
+ * Check the config is valid.
+ *
+ * @param <ENVIRONMENT> the environment type.
*/
-public interface RuntimeEnv {
-
- RuntimeEnv setConfig(Config config);
-
- Config getConfig();
-
- CheckResult checkConfig();
-
- RuntimeEnv prepare();
-
- RuntimeEnv setJobMode(JobMode mode);
-
- JobMode getJobMode();
-
- void registerPlugin(Set<URL> pluginPaths);
+public interface ConfigChecker<ENVIRONMENT extends RuntimeEnv> {
+
+ /**
+ * Check if the config is validated, if check fails, throw exception.
+ *
+ * @param config given config.
+ */
+ void checkConfig(Config config) throws ConfigCheckException;
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/EnvironmentFactory.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/EnvironmentFactory.java
index ff450836f..0386159cc 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/EnvironmentFactory.java
+++ b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/EnvironmentFactory.java
@@ -19,6 +19,7 @@ package org.apache.seatunnel.core.base.config;
import org.apache.seatunnel.apis.base.env.RuntimeEnv;
import org.apache.seatunnel.common.constants.JobMode;
+import org.apache.seatunnel.common.constants.PluginType;
import org.apache.seatunnel.flink.FlinkEnvironment;
import org.apache.seatunnel.spark.SparkEnvironment;
@@ -43,6 +44,7 @@ public class EnvironmentFactory<ENVIRONMENT extends RuntimeEnv> {
this.engine = engine;
}
+ // todo:put this method into submodule to avoid dependency on the engine
public synchronized ENVIRONMENT getEnvironment() {
Config envConfig = config.getConfig("env");
boolean enableHive = checkIsContainHive();
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ExecutionFactory.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ExecutionFactory.java
index 62b807d13..0c4fd178c 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ExecutionFactory.java
+++ b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/ExecutionFactory.java
@@ -42,9 +42,9 @@ public class ExecutionFactory<ENVIRONMENT extends RuntimeEnv> {
private static final Logger LOGGER = LoggerFactory.getLogger(ExecutionFactory.class);
- public ExecutionContext<ENVIRONMENT> executionContext;
+ public AbstractExecutionContext<ENVIRONMENT> executionContext;
- public ExecutionFactory(ExecutionContext<ENVIRONMENT> executionContext) {
+ public ExecutionFactory(AbstractExecutionContext<ENVIRONMENT> executionContext) {
this.executionContext = executionContext;
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginFactory.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginFactory.java
deleted file mode 100644
index 0dbdb0ef1..000000000
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginFactory.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.seatunnel.core.base.config;
-
-import org.apache.seatunnel.apis.base.env.RuntimeEnv;
-import org.apache.seatunnel.apis.base.plugin.Plugin;
-import org.apache.seatunnel.common.config.Common;
-import org.apache.seatunnel.flink.BaseFlinkSink;
-import org.apache.seatunnel.flink.BaseFlinkSource;
-import org.apache.seatunnel.flink.BaseFlinkTransform;
-import org.apache.seatunnel.spark.BaseSparkSink;
-import org.apache.seatunnel.spark.BaseSparkSource;
-import org.apache.seatunnel.spark.BaseSparkTransform;
-
-import org.apache.seatunnel.shade.com.typesafe.config.Config;
-import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
-import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions;
-
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nonnull;
-
-import java.io.File;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.ServiceConfigurationError;
-import java.util.ServiceLoader;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * Used to load the plugins.
- *
- * @param <ENVIRONMENT> environment
- */
-public class PluginFactory<ENVIRONMENT extends RuntimeEnv> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PluginFactory.class);
- private final Config config;
- private final EngineType engineType;
- private static final Map<EngineType, Map<PluginType, Class<?>>> PLUGIN_BASE_CLASS_MAP;
-
- private static final String PLUGIN_NAME_KEY = "plugin_name";
- private static final String PLUGIN_MAPPING_FILE = "plugin-mapping.properties";
-
- private final Set<URL> pluginJarPaths;
- private final ClassLoader defaultClassLoader;
-
- static {
- PLUGIN_BASE_CLASS_MAP = new HashMap<>();
- Map<PluginType, Class<?>> sparkBaseClassMap = new HashMap<>();
- sparkBaseClassMap.put(PluginType.SOURCE, BaseSparkSource.class);
- sparkBaseClassMap.put(PluginType.TRANSFORM, BaseSparkTransform.class);
- sparkBaseClassMap.put(PluginType.SINK, BaseSparkSink.class);
- PLUGIN_BASE_CLASS_MAP.put(EngineType.SPARK, sparkBaseClassMap);
-
- Map<PluginType, Class<?>> flinkBaseClassMap = new HashMap<>();
- flinkBaseClassMap.put(PluginType.SOURCE, BaseFlinkSource.class);
- flinkBaseClassMap.put(PluginType.TRANSFORM, BaseFlinkTransform.class);
- flinkBaseClassMap.put(PluginType.SINK, BaseFlinkSink.class);
- PLUGIN_BASE_CLASS_MAP.put(EngineType.FLINK, flinkBaseClassMap);
- }
-
- public PluginFactory(Config config, EngineType engineType) {
- this.config = config;
- this.engineType = engineType;
- this.pluginJarPaths = searchPluginJar();
- this.defaultClassLoader = initClassLoaderWithPaths(this.pluginJarPaths);
- }
-
- private ClassLoader initClassLoaderWithPaths(Set<URL> pluginJarPaths) {
- return new URLClassLoader(pluginJarPaths.toArray(new URL[0]),
- Thread.currentThread().getContextClassLoader());
- }
-
- @Nonnull
- private Set<URL> searchPluginJar() {
-
- File pluginDir = Common.connectorJarDir(this.engineType.getEngine()).toFile();
- if (!pluginDir.exists() || pluginDir.listFiles() == null) {
- return new HashSet<>();
- }
- Config pluginMapping = ConfigFactory
- .parseFile(new File(getPluginMappingPath()))
- .resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
- .resolveWith(ConfigFactory.systemProperties(),
- ConfigResolveOptions.defaults().setAllowUnresolved(true));
- File[] plugins =
- Arrays.stream(pluginDir.listFiles()).filter(f -> f.getName().endsWith(".jar")).toArray(File[]::new);
-
- return Arrays.stream(PluginType.values()).filter(type -> !PluginType.TRANSFORM.equals(type))
- .flatMap(type -> {
- List<URL> pluginList = new ArrayList<>();
- List<? extends Config> configList = config.getConfigList(type.getType());
- configList.forEach(pluginConfig -> {
- Optional<String> mappingValue = getPluginMappingValue(pluginMapping, type,
- pluginConfig.getString(PLUGIN_NAME_KEY));
- if (mappingValue.isPresent()) {
- try {
- for (File plugin : plugins) {
- if (plugin.getName().startsWith(mappingValue.get())) {
- pluginList.add(plugin.toURI().toURL());
- break;
- }
- }
- } catch (MalformedURLException e) {
- LOGGER.warn("can get plugin url", e);
- }
- } else {
- throw new IllegalArgumentException(String.format("can't find connector %s in " +
- "%s. If you add connector to connectors dictionary, please modify this " +
- "file.", getPluginMappingKey(type, pluginConfig.getString(PLUGIN_NAME_KEY)),
- getPluginMappingPath()));
- }
-
- });
- return pluginList.stream();
- }).collect(Collectors.toSet());
- }
-
- public Set<URL> getPluginJarPaths() {
- return this.pluginJarPaths;
- }
-
- private String getPluginMappingPath() {
- return Common.connectorDir() + "/" + PLUGIN_MAPPING_FILE;
- }
-
- private String getPluginMappingKey(PluginType type, String pluginName) {
- return this.engineType.getEngine() + "." + type.getType() + "." + pluginName;
-
- }
-
- Optional<String> getPluginMappingValue(Config pluginMapping, PluginType type, String pluginName) {
-
- return pluginMapping.getConfig(this.engineType.getEngine()).getConfig(type.getType()).entrySet()
- .stream().filter(entry -> entry.getKey().equalsIgnoreCase(pluginName))
- .map(entry -> entry.getValue().unwrapped().toString()).findAny();
-
- }
-
- /**
- * Create the plugins by plugin type.
- *
- * @param type plugin type
- * @param <T> plugin
- * @return plugin list.
- */
- @SuppressWarnings("unchecked")
- public <T extends Plugin<ENVIRONMENT>> List<T> createPlugins(PluginType type) {
- Objects.requireNonNull(type, "PluginType can not be null when create plugins!");
- List<T> basePluginList = new ArrayList<>();
- List<? extends Config> configList = config.getConfigList(type.getType());
- configList.forEach(plugin -> {
- try {
- T t = (T) createPluginInstanceIgnoreCase(type, plugin.getString(PLUGIN_NAME_KEY), this.defaultClassLoader);
- t.setConfig(plugin);
- basePluginList.add(t);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- });
-
- return basePluginList;
- }
-
- /**
- * create plugin class instance, ignore case.
- **/
- @SuppressWarnings("unchecked")
- private Plugin<?> createPluginInstanceIgnoreCase(PluginType pluginType, String pluginName,
- ClassLoader classLoader) throws Exception {
- Class<Plugin<?>> pluginBaseClass = (Class<Plugin<?>>) getPluginBaseClass(engineType, pluginType);
-
- if (pluginName.split("\\.").length != 1) {
- // canonical class name
- Class<Plugin<?>> pluginClass = (Class<Plugin<?>>) Class.forName(pluginName);
- if (pluginClass.isAssignableFrom(pluginBaseClass)) {
- throw new IllegalArgumentException("plugin: " + pluginName + " is not extends from " + pluginBaseClass);
- }
- return pluginClass.getDeclaredConstructor().newInstance();
- }
- ServiceLoader<Plugin<?>> plugins = ServiceLoader.load(pluginBaseClass, classLoader);
- for (Iterator<Plugin<?>> it = plugins.iterator(); it.hasNext(); ) {
- try {
- Plugin<?> plugin = it.next();
- if (StringUtils.equalsIgnoreCase(plugin.getPluginName(), pluginName)) {
- return plugin;
- }
- } catch (ServiceConfigurationError e) {
- // Iterator.next() may throw ServiceConfigurationError,
- // but maybe caused by a not used plugin in this job
- LOGGER.warn("Error when load plugin: [{}]", pluginName, e);
- }
- }
- throw new ClassNotFoundException("Plugin class not found by name :[" + pluginName + "]");
- }
-
- private Class<?> getPluginBaseClass(EngineType engineType, PluginType pluginType) {
- if (!PLUGIN_BASE_CLASS_MAP.containsKey(engineType)) {
- throw new IllegalStateException("PluginType not support : [" + pluginType + "]");
- }
- Map<PluginType, Class<?>> pluginTypeClassMap = PLUGIN_BASE_CLASS_MAP.get(engineType);
- if (!pluginTypeClassMap.containsKey(pluginType)) {
- throw new IllegalStateException(pluginType + " is not supported in engine " + engineType);
- }
- return pluginTypeClassMap.get(pluginType);
- }
-
-}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/exception/CommandException.java
similarity index 75%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/exception/CommandException.java
index f5f308154..33f2013b2 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/exception/CommandException.java
@@ -15,18 +15,14 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.core.base.exception;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
-
- private final String type;
-
- PluginType(String type) {
- this.type = type;
+public class CommandException extends Exception {
+ public CommandException(String message) {
+ super(message);
}
- public String getType() {
- return type;
+ public CommandException(String message, Throwable cause) {
+ super(message, cause);
}
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/exception/CommandExecuteException.java
similarity index 73%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/exception/CommandExecuteException.java
index f5f308154..a3a8ee421 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/exception/CommandExecuteException.java
@@ -15,18 +15,14 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.core.base.exception;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
-
- private final String type;
-
- PluginType(String type) {
- this.type = type;
+public class CommandExecuteException extends CommandException {
+ public CommandExecuteException(String message) {
+ super(message);
}
- public String getType() {
- return type;
+ public CommandExecuteException(String message, Throwable cause) {
+ super(message, cause);
}
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/exception/ConfigCheckException.java
similarity index 74%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/exception/ConfigCheckException.java
index f5f308154..22d193bf5 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/exception/ConfigCheckException.java
@@ -15,18 +15,16 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.core.base.exception;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+public class ConfigCheckException extends CommandException {
- private final String type;
-
- PluginType(String type) {
- this.type = type;
+ public ConfigCheckException(String message) {
+ super(message);
}
- public String getType() {
- return type;
+ public ConfigCheckException(String message, Throwable cause) {
+ super(message, cause);
}
+
}
diff --git a/seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/core/base/config/PluginFactoryTest.java b/seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/core/base/config/PluginFactoryTest.java
deleted file mode 100644
index 4ba315e58..000000000
--- a/seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/core/base/config/PluginFactoryTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.seatunnel.core.base.config;
-
-import org.apache.seatunnel.common.config.Common;
-import org.apache.seatunnel.spark.SparkEnvironment;
-
-import org.apache.seatunnel.shade.com.typesafe.config.Config;
-import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
-import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-import java.nio.file.Paths;
-import java.util.Objects;
-import java.util.Optional;
-
-public class PluginFactoryTest {
-
- @Test
- public void getPluginMappingValueTest() throws Exception {
-
- Common.setDeployMode("cluster");
- Config config = new ConfigBuilder<>(Paths.get(Objects.requireNonNull(ClassLoader.getSystemResource("flink.batch" +
- ".conf")).toURI()), EngineType.SPARK).getConfig();
-
- Config pluginMapping = ConfigFactory
- .parseFile(new File(Objects.requireNonNull(ClassLoader.getSystemResource("plugin-mapping.properties")).toURI()))
- .resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
- .resolveWith(ConfigFactory.systemProperties(),
- ConfigResolveOptions.defaults().setAllowUnresolved(true));
-
- PluginFactory<SparkEnvironment> factory = new PluginFactory<>(config, EngineType.SPARK);
-
- Object jarPrefix = factory.getPluginMappingValue(pluginMapping, PluginType.SOURCE, "fake");
- Assert.assertEquals(jarPrefix, Optional.of("seatunnel-connector-spark-fake"));
-
- Object jarPrefix2 = factory.getPluginMappingValue(pluginMapping, PluginType.SINK, "console");
- Assert.assertEquals(jarPrefix2, Optional.of("seatunnel-connector-spark-console"));
-
- Object jarPrefix3 = factory.getPluginMappingValue(pluginMapping, PluginType.SOURCE, "FaKE");
- Assert.assertEquals(jarPrefix3, Optional.of("seatunnel-connector-spark-fake"));
-
- Object jarPrefix4 = factory.getPluginMappingValue(pluginMapping, PluginType.SINK, "HbASe");
- Assert.assertEquals(jarPrefix4, Optional.of("seatunnel-connector-spark-hbase"));
- }
-
-}
diff --git a/seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/core/base/utils/FileUtilsTest.java b/seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/core/base/utils/FileUtilsTest.java
index 16fbd39e7..86e64d0bb 100644
--- a/seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/core/base/utils/FileUtilsTest.java
+++ b/seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/core/base/utils/FileUtilsTest.java
@@ -50,6 +50,7 @@ public class FileUtilsTest {
this.deployMode = deployMode;
}
+ @Override
public DeployMode getDeployMode() {
return deployMode;
}
diff --git a/seatunnel-core/seatunnel-core-flink-sql/pom.xml b/seatunnel-core/seatunnel-core-flink-sql/pom.xml
index dd51e342e..60d5b2477 100644
--- a/seatunnel-core/seatunnel-core-flink-sql/pom.xml
+++ b/seatunnel-core/seatunnel-core-flink-sql/pom.xml
@@ -17,7 +17,9 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-core</artifactId>
diff --git a/seatunnel-core/seatunnel-core-flink-sql/src/main/java/org/apache/seatunnel/core/sql/FlinkSqlStarter.java b/seatunnel-core/seatunnel-core-flink-sql/src/main/java/org/apache/seatunnel/core/sql/FlinkSqlStarter.java
index 313b966c4..141a2fd92 100644
--- a/seatunnel-core/seatunnel-core-flink-sql/src/main/java/org/apache/seatunnel/core/sql/FlinkSqlStarter.java
+++ b/seatunnel-core/seatunnel-core-flink-sql/src/main/java/org/apache/seatunnel/core/sql/FlinkSqlStarter.java
@@ -39,17 +39,18 @@ public class FlinkSqlStarter implements Starter {
FlinkSqlStarter(String[] args) {
this.flinkCommandArgs = CommandLineUtils.parseCommandArgs(args, FlinkJobType.SQL);
// set the deployment mode, used to get the job jar path.
- Common.setDeployMode(flinkCommandArgs.getDeployMode().getName());
+ Common.setStarter(true);
+ Common.setDeployMode(flinkCommandArgs.getDeployMode());
this.appJar = Common.appLibDir().resolve(APP_JAR_NAME).toString();
}
@Override
- public List<String> buildCommands() throws Exception {
+ public List<String> buildCommands() {
return CommandLineUtils.buildFlinkCommand(flinkCommandArgs, CLASS_NAME, appJar);
}
@SuppressWarnings("checkstyle:RegexpSingleline")
- public static void main(String[] args) throws Exception {
+ public static void main(String[] args) {
FlinkSqlStarter flinkSqlStarter = new FlinkSqlStarter(args);
System.out.println(String.join(" ", flinkSqlStarter.buildCommands()));
}
diff --git a/seatunnel-core/seatunnel-core-flink-sql/src/main/java/org/apache/seatunnel/core/sql/job/Executor.java b/seatunnel-core/seatunnel-core-flink-sql/src/main/java/org/apache/seatunnel/core/sql/job/Executor.java
index 2320e31b9..044262ee8 100644
--- a/seatunnel-core/seatunnel-core-flink-sql/src/main/java/org/apache/seatunnel/core/sql/job/Executor.java
+++ b/seatunnel-core/seatunnel-core-flink-sql/src/main/java/org/apache/seatunnel/core/sql/job/Executor.java
@@ -44,7 +44,6 @@ import java.io.File;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@@ -137,21 +136,16 @@ public class Executor {
* Refer https://github.com/apache/incubator-seatunnel/pull/1850
*/
private static void loadConnector(String connectorType, Configuration configuration) {
- Iterator<Factory> factories = ServiceLoader.load(Factory.class, CLASSLOADER).iterator();
- while (factories.hasNext()) {
- Factory factory = factories.next();
-
- /**
- * Handle for two cases:
- * 1. Flink built-in connectors.
- * 2. Connectors have been placed in classpath.
- */
+ // Handle for two cases:
+ // 1. Flink built-in connectors.
+ // 2. Connectors have been placed in classpath.
+ for (Factory factory : ServiceLoader.load(Factory.class, CLASSLOADER)) {
if (factory.factoryIdentifier().equals(connectorType)) {
return;
}
}
- Common.setDeployMode(DeployMode.CLIENT.getName());
+ Common.setDeployMode(DeployMode.CLIENT);
File connectorDir = Common.connectorJarDir(SQL_CONNECTOR_PREFIX).toFile();
if (!connectorDir.exists() || connectorDir.listFiles() == null) {
return;
diff --git a/seatunnel-core/seatunnel-core-flink/pom.xml b/seatunnel-core/seatunnel-core-flink/pom.xml
index ea9939571..75ab06207 100644
--- a/seatunnel-core/seatunnel-core-flink/pom.xml
+++ b/seatunnel-core/seatunnel-core-flink/pom.xml
@@ -17,7 +17,9 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-core</artifactId>
@@ -40,6 +42,25 @@
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-api-flink</artifactId>
@@ -82,11 +103,6 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <scope>test</scope>
- </dependency>
</dependencies>
diff --git a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/FlinkStarter.java b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/FlinkStarter.java
index f6ba80712..cd4a2be7a 100644
--- a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/FlinkStarter.java
+++ b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/FlinkStarter.java
@@ -46,12 +46,13 @@ public class FlinkStarter implements Starter {
FlinkStarter(String[] args) {
this.flinkCommandArgs = CommandLineUtils.parseCommandArgs(args, FlinkJobType.JAR);
// set the deployment mode, used to get the job jar path.
- Common.setDeployMode(flinkCommandArgs.getDeployMode().getName());
+ Common.setDeployMode(flinkCommandArgs.getDeployMode());
+ Common.setStarter(true);
this.appJar = Common.appLibDir().resolve(APP_JAR_NAME).toString();
}
@SuppressWarnings("checkstyle:RegexpSingleline")
- public static void main(String[] args) throws Exception {
+ public static void main(String[] args) {
FlinkStarter flinkStarter = new FlinkStarter(args);
System.out.println(String.join(" ", flinkStarter.buildCommands()));
}
diff --git a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/SeatunnelFlink.java b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/SeatunnelFlink.java
index 45792d004..f1db4a15a 100644
--- a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/SeatunnelFlink.java
+++ b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/SeatunnelFlink.java
@@ -19,6 +19,7 @@ package org.apache.seatunnel.core.flink;
import org.apache.seatunnel.core.base.Seatunnel;
import org.apache.seatunnel.core.base.command.Command;
+import org.apache.seatunnel.core.base.exception.CommandException;
import org.apache.seatunnel.core.flink.args.FlinkCommandArgs;
import org.apache.seatunnel.core.flink.command.FlinkCommandBuilder;
import org.apache.seatunnel.core.flink.config.FlinkJobType;
@@ -26,7 +27,7 @@ import org.apache.seatunnel.core.flink.utils.CommandLineUtils;
public class SeatunnelFlink {
- public static void main(String[] args) {
+ public static void main(String[] args) throws CommandException {
FlinkCommandArgs flinkCommandArgs = CommandLineUtils.parseCommandArgs(args, FlinkJobType.JAR);
Command<FlinkCommandArgs> flinkCommand = new FlinkCommandBuilder()
.buildCommand(flinkCommandArgs);
diff --git a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/args/FlinkCommandArgs.java b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/args/FlinkCommandArgs.java
index fb892f9cd..f6221c803 100644
--- a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/args/FlinkCommandArgs.java
+++ b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/args/FlinkCommandArgs.java
@@ -30,8 +30,8 @@ import java.util.List;
public class FlinkCommandArgs extends AbstractCommandArgs {
@Parameter(names = {"-r", "--run-mode"},
- converter = RunModeConverter.class,
- description = "job run mode, run or run-application")
+ converter = RunModeConverter.class,
+ description = "job run mode, run or run-application")
private FlinkRunMode runMode = FlinkRunMode.RUN;
/**
diff --git a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkConfValidateCommand.java b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkApiConfValidateCommand.java
similarity index 74%
rename from seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkConfValidateCommand.java
rename to seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkApiConfValidateCommand.java
index 5868e2522..f03522a04 100644
--- a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkConfValidateCommand.java
+++ b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkApiConfValidateCommand.java
@@ -19,9 +19,10 @@ package org.apache.seatunnel.core.flink.command;
import org.apache.seatunnel.core.base.command.Command;
import org.apache.seatunnel.core.base.config.ConfigBuilder;
+import org.apache.seatunnel.core.base.exception.ConfigCheckException;
import org.apache.seatunnel.core.base.utils.FileUtils;
import org.apache.seatunnel.core.flink.args.FlinkCommandArgs;
-import org.apache.seatunnel.flink.FlinkEnvironment;
+import org.apache.seatunnel.core.flink.config.FlinkApiConfigChecker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,20 +32,21 @@ import java.nio.file.Path;
/**
* Used to check the Flink conf is validated.
*/
-public class FlinkConfValidateCommand implements Command<FlinkCommandArgs> {
+public class FlinkApiConfValidateCommand implements Command<FlinkCommandArgs> {
- private static final Logger LOGGER = LoggerFactory.getLogger(FlinkConfValidateCommand.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(FlinkApiConfValidateCommand.class);
private final FlinkCommandArgs flinkCommandArgs;
- public FlinkConfValidateCommand(FlinkCommandArgs flinkCommandArgs) {
+ public FlinkApiConfValidateCommand(FlinkCommandArgs flinkCommandArgs) {
this.flinkCommandArgs = flinkCommandArgs;
}
@Override
- public void execute() {
+ public void execute() throws ConfigCheckException {
Path configPath = FileUtils.getConfigPath(flinkCommandArgs);
- new ConfigBuilder<FlinkEnvironment>(configPath, flinkCommandArgs.getEngineType()).checkConfig();
+ ConfigBuilder configBuilder = new ConfigBuilder(configPath);
+ new FlinkApiConfigChecker().checkConfig(configBuilder.getConfig());
LOGGER.info("config OK !");
}
}
diff --git a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkTaskExecuteCommand.java b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkApiTaskExecuteCommand.java
similarity index 75%
rename from seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkTaskExecuteCommand.java
rename to seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkApiTaskExecuteCommand.java
index 92ce52088..f5f0cdf8c 100644
--- a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkTaskExecuteCommand.java
+++ b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkApiTaskExecuteCommand.java
@@ -26,10 +26,11 @@ import org.apache.seatunnel.common.constants.JobMode;
import org.apache.seatunnel.core.base.command.BaseTaskExecuteCommand;
import org.apache.seatunnel.core.base.config.ConfigBuilder;
import org.apache.seatunnel.core.base.config.EngineType;
-import org.apache.seatunnel.core.base.config.ExecutionContext;
import org.apache.seatunnel.core.base.config.ExecutionFactory;
+import org.apache.seatunnel.core.base.exception.CommandExecuteException;
import org.apache.seatunnel.core.base.utils.FileUtils;
import org.apache.seatunnel.core.flink.args.FlinkCommandArgs;
+import org.apache.seatunnel.core.flink.config.FlinkExecutionContext;
import org.apache.seatunnel.flink.FlinkEnvironment;
import org.apache.seatunnel.flink.batch.FlinkBatchSink;
import org.apache.seatunnel.flink.batch.FlinkBatchSource;
@@ -48,40 +49,40 @@ import java.util.List;
import java.util.stream.Stream;
/**
- * Used to execute Flink Job.
+ * Used to execute Flink Job by Flink API.
*/
-public class FlinkTaskExecuteCommand extends BaseTaskExecuteCommand<FlinkCommandArgs, FlinkEnvironment> {
+public class FlinkApiTaskExecuteCommand extends BaseTaskExecuteCommand<FlinkCommandArgs, FlinkEnvironment> {
private final FlinkCommandArgs flinkCommandArgs;
- public FlinkTaskExecuteCommand(FlinkCommandArgs flinkCommandArgs) {
+ public FlinkApiTaskExecuteCommand(FlinkCommandArgs flinkCommandArgs) {
this.flinkCommandArgs = flinkCommandArgs;
}
@Override
- public void execute() {
+ public void execute() throws CommandExecuteException {
EngineType engine = flinkCommandArgs.getEngineType();
Path configFile = FileUtils.getConfigPath(flinkCommandArgs);
- Config config = new ConfigBuilder<>(configFile, engine).getConfig();
- ExecutionContext<FlinkEnvironment> executionContext = new ExecutionContext<>(config, engine);
+ Config config = new ConfigBuilder(configFile).getConfig();
+ FlinkExecutionContext executionContext = new FlinkExecutionContext(config, engine);
List<BaseSource<FlinkEnvironment>> sources = executionContext.getSources();
List<BaseTransform<FlinkEnvironment>> transforms = executionContext.getTransforms();
List<BaseSink<FlinkEnvironment>> sinks = executionContext.getSinks();
checkPluginType(executionContext.getJobMode(), sources, transforms, sinks);
- baseCheckConfig(sources, transforms, sinks);
+ baseCheckConfig(sinks, transforms, sinks);
showAsciiLogo();
try (Execution<BaseSource<FlinkEnvironment>,
- BaseTransform<FlinkEnvironment>,
- BaseSink<FlinkEnvironment>,
- FlinkEnvironment> execution = new ExecutionFactory<>(executionContext).createExecution()) {
+ BaseTransform<FlinkEnvironment>,
+ BaseSink<FlinkEnvironment>,
+ FlinkEnvironment> execution = new ExecutionFactory<>(executionContext).createExecution()) {
prepare(executionContext.getEnvironment(), sources, transforms, sinks);
execution.start(sources, transforms, sinks);
close(sources, transforms, sinks);
} catch (Exception e) {
- throw new RuntimeException("Execute Flink task error", e);
+ throw new CommandExecuteException("Execute Flink task error", e);
}
}
@@ -92,9 +93,9 @@ public class FlinkTaskExecuteCommand extends BaseTaskExecuteCommand<FlinkCommand
switch (jobMode) {
case STREAMING:
pluginStream.forEach(plugin -> {
- boolean isStream = (plugin instanceof FlinkStreamSource)
- || (plugin instanceof FlinkStreamTransform)
- || (plugin instanceof FlinkStreamSink);
+ boolean isStream = plugin instanceof FlinkStreamSource
+ || plugin instanceof FlinkStreamTransform
+ || plugin instanceof FlinkStreamSink;
if (!isStream) {
throw new IllegalArgumentException(String.format("Cannot use batch plugin: %s in stream mode", plugin.getPluginName()));
}
@@ -102,9 +103,9 @@ public class FlinkTaskExecuteCommand extends BaseTaskExecuteCommand<FlinkCommand
break;
case BATCH:
pluginStream.forEach(plugin -> {
- boolean isBatch = (plugin instanceof FlinkBatchSource)
- || (plugin instanceof FlinkBatchTransform)
- || (plugin instanceof FlinkBatchSink);
+ boolean isBatch = plugin instanceof FlinkBatchSource
+ || plugin instanceof FlinkBatchTransform
+ || plugin instanceof FlinkBatchSink;
if (!isBatch) {
throw new IllegalArgumentException(String.format("Cannot use stream plugin: %s in batch mode", plugin.getPluginName()));
}
diff --git a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkCommandBuilder.java b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkCommandBuilder.java
index 595f01a54..723689606 100644
--- a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkCommandBuilder.java
+++ b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/command/FlinkCommandBuilder.java
@@ -26,11 +26,8 @@ public class FlinkCommandBuilder implements CommandBuilder<FlinkCommandArgs> {
@Override
public Command<FlinkCommandArgs> buildCommand(FlinkCommandArgs commandArgs) {
- if (!Common.setDeployMode(commandArgs.getDeployMode().getName())) {
- throw new IllegalArgumentException(
- String.format("Deploy mode: %s is Illegal", commandArgs.getDeployMode()));
- }
- return commandArgs.isCheckConfig() ? new FlinkConfValidateCommand(commandArgs)
- : new FlinkTaskExecuteCommand(commandArgs);
+ Common.setDeployMode(commandArgs.getDeployMode());
+ return commandArgs.isCheckConfig() ? new FlinkApiConfValidateCommand(commandArgs)
+ : new FlinkApiTaskExecuteCommand(commandArgs);
}
}
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/config/FlinkApiConfigChecker.java
similarity index 50%
copy from seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java
copy to seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/config/FlinkApiConfigChecker.java
index e9052bfe6..ee82bc260 100644
--- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java
+++ b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/config/FlinkApiConfigChecker.java
@@ -15,28 +15,27 @@
* limitations under the License.
*/
-package org.apache.seatunnel.common;
+package org.apache.seatunnel.core.flink.config;
-import org.apache.seatunnel.shade.com.typesafe.config.Config;
-
-import java.util.Properties;
+import org.apache.seatunnel.core.base.config.ConfigChecker;
+import org.apache.seatunnel.core.base.config.EngineType;
+import org.apache.seatunnel.core.base.exception.ConfigCheckException;
+import org.apache.seatunnel.flink.FlinkEnvironment;
-public final class PropertiesUtil {
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
- private PropertiesUtil() {
- }
+public class FlinkApiConfigChecker implements ConfigChecker<FlinkEnvironment> {
- public static void setProperties(Config config, Properties properties, String prefix, boolean keepPrefix) {
- config.entrySet().forEach(entry -> {
- String key = entry.getKey();
- Object value = entry.getValue().unwrapped();
- if (key.startsWith(prefix)) {
- if (keepPrefix) {
- properties.put(key, value);
- } else {
- properties.put(key.substring(prefix.length()), value);
- }
- }
- });
+ @Override
+ public void checkConfig(Config config) throws ConfigCheckException {
+ try {
+ // check plugins
+ FlinkExecutionContext flinkExecutionContext = new FlinkExecutionContext(config, EngineType.FLINK);
+ flinkExecutionContext.getSources();
+ flinkExecutionContext.getTransforms();
+ flinkExecutionContext.getSinks();
+ } catch (Exception ex) {
+ throw new ConfigCheckException("Config check fail", ex);
+ }
}
}
diff --git a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/config/FlinkExecutionContext.java b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/config/FlinkExecutionContext.java
new file mode 100644
index 000000000..2acb83699
--- /dev/null
+++ b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/config/FlinkExecutionContext.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.core.flink.config;
+
+import org.apache.seatunnel.apis.base.api.BaseSink;
+import org.apache.seatunnel.apis.base.api.BaseSource;
+import org.apache.seatunnel.apis.base.api.BaseTransform;
+import org.apache.seatunnel.common.constants.PluginType;
+import org.apache.seatunnel.core.base.config.AbstractExecutionContext;
+import org.apache.seatunnel.core.base.config.EngineType;
+import org.apache.seatunnel.flink.FlinkEnvironment;
+import org.apache.seatunnel.plugin.discovery.PluginIdentifier;
+import org.apache.seatunnel.plugin.discovery.flink.FlinkSinkPluginDiscovery;
+import org.apache.seatunnel.plugin.discovery.flink.FlinkSourcePluginDiscovery;
+import org.apache.seatunnel.plugin.discovery.flink.FlinkTransformPluginDiscovery;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class FlinkExecutionContext extends AbstractExecutionContext<FlinkEnvironment> {
+ private final FlinkSourcePluginDiscovery flinkSourcePluginDiscovery;
+ private final FlinkTransformPluginDiscovery flinkTransformPluginDiscovery;
+ private final FlinkSinkPluginDiscovery flinkSinkPluginDiscovery;
+ private final List<URL> pluginJars;
+
+ public FlinkExecutionContext(Config config, EngineType engine) {
+ super(config, engine);
+ this.flinkSourcePluginDiscovery = new FlinkSourcePluginDiscovery();
+ this.flinkTransformPluginDiscovery = new FlinkTransformPluginDiscovery();
+ this.flinkSinkPluginDiscovery = new FlinkSinkPluginDiscovery();
+ Set<URL> pluginJars = new HashSet<>();
+ // since we didn't split the transform plugin jars, we just need to register the source/sink plugin jars
+ pluginJars.addAll(flinkSourcePluginDiscovery.getPluginJarPaths(getPluginIdentifiers(PluginType.SOURCE)));
+ pluginJars.addAll(flinkSinkPluginDiscovery.getPluginJarPaths(getPluginIdentifiers(PluginType.SINK)));
+ this.pluginJars = new ArrayList<>(pluginJars);
+ this.getEnvironment().registerPlugin(this.pluginJars);
+ }
+
+ @Override
+ public List<BaseSource<FlinkEnvironment>> getSources() {
+ final String pluginType = PluginType.SOURCE.getType();
+ final String engineType = EngineType.FLINK.getEngine();
+ final List<? extends Config> configList = getRootConfig().getConfigList(pluginType);
+ return configList.stream()
+ .map(pluginConfig -> {
+ PluginIdentifier pluginIdentifier = PluginIdentifier.of(engineType, pluginType, pluginConfig.getString("plugin_name"));
+ BaseSource<FlinkEnvironment> pluginInstance = flinkSourcePluginDiscovery.createPluginInstance(pluginIdentifier);
+ pluginInstance.setConfig(pluginConfig);
+ return pluginInstance;
+ }).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<BaseTransform<FlinkEnvironment>> getTransforms() {
+ final String pluginType = PluginType.TRANSFORM.getType();
+ final String engineType = EngineType.FLINK.getEngine();
+ final List<? extends Config> configList = getRootConfig().getConfigList(pluginType);
+ return configList.stream()
+ .map(pluginConfig -> {
+ PluginIdentifier pluginIdentifier = PluginIdentifier.of(engineType, pluginType, pluginConfig.getString("plugin_name"));
+ BaseTransform<FlinkEnvironment> pluginInstance = flinkTransformPluginDiscovery.createPluginInstance(pluginIdentifier);
+ pluginInstance.setConfig(pluginConfig);
+ return pluginInstance;
+ }).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<BaseSink<FlinkEnvironment>> getSinks() {
+ final String pluginType = PluginType.SINK.getType();
+ final String engineType = EngineType.FLINK.getEngine();
+ final List<? extends Config> configList = getRootConfig().getConfigList(pluginType);
+ return configList.stream()
+ .map(pluginConfig -> {
+ PluginIdentifier pluginIdentifier = PluginIdentifier.of(engineType, pluginType, pluginConfig.getString("plugin_name"));
+ BaseSink<FlinkEnvironment> pluginInstance = flinkSinkPluginDiscovery.createPluginInstance(pluginIdentifier);
+ pluginInstance.setConfig(pluginConfig);
+ return pluginInstance;
+ }).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<URL> getPluginJars() {
+ return pluginJars;
+ }
+}
diff --git a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/utils/CommandLineUtils.java b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/utils/CommandLineUtils.java
index 51ba3c819..cdbf610ae 100644
--- a/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/utils/CommandLineUtils.java
+++ b/seatunnel-core/seatunnel-core-flink/src/main/java/org/apache/seatunnel/core/flink/utils/CommandLineUtils.java
@@ -27,6 +27,7 @@ import com.beust.jcommander.UnixStyleUsageFormatter;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
public class CommandLineUtils {
@@ -34,6 +35,15 @@ public class CommandLineUtils {
throw new UnsupportedOperationException("CommandLineUtils is a utility class and cannot be instantiated");
}
+ public static FlinkCommandArgs parseFlinkArgs(String[] args) {
+ FlinkCommandArgs flinkCommandArgs = new FlinkCommandArgs();
+ JCommander.newBuilder()
+ .addObject(flinkCommandArgs)
+ .build()
+ .parse(args);
+ return flinkCommandArgs;
+ }
+
public static FlinkCommandArgs parseCommandArgs(String[] args, FlinkJobType jobType) {
FlinkCommandArgs flinkCommandArgs = new FlinkCommandArgs();
JCommander jCommander = JCommander.newBuilder()
@@ -66,7 +76,12 @@ public class CommandLineUtils {
if (flinkCommandArgs.isCheckConfig()) {
command.add("--check");
}
-
+ // set System properties
+ flinkCommandArgs.getVariables().stream()
+ .filter(Objects::nonNull)
+ .map(String::trim)
+ .forEach(variable -> command.add("-D" + variable));
return command;
+
}
}
diff --git a/seatunnel-core/seatunnel-core-flink/src/test/java/org/apache/seatunnel/core/flink/args/FlinkCommandArgsTest.java b/seatunnel-core/seatunnel-core-flink/src/test/java/org/apache/seatunnel/core/flink/args/FlinkCommandArgsTest.java
index 56f4f5e63..a4dba1b27 100644
--- a/seatunnel-core/seatunnel-core-flink/src/test/java/org/apache/seatunnel/core/flink/args/FlinkCommandArgsTest.java
+++ b/seatunnel-core/seatunnel-core-flink/src/test/java/org/apache/seatunnel/core/flink/args/FlinkCommandArgsTest.java
@@ -27,7 +27,7 @@ public class FlinkCommandArgsTest {
@Test
public void testParseFlinkArgs() {
- String[] args = {"-c", "app.conf", "-t", "-i", "city=shenyang", "-i", "date=20200202"};
+ String[] args = {"-c", "app.conf", "-ck", "-i", "city=shenyang", "-i", "date=20200202"};
FlinkCommandArgs flinkArgs = new FlinkCommandArgs();
JCommander.newBuilder()
.addObject(flinkArgs)
diff --git a/seatunnel-core/seatunnel-core-flink/src/test/java/org/apache/seatunnel/core/flink/command/FlinkTaskExecuteCommandTest.java b/seatunnel-core/seatunnel-core-flink/src/test/java/org/apache/seatunnel/core/flink/command/FlinkTaskExecuteCommandTest.java
index fc8b3edfc..aa3db5c16 100644
--- a/seatunnel-core/seatunnel-core-flink/src/test/java/org/apache/seatunnel/core/flink/command/FlinkTaskExecuteCommandTest.java
+++ b/seatunnel-core/seatunnel-core-flink/src/test/java/org/apache/seatunnel/core/flink/command/FlinkTaskExecuteCommandTest.java
@@ -36,7 +36,7 @@ public class FlinkTaskExecuteCommandTest {
@Test
public void checkPluginType() {
List<MockBatchSource> sources = Lists.newArrayList(new MockBatchSource());
- FlinkTaskExecuteCommand flinkTaskExecuteCommand = new FlinkTaskExecuteCommand(null);
+ FlinkApiTaskExecuteCommand flinkTaskExecuteCommand = new FlinkApiTaskExecuteCommand(null);
// check success
flinkTaskExecuteCommand.checkPluginType(JobMode.BATCH, sources);
Assert.assertThrows("checkPluginType should throw IllegalException", IllegalArgumentException.class, () -> {
diff --git a/seatunnel-core/seatunnel-core-flink/src/test/java/org/apache/seatunnel/core/flink/utils/CommandLineUtilsTest.java b/seatunnel-core/seatunnel-core-flink/src/test/java/org/apache/seatunnel/core/flink/utils/CommandLineUtilsTest.java
index 96b71371f..2befcdbe8 100644
--- a/seatunnel-core/seatunnel-core-flink/src/test/java/org/apache/seatunnel/core/flink/utils/CommandLineUtilsTest.java
+++ b/seatunnel-core/seatunnel-core-flink/src/test/java/org/apache/seatunnel/core/flink/utils/CommandLineUtilsTest.java
@@ -34,14 +34,14 @@ public class CommandLineUtilsTest {
@Test
public void testParseCommandArgs() {
- String[] args = {"--detached", "-c", "app.conf", "-t", "-i", "city=shenyang", "-i", "date=20200202",
+ String[] args = {"--detached", "-c", "app.conf", "-ck", "-i", "city=shenyang", "-i", "date=20200202",
"-r", "run-application", "--unkown", "unkown-command"};
FlinkCommandArgs flinkCommandArgs = CommandLineUtils.parseCommandArgs(args, FlinkJobType.JAR);
Assert.assertEquals(flinkCommandArgs.getFlinkParams(), Arrays.asList("--detached", "--unkown", "unkown-command"));
Assert.assertEquals(flinkCommandArgs.getRunMode(), FlinkRunMode.APPLICATION_RUN);
Assert.assertEquals(flinkCommandArgs.getVariables(), Arrays.asList("city=shenyang", "date=20200202"));
- String[] args1 = {"--detached", "-c", "app.conf", "-t", "-i", "city=shenyang", "-i", "date=20200202",
+ String[] args1 = {"--detached", "-c", "app.conf", "-ck", "-i", "city=shenyang", "-i", "date=20200202",
"-r", "run-application", "--unkown", "unkown-command"};
flinkCommandArgs = CommandLineUtils.parseCommandArgs(args1, FlinkJobType.SQL);
Assert.assertEquals(flinkCommandArgs.getFlinkParams(), Arrays.asList("--detached", "--unkown", "unkown-command"));
@@ -51,40 +51,33 @@ public class CommandLineUtilsTest {
@Test
public void testBuildFlinkJarCommand() throws FileNotFoundException {
- String[] args = {"--detached", "-c", APP_CONF_PATH, "-t", "-i", "city=shenyang", "-i", "date=20200202",
+ String[] args = {"--detached", "-c", APP_CONF_PATH, "-ck", "-i", "city=shenyang", "-i", "date=20200202",
"-r", "run-application", "--unkown", "unkown-command"};
FlinkCommandArgs flinkCommandArgs = CommandLineUtils.parseCommandArgs(args, FlinkJobType.JAR);
List<String> commands = CommandLineUtils.buildFlinkCommand(flinkCommandArgs, "CLASS_NAME", "/path/to/jar");
Assert.assertEquals(commands,
- Arrays.asList("${FLINK_HOME}/bin/flink", "run-application", "--detached", "--unkown", "unkown-command", "-c",
- "CLASS_NAME", "/path/to/jar", "--config", APP_CONF_PATH, "--check"));
+ Arrays.asList("${FLINK_HOME}/bin/flink", "run-application", "--detached", "--unkown", "unkown-command", "-c",
+ "CLASS_NAME", "/path/to/jar", "--config", APP_CONF_PATH, "--check", "-Dcity=shenyang", "-Ddate=20200202"));
flinkCommandArgs = CommandLineUtils.parseCommandArgs(args, FlinkJobType.JAR);
commands = CommandLineUtils.buildFlinkCommand(flinkCommandArgs, "CLASS_NAME", "/path/to/jar");
Assert.assertEquals(commands,
Arrays.asList("${FLINK_HOME}/bin/flink", "run-application", "--detached", "--unkown", "unkown-command", "-c",
- "CLASS_NAME", "/path/to/jar", "--config", APP_CONF_PATH, "--check"));
+ "CLASS_NAME", "/path/to/jar", "--config", APP_CONF_PATH, "--check", "-Dcity=shenyang", "-Ddate=20200202"));
- String[] args1 = {"--detached", "-c", "app.conf", "-t", "-i", "city=shenyang", "-i", "date=20200202",
+ String[] args1 = {"--detached", "-c", "app.conf", "-ck", "-i", "city=shenyang", "-i", "date=20200202",
"-r", "run-application", "--unkown", "unkown-command"};
- List<String> command = CommandLineUtils.buildFlinkCommand(
- CommandLineUtils.parseCommandArgs(args1, FlinkJobType.SQL), "CLASS_NAME", "/path/to/jar");
- Assert.assertEquals(
- Arrays.asList("${FLINK_HOME}/bin/flink", "run-application", "--detached", "--unkown", "unkown-command", "-c",
- "CLASS_NAME", "/path/to/jar", "--config", "app.conf", "--check"),
- command);
-
}
@Test
public void testBuildFlinkSQLCommand() throws FileNotFoundException{
- String[] args = {"--detached", "-c", SQL_CONF_PATH, "-t", "-i", "city=shenyang", "-i", "date=20200202",
+ String[] args = {"--detached", "-c", SQL_CONF_PATH, "-ck", "-i", "city=shenyang", "-i", "date=20200202",
"-r", "run-application", "--unkown", "unkown-command"};
FlinkCommandArgs flinkCommandArgs = CommandLineUtils.parseCommandArgs(args, FlinkJobType.SQL);
List<String> commands = CommandLineUtils.buildFlinkCommand(flinkCommandArgs, "CLASS_NAME", "/path/to/jar");
Assert.assertEquals(commands,
- Arrays.asList("${FLINK_HOME}/bin/flink", "run-application", "--detached", "--unkown", "unkown-command", "-c",
- "CLASS_NAME", "/path/to/jar", "--config", SQL_CONF_PATH, "--check"));
+ Arrays.asList("${FLINK_HOME}/bin/flink", "run-application", "--detached", "--unkown", "unkown-command", "-c",
+ "CLASS_NAME", "/path/to/jar", "--config", SQL_CONF_PATH, "--check", "-Dcity=shenyang", "-Ddate=20200202"));
}
}
diff --git a/seatunnel-core/seatunnel-core-spark/pom.xml b/seatunnel-core/seatunnel-core-spark/pom.xml
index e173fc67d..ebeeab6ef 100644
--- a/seatunnel-core/seatunnel-core-spark/pom.xml
+++ b/seatunnel-core/seatunnel-core-spark/pom.xml
@@ -17,7 +17,9 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-core</artifactId>
@@ -32,6 +34,13 @@
</properties>
<dependencies>
+
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-core-base</artifactId>
@@ -44,6 +53,19 @@
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ </dependency>
+
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
diff --git a/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/SeatunnelSpark.java b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/SeatunnelSpark.java
index d88cc7eca..05a1a1d1e 100644
--- a/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/SeatunnelSpark.java
+++ b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/SeatunnelSpark.java
@@ -19,13 +19,14 @@ package org.apache.seatunnel.core.spark;
import org.apache.seatunnel.core.base.Seatunnel;
import org.apache.seatunnel.core.base.command.Command;
+import org.apache.seatunnel.core.base.exception.CommandException;
import org.apache.seatunnel.core.spark.args.SparkCommandArgs;
import org.apache.seatunnel.core.spark.command.SparkCommandBuilder;
import org.apache.seatunnel.core.spark.utils.CommandLineUtils;
public class SeatunnelSpark {
- public static void main(String[] args) {
+ public static void main(String[] args) throws CommandException {
SparkCommandArgs sparkArgs = CommandLineUtils.parseSparkArgs(args);
Command<SparkCommandArgs> sparkCommand =
new SparkCommandBuilder().buildCommand(sparkArgs);
diff --git a/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/SparkStarter.java b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/SparkStarter.java
index 2f7ca546f..7eca89b4b 100644
--- a/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/SparkStarter.java
+++ b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/SparkStarter.java
@@ -19,17 +19,19 @@ package org.apache.seatunnel.core.spark;
import static java.nio.file.FileVisitOption.FOLLOW_LINKS;
-import org.apache.seatunnel.apis.base.env.RuntimeEnv;
import org.apache.seatunnel.common.Constants;
import org.apache.seatunnel.common.config.Common;
import org.apache.seatunnel.common.config.DeployMode;
+import org.apache.seatunnel.common.constants.PluginType;
import org.apache.seatunnel.core.base.Starter;
import org.apache.seatunnel.core.base.config.ConfigBuilder;
import org.apache.seatunnel.core.base.config.ConfigParser;
import org.apache.seatunnel.core.base.config.EngineType;
-import org.apache.seatunnel.core.base.config.PluginFactory;
import org.apache.seatunnel.core.base.utils.CompressionUtils;
import org.apache.seatunnel.core.spark.args.SparkCommandArgs;
+import org.apache.seatunnel.plugin.discovery.PluginIdentifier;
+import org.apache.seatunnel.plugin.discovery.spark.SparkSinkPluginDiscovery;
+import org.apache.seatunnel.plugin.discovery.spark.SparkSourcePluginDiscovery;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
@@ -40,6 +42,7 @@ import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -47,9 +50,12 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -143,9 +149,9 @@ public class SparkStarter implements Starter {
@Override
public List<String> buildCommands() throws IOException {
setSparkConf();
- Common.setDeployMode(commandArgs.getDeployMode().getName());
+ Common.setDeployMode(commandArgs.getDeployMode());
+ Common.setStarter(true);
this.jars.addAll(getPluginsJarDependencies());
- this.jars.addAll(listJars(Common.appLibDir()));
this.jars.addAll(getConnectorJarDependencies());
this.appName = this.sparkConf.getOrDefault("spark.app.name", Constants.LOGO);
return buildFinal();
@@ -204,25 +210,17 @@ public class SparkStarter implements Starter {
* return connector's jars, which located in 'connectors/spark/*'.
*/
private List<Path> getConnectorJarDependencies() {
- Path pluginRootDir = Common.connectorJarDir("SPARK");
+ Path pluginRootDir = Common.connectorJarDir("spark");
if (!Files.exists(pluginRootDir) || !Files.isDirectory(pluginRootDir)) {
return Collections.emptyList();
}
- Config config = new ConfigBuilder<>(Paths.get(commandArgs.getConfigFile()), EngineType.SPARK).getConfig();
- PluginFactory<RuntimeEnv> pluginFactory = new PluginFactory<>(config, EngineType.SPARK);
- return pluginFactory.getPluginJarPaths().stream().map(url -> new File(url.getPath()).toPath()).collect(Collectors.toList());
- }
-
- /**
- * list jars in given directory
- */
- private List<Path> listJars(Path dir) throws IOException {
- try (Stream<Path> stream = Files.list(dir)) {
- return stream
- .filter(it -> !Files.isDirectory(it))
- .filter(it -> it.getFileName().endsWith("jar"))
- .collect(Collectors.toList());
- }
+ Config config = new ConfigBuilder(Paths.get(commandArgs.getConfigFile())).getConfig();
+ Set<URL> pluginJars = new HashSet<>();
+ SparkSourcePluginDiscovery sparkSourcePluginDiscovery = new SparkSourcePluginDiscovery();
+ SparkSinkPluginDiscovery sparkSinkPluginDiscovery = new SparkSinkPluginDiscovery();
+ pluginJars.addAll(sparkSourcePluginDiscovery.getPluginJarPaths(getPluginIdentifiers(config, PluginType.SOURCE)));
+ pluginJars.addAll(sparkSinkPluginDiscovery.getPluginJarPaths(getPluginIdentifiers(config, PluginType.SINK)));
+ return pluginJars.stream().map(url -> new File(url.getPath()).toPath()).collect(Collectors.toList());
}
/**
@@ -302,6 +300,18 @@ public class SparkStarter implements Starter {
commands.add(Common.appLibDir().resolve("seatunnel-core-spark.jar").toString());
}
+ @SuppressWarnings("checkstyle:Indentation")
+ private List<PluginIdentifier> getPluginIdentifiers(Config config, PluginType... pluginTypes) {
+ return Arrays.stream(pluginTypes).flatMap((Function<PluginType, Stream<PluginIdentifier>>) pluginType -> {
+ List<? extends Config> configList = config.getConfigList(pluginType.getType());
+ return configList.stream()
+ .map(pluginConfig -> PluginIdentifier
+ .of(EngineType.SPARK.getEngine(),
+ pluginType.getType(),
+ pluginConfig.getString("plugin_name")));
+ }).collect(Collectors.toList());
+ }
+
/**
* a Starter for building spark-submit commands with client mode options
*/
@@ -384,11 +394,10 @@ public class SparkStarter implements Starter {
@Override
public List<String> buildCommands() throws IOException {
- Common.setDeployMode(commandArgs.getDeployMode().getName());
+ Common.setDeployMode(commandArgs.getDeployMode());
+ Common.setStarter(true);
Path pluginTarball = Common.pluginTarball();
- if (Files.notExists(pluginTarball)) {
- CompressionUtils.tarGzip(Common.pluginRootDir(), pluginTarball);
- }
+ CompressionUtils.tarGzip(Common.pluginRootDir(), pluginTarball);
this.files.add(pluginTarball);
this.files.add(Paths.get(commandArgs.getConfigFile()));
return super.buildCommands();
diff --git a/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/command/SparkCommandBuilder.java b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/command/SparkCommandBuilder.java
index 4ea8ed1eb..bb8739e6f 100644
--- a/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/command/SparkCommandBuilder.java
+++ b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/command/SparkCommandBuilder.java
@@ -26,10 +26,7 @@ public class SparkCommandBuilder implements CommandBuilder<SparkCommandArgs> {
@Override
public Command<SparkCommandArgs> buildCommand(SparkCommandArgs commandArgs) {
- if (!Common.setDeployMode(commandArgs.getDeployMode().getName())) {
- throw new IllegalArgumentException(
- String.format("Deploy mode: %s is Illegal", commandArgs.getDeployMode()));
- }
+ Common.setDeployMode(commandArgs.getDeployMode());
return commandArgs.isCheckConfig() ? new SparkConfValidateCommand(commandArgs)
: new SparkTaskExecuteCommand(commandArgs);
}
diff --git a/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/command/SparkConfValidateCommand.java b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/command/SparkConfValidateCommand.java
index 5629e51d1..0f4a93077 100644
--- a/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/command/SparkConfValidateCommand.java
+++ b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/command/SparkConfValidateCommand.java
@@ -19,9 +19,10 @@ package org.apache.seatunnel.core.spark.command;
import org.apache.seatunnel.core.base.command.Command;
import org.apache.seatunnel.core.base.config.ConfigBuilder;
+import org.apache.seatunnel.core.base.exception.ConfigCheckException;
import org.apache.seatunnel.core.base.utils.FileUtils;
import org.apache.seatunnel.core.spark.args.SparkCommandArgs;
-import org.apache.seatunnel.spark.SparkEnvironment;
+import org.apache.seatunnel.core.spark.config.SparkApiConfigChecker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,9 +43,10 @@ public class SparkConfValidateCommand implements Command<SparkCommandArgs> {
}
@Override
- public void execute() {
+ public void execute() throws ConfigCheckException {
Path confPath = FileUtils.getConfigPath(sparkCommandArgs);
- new ConfigBuilder<SparkEnvironment>(confPath, sparkCommandArgs.getEngineType()).checkConfig();
+ ConfigBuilder configBuilder = new ConfigBuilder(confPath);
+ new SparkApiConfigChecker().checkConfig(configBuilder.getConfig());
LOGGER.info("config OK !");
}
}
diff --git a/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/command/SparkTaskExecuteCommand.java b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/command/SparkTaskExecuteCommand.java
index 08eb4f2e9..e9c54b441 100644
--- a/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/command/SparkTaskExecuteCommand.java
+++ b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/command/SparkTaskExecuteCommand.java
@@ -21,29 +21,20 @@ import org.apache.seatunnel.apis.base.api.BaseSink;
import org.apache.seatunnel.apis.base.api.BaseSource;
import org.apache.seatunnel.apis.base.api.BaseTransform;
import org.apache.seatunnel.apis.base.env.Execution;
-import org.apache.seatunnel.apis.base.plugin.Plugin;
-import org.apache.seatunnel.common.constants.JobMode;
import org.apache.seatunnel.core.base.command.BaseTaskExecuteCommand;
import org.apache.seatunnel.core.base.config.ConfigBuilder;
import org.apache.seatunnel.core.base.config.EngineType;
-import org.apache.seatunnel.core.base.config.ExecutionContext;
import org.apache.seatunnel.core.base.config.ExecutionFactory;
+import org.apache.seatunnel.core.base.exception.CommandExecuteException;
import org.apache.seatunnel.core.base.utils.FileUtils;
import org.apache.seatunnel.core.spark.args.SparkCommandArgs;
+import org.apache.seatunnel.core.spark.config.SparkExecutionContext;
import org.apache.seatunnel.spark.SparkEnvironment;
-import org.apache.seatunnel.spark.batch.SparkBatchSink;
-import org.apache.seatunnel.spark.batch.SparkBatchSource;
-import org.apache.seatunnel.spark.stream.SparkStreamingSink;
-import org.apache.seatunnel.spark.stream.SparkStreamingSource;
-import org.apache.seatunnel.spark.structuredstream.StructuredStreamingSink;
-import org.apache.seatunnel.spark.structuredstream.StructuredStreamingSource;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
import java.nio.file.Path;
-import java.util.Arrays;
import java.util.List;
-import java.util.stream.Stream;
public class SparkTaskExecuteCommand extends BaseTaskExecuteCommand<SparkCommandArgs, SparkEnvironment> {
@@ -54,12 +45,12 @@ public class SparkTaskExecuteCommand extends BaseTaskExecuteCommand<SparkCommand
}
@Override
- public void execute() {
+ public void execute() throws CommandExecuteException {
EngineType engine = sparkCommandArgs.getEngineType();
Path confFile = FileUtils.getConfigPath(sparkCommandArgs);
- Config config = new ConfigBuilder<>(confFile, engine).getConfig();
- ExecutionContext<SparkEnvironment> executionContext = new ExecutionContext<>(config, engine);
+ Config config = new ConfigBuilder(confFile).getConfig();
+ SparkExecutionContext executionContext = new SparkExecutionContext(config, engine);
List<BaseSource<SparkEnvironment>> sources = executionContext.getSources();
List<BaseTransform<SparkEnvironment>> transforms = executionContext.getTransforms();
@@ -76,45 +67,7 @@ public class SparkTaskExecuteCommand extends BaseTaskExecuteCommand<SparkCommand
execution.start(sources, transforms, sinks);
close(sources, transforms, sinks);
} catch (Exception e) {
- throw new RuntimeException("Execute Spark task error", e);
- }
- }
-
- private void checkPluginType(JobMode jobMode, List<? extends Plugin<?>>... plugins) {
- Stream<? extends Plugin<?>> pluginStream = Arrays.stream(plugins).flatMap(List::stream);
- switch (jobMode) {
- case STREAMING:
- pluginStream.forEach(plugin -> {
- boolean isStream = (plugin instanceof SparkStreamingSource)
- || (plugin instanceof SparkStreamingSink);
- if (!isStream) {
- throw new IllegalArgumentException(
- String.format("Current execute mode is Streaming, but %s is not Streaming plugin", plugin.getPluginName()));
- }
- });
- break;
- case BATCH:
- pluginStream.forEach(plugin -> {
- boolean isBatch = (plugin instanceof SparkBatchSource)
- || (plugin instanceof SparkBatchSink);
- if (!isBatch) {
- throw new IllegalArgumentException(
- String.format("Current execute mode is Batch, but %s is not Batch plugin", plugin.getPluginName()));
- }
- });
- break;
- case STRUCTURED_STREAMING:
- pluginStream.forEach(plugin -> {
- boolean isStructuredStreaming = (plugin instanceof StructuredStreamingSource)
- || (plugin instanceof StructuredStreamingSink);
- if (!isStructuredStreaming) {
- throw new IllegalArgumentException(
- String.format("Current execute mode is StructuredStreaming, but %s is not StructuredStreaming plugin", plugin.getPluginName()));
- }
- });
- break;
- default:
- throw new IllegalArgumentException("Unsupported job mode: " + jobMode);
+ throw new CommandExecuteException("Execute Spark task error", e);
}
}
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/config/SparkApiConfigChecker.java
similarity index 50%
copy from seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java
copy to seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/config/SparkApiConfigChecker.java
index e9052bfe6..564ee2896 100644
--- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/PropertiesUtil.java
+++ b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/config/SparkApiConfigChecker.java
@@ -15,28 +15,27 @@
* limitations under the License.
*/
-package org.apache.seatunnel.common;
+package org.apache.seatunnel.core.spark.config;
-import org.apache.seatunnel.shade.com.typesafe.config.Config;
-
-import java.util.Properties;
+import org.apache.seatunnel.core.base.config.ConfigChecker;
+import org.apache.seatunnel.core.base.config.EngineType;
+import org.apache.seatunnel.core.base.exception.ConfigCheckException;
+import org.apache.seatunnel.spark.SparkEnvironment;
-public final class PropertiesUtil {
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
- private PropertiesUtil() {
- }
+public class SparkApiConfigChecker implements ConfigChecker<SparkEnvironment> {
- public static void setProperties(Config config, Properties properties, String prefix, boolean keepPrefix) {
- config.entrySet().forEach(entry -> {
- String key = entry.getKey();
- Object value = entry.getValue().unwrapped();
- if (key.startsWith(prefix)) {
- if (keepPrefix) {
- properties.put(key, value);
- } else {
- properties.put(key.substring(prefix.length()), value);
- }
- }
- });
+ @Override
+ public void checkConfig(Config config) throws ConfigCheckException {
+ try {
+ // check plugins
+ SparkExecutionContext sparkExecutionContext = new SparkExecutionContext(config, EngineType.SPARK);
+ sparkExecutionContext.getSources();
+ sparkExecutionContext.getTransforms();
+ sparkExecutionContext.getSinks();
+ } catch (Exception ex) {
+ throw new ConfigCheckException("Config check fail", ex);
+ }
}
}
diff --git a/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/config/SparkExecutionContext.java b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/config/SparkExecutionContext.java
new file mode 100644
index 000000000..07854b83c
--- /dev/null
+++ b/seatunnel-core/seatunnel-core-spark/src/main/java/org/apache/seatunnel/core/spark/config/SparkExecutionContext.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.core.spark.config;
+
+import org.apache.seatunnel.apis.base.api.BaseSink;
+import org.apache.seatunnel.apis.base.api.BaseSource;
+import org.apache.seatunnel.apis.base.api.BaseTransform;
+import org.apache.seatunnel.common.constants.PluginType;
+import org.apache.seatunnel.core.base.config.AbstractExecutionContext;
+import org.apache.seatunnel.core.base.config.EngineType;
+import org.apache.seatunnel.plugin.discovery.PluginIdentifier;
+import org.apache.seatunnel.plugin.discovery.spark.SparkSinkPluginDiscovery;
+import org.apache.seatunnel.plugin.discovery.spark.SparkSourcePluginDiscovery;
+import org.apache.seatunnel.plugin.discovery.spark.SparkTransformPluginDiscovery;
+import org.apache.seatunnel.spark.SparkEnvironment;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class SparkExecutionContext extends AbstractExecutionContext<SparkEnvironment> {
+ private final SparkSourcePluginDiscovery sparkSourcePluginDiscovery;
+ private final SparkTransformPluginDiscovery sparkTransformPluginDiscovery;
+ private final SparkSinkPluginDiscovery sparkSinkPluginDiscovery;
+ private final List<URL> pluginJars;
+
+ public SparkExecutionContext(Config config, EngineType engine) {
+ super(config, engine);
+ this.sparkSourcePluginDiscovery = new SparkSourcePluginDiscovery();
+ this.sparkTransformPluginDiscovery = new SparkTransformPluginDiscovery();
+ this.sparkSinkPluginDiscovery = new SparkSinkPluginDiscovery();
+ Set<URL> pluginJars = new HashSet<>();
+ pluginJars.addAll(sparkSourcePluginDiscovery.getPluginJarPaths(getPluginIdentifiers(PluginType.SOURCE)));
+ pluginJars.addAll(sparkSinkPluginDiscovery.getPluginJarPaths(getPluginIdentifiers(PluginType.SINK)));
+ this.pluginJars = new ArrayList<>(pluginJars);
+ this.getEnvironment().registerPlugin(this.pluginJars);
+ }
+
+ @Override
+ public List<BaseSource<SparkEnvironment>> getSources() {
+ final String pluginType = PluginType.SOURCE.getType();
+ final String engineType = EngineType.SPARK.getEngine();
+ final List<? extends Config> configList = getRootConfig().getConfigList(pluginType);
+ return configList.stream()
+ .map(pluginConfig -> {
+ PluginIdentifier pluginIdentifier = PluginIdentifier.of(engineType, pluginType, pluginConfig.getString("plugin_name"));
+ BaseSource<SparkEnvironment> pluginInstance = sparkSourcePluginDiscovery.createPluginInstance(pluginIdentifier);
+ pluginInstance.setConfig(pluginConfig);
+ return pluginInstance;
+ }).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<BaseTransform<SparkEnvironment>> getTransforms() {
+ final String pluginType = PluginType.TRANSFORM.getType();
+ final String engineType = EngineType.SPARK.getEngine();
+ final List<? extends Config> configList = getRootConfig().getConfigList(pluginType);
+ return configList.stream()
+ .map(pluginConfig -> {
+ PluginIdentifier pluginIdentifier = PluginIdentifier.of(engineType, pluginType, pluginConfig.getString("plugin_name"));
+ BaseTransform<SparkEnvironment> pluginInstance = sparkTransformPluginDiscovery.createPluginInstance(pluginIdentifier);
+ pluginInstance.setConfig(pluginConfig);
+ return pluginInstance;
+ }).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<BaseSink<SparkEnvironment>> getSinks() {
+ final String pluginType = PluginType.SINK.getType();
+ final String engineType = EngineType.SPARK.getEngine();
+ final List<? extends Config> configList = getRootConfig().getConfigList(pluginType);
+ return configList.stream()
+ .map(pluginConfig -> {
+ PluginIdentifier pluginIdentifier = PluginIdentifier.of(engineType, pluginType, pluginConfig.getString("plugin_name"));
+ BaseSink<SparkEnvironment> pluginInstance = sparkSinkPluginDiscovery.createPluginInstance(pluginIdentifier);
+ pluginInstance.setConfig(pluginConfig);
+ return pluginInstance;
+ }).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<URL> getPluginJars() {
+ return pluginJars;
+ }
+}
diff --git a/seatunnel-dist/release-docs/LICENSE b/seatunnel-dist/release-docs/LICENSE
index f0964b6a7..f571a4768 100644
--- a/seatunnel-dist/release-docs/LICENSE
+++ b/seatunnel-dist/release-docs/LICENSE
@@ -342,12 +342,9 @@ The text of each license is the standard Apache 2.0 license.
(Apache License, Version 2.0) Apache Commons BeanUtils (commons-beanutils:commons-beanutils:1.9.3 - https://commons.apache.org/proper/commons-beanutils/)
(Apache License, Version 2.0) Apache Commons CLI (commons-cli:commons-cli:1.3.1 - http://commons.apache.org/proper/commons-cli/)
(Apache License, Version 2.0) Apache Commons CLI (commons-cli:commons-cli:1.4 - http://commons.apache.org/proper/commons-cli/)
- (Apache License, Version 2.0) Apache Commons Codec (commons-codec:commons-codec:1.10 - http://commons.apache.org/proper/commons-codec/)
- (Apache License, Version 2.0) Apache Commons Codec (commons-codec:commons-codec:1.11 - http://commons.apache.org/proper/commons-codec/)
(Apache License, Version 2.0) Apache Commons Codec (commons-codec:commons-codec:1.13 - https://commons.apache.org/proper/commons-codec/)
(Apache License, Version 2.0) Apache Commons Collections (commons-collections:commons-collections:3.2.2 - http://commons.apache.org/collections/)
- (Apache License, Version 2.0) Apache Commons Collections (org.apache.commons:commons-collections4:4.2 - http://commons.apache.org/proper/commons-collections/)
- (Apache License, Version 2.0) Apache Commons Collections (org.apache.commons:commons-collections4:4.4 - https://commons.apache.org/proper/commons-collections/)
+ (Apache License, Version 2.0) Apache Commons Collections (org.apache.commons:commons-collections4:4.4 - http://commons.apache.org/proper/commons-collections/)
(Apache License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.18 - https://commons.apache.org/proper/commons-compress/)
(Apache License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.20 - https://commons.apache.org/proper/commons-compress/)
(Apache License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.21 - https://commons.apache.org/proper/commons-compress/)
diff --git a/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/example/flink/LocalFlinkExample.java b/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/example/flink/LocalFlinkExample.java
index a740b8eea..b41236e5c 100644
--- a/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/example/flink/LocalFlinkExample.java
+++ b/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/example/flink/LocalFlinkExample.java
@@ -19,6 +19,7 @@ package org.apache.seatunnel.example.flink;
import org.apache.seatunnel.core.base.Seatunnel;
import org.apache.seatunnel.core.base.command.Command;
+import org.apache.seatunnel.core.base.exception.CommandException;
import org.apache.seatunnel.core.flink.args.FlinkCommandArgs;
import org.apache.seatunnel.core.flink.command.FlinkCommandBuilder;
@@ -29,7 +30,7 @@ import java.nio.file.Paths;
public class LocalFlinkExample {
- public static void main(String[] args) throws FileNotFoundException, URISyntaxException {
+ public static void main(String[] args) throws FileNotFoundException, URISyntaxException, CommandException {
String configFile = getTestConfigFile("/examples/fake_to_console.conf");
FlinkCommandArgs flinkCommandArgs = new FlinkCommandArgs();
flinkCommandArgs.setConfigFile(configFile);
diff --git a/seatunnel-examples/seatunnel-spark-examples/src/main/java/org/apache/seatunnel/example/spark/LocalSparkExample.java b/seatunnel-examples/seatunnel-spark-examples/src/main/java/org/apache/seatunnel/example/spark/LocalSparkExample.java
index 5469ec736..aa121a909 100644
--- a/seatunnel-examples/seatunnel-spark-examples/src/main/java/org/apache/seatunnel/example/spark/LocalSparkExample.java
+++ b/seatunnel-examples/seatunnel-spark-examples/src/main/java/org/apache/seatunnel/example/spark/LocalSparkExample.java
@@ -20,6 +20,7 @@ package org.apache.seatunnel.example.spark;
import org.apache.seatunnel.common.config.DeployMode;
import org.apache.seatunnel.core.base.Seatunnel;
import org.apache.seatunnel.core.base.command.Command;
+import org.apache.seatunnel.core.base.exception.CommandException;
import org.apache.seatunnel.core.spark.args.SparkCommandArgs;
import org.apache.seatunnel.core.spark.command.SparkCommandBuilder;
@@ -30,7 +31,7 @@ import java.nio.file.Paths;
public class LocalSparkExample {
- public static void main(String[] args) throws URISyntaxException, FileNotFoundException {
+ public static void main(String[] args) throws URISyntaxException, FileNotFoundException, CommandException {
String configFile = getTestConfigFile("/examples/spark.batch.conf");
SparkCommandArgs sparkArgs = new SparkCommandArgs();
sparkArgs.setConfigFile(configFile);
diff --git a/seatunnel-core/seatunnel-core-base/pom.xml b/seatunnel-plugin-discovery/pom.xml
similarity index 71%
copy from seatunnel-core/seatunnel-core-base/pom.xml
copy to seatunnel-plugin-discovery/pom.xml
index c58536a8f..2a1346e40 100644
--- a/seatunnel-core/seatunnel-core-base/pom.xml
+++ b/seatunnel-plugin-discovery/pom.xml
@@ -1,61 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
-
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
-
http://www.apache.org/licenses/LICENSE-2.0
-
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-core</artifactId>
+ <artifactId>seatunnel</artifactId>
<version>2.1.3-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>seatunnel-core-base</artifactId>
+ <artifactId>seatunnel-plugin-discovery</artifactId>
<dependencies>
-
<dependency>
<groupId>org.apache.seatunnel</groupId>
- <artifactId>seatunnel-api-base</artifactId>
+ <artifactId>seatunnel-api</artifactId>
<version>${project.version}</version>
+ <scope>provided</scope>
</dependency>
-
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-api-flink</artifactId>
<version>${project.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-api-spark</artifactId>
<version>${project.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-common</artifactId>
+ <version>${project.version}</version>
</dependency>
</dependencies>
-</project>
+
+</project>
\ No newline at end of file
diff --git a/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/AbstractPluginDiscovery.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/AbstractPluginDiscovery.java
new file mode 100644
index 000000000..529049455
--- /dev/null
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/AbstractPluginDiscovery.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.plugin.discovery;
+
+import org.apache.seatunnel.api.common.PluginIdentifierInterface;
+import org.apache.seatunnel.apis.base.plugin.Plugin;
+import org.apache.seatunnel.common.config.Common;
+import org.apache.seatunnel.common.utils.ReflectionUtils;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.ServiceLoader;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+
+public abstract class AbstractPluginDiscovery<T> implements PluginDiscovery<T> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(AbstractPluginDiscovery.class);
+ private final Path pluginDir;
+
+ /**
+ * Add jar url to classloader. The different engine should have different logic to add url into
+ * their own classloader
+ */
+ private BiConsumer<ClassLoader, URL> addURLToClassLoader = (classLoader, url) -> {
+ if (classLoader instanceof URLClassLoader) {
+ ReflectionUtils.invoke(classLoader, "addURL", url);
+ } else {
+ throw new UnsupportedOperationException("can't support custom load jar");
+ }
+ };
+
+ protected final ConcurrentHashMap<PluginIdentifier, Optional<URL>> pluginJarPath =
+ new ConcurrentHashMap<>(Common.COLLECTION_SIZE);
+
+ public AbstractPluginDiscovery(String pluginSubDir, BiConsumer<ClassLoader, URL> addURLToClassloader) {
+ this.pluginDir = Common.connectorJarDir(pluginSubDir);
+ this.addURLToClassLoader = addURLToClassloader;
+ LOGGER.info("Load {} Plugin from {}", getPluginBaseClass().getSimpleName(), pluginDir);
+ }
+
+ public AbstractPluginDiscovery(String pluginSubDir) {
+ this.pluginDir = Common.connectorJarDir(pluginSubDir);
+ LOGGER.info("Load {} Plugin from {}", getPluginBaseClass().getSimpleName(), pluginDir);
+ }
+
+ @Override
+ public List<URL> getPluginJarPaths(List<PluginIdentifier> pluginIdentifiers) {
+ return pluginIdentifiers.stream()
+ .map(this::getPluginJarPath)
+ .filter(Optional::isPresent)
+ .map(Optional::get).distinct()
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List<T> getAllPlugins(List<PluginIdentifier> pluginIdentifiers) {
+ return pluginIdentifiers.stream()
+ .map(this::createPluginInstance).distinct()
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public T createPluginInstance(PluginIdentifier pluginIdentifier) {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ T pluginInstance = loadPluginInstance(pluginIdentifier, classLoader);
+ if (pluginInstance != null) {
+ LOGGER.info("Load plugin: {} from classpath", pluginIdentifier);
+ return pluginInstance;
+ }
+ Optional<URL> pluginJarPath = getPluginJarPath(pluginIdentifier);
+ // if the plugin jar not exist in classpath, will load from plugin dir.
+ if (pluginJarPath.isPresent()) {
+ try {
+ // use current thread classloader to avoid different classloader load same class error.
+ this.addURLToClassLoader.accept(classLoader, pluginJarPath.get());
+ } catch (Exception e) {
+ LOGGER.warn("can't load jar use current thread classloader, use URLClassLoader instead now." +
+ " message: " + e.getMessage());
+ classLoader = new URLClassLoader(new URL[]{pluginJarPath.get()}, Thread.currentThread().getContextClassLoader());
+ }
+ pluginInstance = loadPluginInstance(pluginIdentifier, classLoader);
+ if (pluginInstance != null) {
+ LOGGER.info("Load plugin: {} from path: {} use classloader: {}",
+ pluginIdentifier, pluginJarPath.get(), classLoader.getClass().getName());
+ return pluginInstance;
+ }
+ }
+ throw new RuntimeException("Plugin " + pluginIdentifier + " not found.");
+ }
+
+ @Nullable
+ private T loadPluginInstance(PluginIdentifier pluginIdentifier, ClassLoader classLoader) {
+ ServiceLoader<T> serviceLoader = ServiceLoader.load(getPluginBaseClass(), classLoader);
+ for (T t : serviceLoader) {
+ if (t instanceof Plugin) {
+ // old api
+ Plugin<?> pluginInstance = (Plugin<?>) t;
+ if (StringUtils.equalsIgnoreCase(pluginInstance.getPluginName(), pluginIdentifier.getPluginName())) {
+ return (T) pluginInstance;
+ }
+ } else if (t instanceof PluginIdentifierInterface) {
+ // new api
+ PluginIdentifierInterface pluginIdentifierInstance = (PluginIdentifierInterface) t;
+ if (StringUtils.equalsIgnoreCase(pluginIdentifierInstance.getPluginName(), pluginIdentifier.getPluginName())) {
+ return (T) pluginIdentifierInstance;
+ }
+ } else {
+ throw new UnsupportedOperationException("Plugin instance: " + t + " is not supported.");
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Get the plugin instance.
+ *
+ * @param pluginIdentifier plugin identifier.
+ * @return plugin instance.
+ */
+ protected Optional<URL> getPluginJarPath(PluginIdentifier pluginIdentifier) {
+ return pluginJarPath.computeIfAbsent(pluginIdentifier, this::findPluginJarPath);
+ }
+
+ /**
+ * Get spark plugin interface.
+ *
+ * @return plugin base class.
+ */
+ protected abstract Class<T> getPluginBaseClass();
+
+ /**
+ * Find the plugin jar path;
+ *
+ * @param pluginIdentifier plugin identifier.
+ * @return plugin jar path.
+ */
+ private Optional<URL> findPluginJarPath(PluginIdentifier pluginIdentifier) {
+ if (PLUGIN_JAR_MAPPING.isEmpty()) {
+ return Optional.empty();
+ }
+ final String engineType = pluginIdentifier.getEngineType().toLowerCase();
+ final String pluginType = pluginIdentifier.getPluginType().toLowerCase();
+ final String pluginName = pluginIdentifier.getPluginName().toLowerCase();
+ if (!PLUGIN_JAR_MAPPING.hasPath(engineType)) {
+ return Optional.empty();
+ }
+ Config engineConfig = PLUGIN_JAR_MAPPING.getConfig(engineType);
+ if (!engineConfig.hasPath(pluginType)) {
+ return Optional.empty();
+ }
+ Config typeConfig = engineConfig.getConfig(pluginType);
+ Optional<Map.Entry<String, ConfigValue>> optional = typeConfig.entrySet().stream()
+ .filter(entry -> StringUtils.equalsIgnoreCase(entry.getKey(), pluginName))
+ .findFirst();
+ if (!optional.isPresent()) {
+ return Optional.empty();
+ }
+ String pluginJarPrefix = optional.get().getValue().unwrapped().toString();
+ File[] targetPluginFiles = pluginDir.toFile().listFiles(new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ return pathname.getName().endsWith(".jar") && StringUtils.startsWithIgnoreCase(pathname.getName(), pluginJarPrefix);
+ }
+ });
+ if (ArrayUtils.isEmpty(targetPluginFiles)) {
+ return Optional.empty();
+ }
+ try {
+ URL pluginJarPath = targetPluginFiles[0].toURI().toURL();
+ LOGGER.info("Discovery plugin jar: {} at: {}", pluginIdentifier.getPluginName(), pluginJarPath);
+ return Optional.of(pluginJarPath);
+ } catch (MalformedURLException e) {
+ LOGGER.warn("Cannot get plugin URL: " + targetPluginFiles[0], e);
+ return Optional.empty();
+ }
+ }
+}
diff --git a/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/PluginDiscovery.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/PluginDiscovery.java
new file mode 100644
index 000000000..8a571f92c
--- /dev/null
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/PluginDiscovery.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.plugin.discovery;
+
+import org.apache.seatunnel.common.config.Common;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions;
+
+import java.net.URL;
+import java.util.List;
+
+/**
+ * Plugins discovery interface, used to find plugin. Each plugin type should have its own implementation.
+ *
+ * @param <T> plugin type
+ */
+public interface PluginDiscovery<T> {
+
+ String PLUGIN_MAPPING_FILE = "plugin-mapping.properties";
+ /**
+ * The plugin mapping config.
+ * e,g.flink.source.DruidSource=seatunnel-connector-flink-druid
+ */
+ Config PLUGIN_JAR_MAPPING =
+ ConfigFactory
+ // todo: rename to plugin dir
+ .parseFile(Common.connectorDir().resolve(PLUGIN_MAPPING_FILE).toFile())
+ .resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
+ .resolveWith(ConfigFactory.systemProperties(), ConfigResolveOptions.defaults().setAllowUnresolved(true));
+
+ /**
+ * Get all plugin jar paths.
+ *
+ * @return plugin jars.
+ */
+ List<URL> getPluginJarPaths(List<PluginIdentifier> pluginIdentifiers);
+
+ /**
+ * Get plugin instance by plugin identifier.
+ *
+ * @param pluginIdentifier plugin identifier.
+ * @return plugin instance. If not found, throw IllegalArgumentException.
+ */
+ T createPluginInstance(PluginIdentifier pluginIdentifier);
+
+ /**
+ * Get all plugin instances.
+ *
+ * @return plugin instances.
+ */
+ List<T> getAllPlugins(List<PluginIdentifier> pluginIdentifiers);
+
+}
diff --git a/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/PluginIdentifier.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/PluginIdentifier.java
new file mode 100644
index 000000000..37e322b73
--- /dev/null
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/PluginIdentifier.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.plugin.discovery;
+
+import java.util.Objects;
+
+/**
+ * Used to identify a plugin.
+ */
+public class PluginIdentifier {
+ private final String engineType;
+ private final String pluginType;
+ private final String pluginName;
+
+ private PluginIdentifier(String engineType, String pluginType, String pluginName) {
+ this.engineType = engineType;
+ this.pluginType = pluginType;
+ this.pluginName = pluginName;
+ }
+
+ public static PluginIdentifier of(String engineType, String pluginType, String pluginName) {
+ return new PluginIdentifier(engineType, pluginType, pluginName);
+ }
+
+ public String getEngineType() {
+ return engineType;
+ }
+
+ public String getPluginType() {
+ return pluginType;
+ }
+
+ public String getPluginName() {
+ return pluginName;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ PluginIdentifier that = (PluginIdentifier) o;
+
+ if (!Objects.equals(engineType, that.engineType)) {
+ return false;
+ }
+ if (!Objects.equals(pluginType, that.pluginType)) {
+ return false;
+ }
+ return Objects.equals(pluginName, that.pluginName);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = engineType != null ? engineType.hashCode() : 0;
+ result = 31 * result + (pluginType != null ? pluginType.hashCode() : 0);
+ result = 31 * result + (pluginName != null ? pluginName.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "PluginIdentifier{" +
+ "engineType='" + engineType + '\'' +
+ ", pluginType='" + pluginType + '\'' +
+ ", pluginName='" + pluginName + '\'' +
+ '}';
+ }
+}
diff --git a/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/flink/FlinkAbstractPluginDiscovery.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/flink/FlinkAbstractPluginDiscovery.java
new file mode 100644
index 000000000..a9956fd3f
--- /dev/null
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/flink/FlinkAbstractPluginDiscovery.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.plugin.discovery.flink;
+
+import org.apache.seatunnel.common.utils.ReflectionUtils;
+import org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery;
+
+import java.net.URLClassLoader;
+
+public abstract class FlinkAbstractPluginDiscovery<T> extends AbstractPluginDiscovery<T> {
+
+ public FlinkAbstractPluginDiscovery(String pluginSubDir) {
+ super(pluginSubDir, (classLoader, url) -> {
+ if (classLoader.getClass().getName().endsWith("SafetyNetWrapperClassLoader")) {
+ URLClassLoader c = (URLClassLoader) ReflectionUtils.getField(classLoader, "inner").get();
+ ReflectionUtils.invoke(c, "addURL", url);
+ } else if (classLoader instanceof URLClassLoader) {
+ ReflectionUtils.invoke(classLoader, "addURL", url);
+ } else {
+ throw new RuntimeException("Unsupported classloader: " + classLoader.getClass().getName());
+ }
+ });
+ }
+
+}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/flink/FlinkSinkPluginDiscovery.java
similarity index 69%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/flink/FlinkSinkPluginDiscovery.java
index f5f308154..8c973cd0e 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/flink/FlinkSinkPluginDiscovery.java
@@ -15,18 +15,18 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.plugin.discovery.flink;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import org.apache.seatunnel.flink.BaseFlinkSink;
- private final String type;
+public class FlinkSinkPluginDiscovery extends FlinkAbstractPluginDiscovery<BaseFlinkSink> {
- PluginType(String type) {
- this.type = type;
+ public FlinkSinkPluginDiscovery() {
+ super("flink");
}
- public String getType() {
- return type;
+ @Override
+ protected Class<BaseFlinkSink> getPluginBaseClass() {
+ return BaseFlinkSink.class;
}
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/flink/FlinkSourcePluginDiscovery.java
similarity index 68%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/flink/FlinkSourcePluginDiscovery.java
index f5f308154..fd9e41564 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/flink/FlinkSourcePluginDiscovery.java
@@ -15,18 +15,17 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.plugin.discovery.flink;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import org.apache.seatunnel.flink.BaseFlinkSource;
- private final String type;
-
- PluginType(String type) {
- this.type = type;
+public class FlinkSourcePluginDiscovery extends FlinkAbstractPluginDiscovery<BaseFlinkSource> {
+ public FlinkSourcePluginDiscovery() {
+ super("flink");
}
- public String getType() {
- return type;
+ @Override
+ protected Class<BaseFlinkSource> getPluginBaseClass() {
+ return BaseFlinkSource.class;
}
}
diff --git a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/flink/FlinkTransformPluginDiscovery.java
similarity index 55%
copy from seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
copy to seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/flink/FlinkTransformPluginDiscovery.java
index f0cf54447..12a91d088 100644
--- a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/flink/FlinkTransformPluginDiscovery.java
@@ -15,33 +15,28 @@
* limitations under the License.
*/
-package org.apache.seatunnel.apis.base.env;
+package org.apache.seatunnel.plugin.discovery.flink;
-import org.apache.seatunnel.common.config.CheckResult;
-import org.apache.seatunnel.common.constants.JobMode;
-
-import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.flink.BaseFlinkTransform;
+import org.apache.seatunnel.plugin.discovery.PluginIdentifier;
import java.net.URL;
-import java.util.Set;
-
-/**
- * engine related runtime environment
- */
-public interface RuntimeEnv {
-
- RuntimeEnv setConfig(Config config);
-
- Config getConfig();
-
- CheckResult checkConfig();
-
- RuntimeEnv prepare();
+import java.util.ArrayList;
+import java.util.List;
- RuntimeEnv setJobMode(JobMode mode);
+public class FlinkTransformPluginDiscovery extends FlinkAbstractPluginDiscovery<BaseFlinkTransform> {
- JobMode getJobMode();
+ public FlinkTransformPluginDiscovery() {
+ super("flink");
+ }
- void registerPlugin(Set<URL> pluginPaths);
+ @Override
+ public List<URL> getPluginJarPaths(List<PluginIdentifier> pluginIdentifiers) {
+ return new ArrayList<>();
+ }
+ @Override
+ protected Class<BaseFlinkTransform> getPluginBaseClass() {
+ return BaseFlinkTransform.class;
+ }
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelFlinkTransformPluginDiscovery.java
similarity index 60%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelFlinkTransformPluginDiscovery.java
index f5f308154..2b5620f4a 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelFlinkTransformPluginDiscovery.java
@@ -15,18 +15,22 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.plugin.discovery.seatunnel;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import org.apache.seatunnel.flink.BaseFlinkTransform;
+import org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery;
- private final String type;
+/**
+ * Discovery for the SeaTunnel Flink transform.
+ */
+public class SeaTunnelFlinkTransformPluginDiscovery extends AbstractPluginDiscovery<BaseFlinkTransform> {
- PluginType(String type) {
- this.type = type;
+ public SeaTunnelFlinkTransformPluginDiscovery() {
+ super("seatunnel");
}
- public String getType() {
- return type;
+ @Override
+ protected Class<BaseFlinkTransform> getPluginBaseClass() {
+ return BaseFlinkTransform.class;
}
}
diff --git a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelSinkPluginDiscovery.java
similarity index 56%
copy from seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
copy to seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelSinkPluginDiscovery.java
index f0cf54447..e2ca9427e 100644
--- a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelSinkPluginDiscovery.java
@@ -15,33 +15,26 @@
* limitations under the License.
*/
-package org.apache.seatunnel.apis.base.env;
+package org.apache.seatunnel.plugin.discovery.seatunnel;
-import org.apache.seatunnel.common.config.CheckResult;
-import org.apache.seatunnel.common.constants.JobMode;
-
-import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.api.sink.SeaTunnelSink;
+import org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery;
import java.net.URL;
-import java.util.Set;
-
-/**
- * engine related runtime environment
- */
-public interface RuntimeEnv {
-
- RuntimeEnv setConfig(Config config);
-
- Config getConfig();
-
- CheckResult checkConfig();
-
- RuntimeEnv prepare();
+import java.util.function.BiConsumer;
- RuntimeEnv setJobMode(JobMode mode);
+public class SeaTunnelSinkPluginDiscovery extends AbstractPluginDiscovery<SeaTunnelSink> {
- JobMode getJobMode();
+ public SeaTunnelSinkPluginDiscovery() {
+ super("seatunnel");
+ }
- void registerPlugin(Set<URL> pluginPaths);
+ public SeaTunnelSinkPluginDiscovery(BiConsumer<ClassLoader, URL> addURLToClassLoader) {
+ super("seatunnel", addURLToClassLoader);
+ }
+ @Override
+ protected Class<SeaTunnelSink> getPluginBaseClass() {
+ return SeaTunnelSink.class;
+ }
}
diff --git a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelSourcePluginDiscovery.java
similarity index 55%
copy from seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
copy to seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelSourcePluginDiscovery.java
index f0cf54447..f9da2a0a9 100644
--- a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelSourcePluginDiscovery.java
@@ -15,33 +15,26 @@
* limitations under the License.
*/
-package org.apache.seatunnel.apis.base.env;
+package org.apache.seatunnel.plugin.discovery.seatunnel;
-import org.apache.seatunnel.common.config.CheckResult;
-import org.apache.seatunnel.common.constants.JobMode;
-
-import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.api.source.SeaTunnelSource;
+import org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery;
import java.net.URL;
-import java.util.Set;
-
-/**
- * engine related runtime environment
- */
-public interface RuntimeEnv {
-
- RuntimeEnv setConfig(Config config);
-
- Config getConfig();
-
- CheckResult checkConfig();
-
- RuntimeEnv prepare();
+import java.util.function.BiConsumer;
- RuntimeEnv setJobMode(JobMode mode);
+public class SeaTunnelSourcePluginDiscovery extends AbstractPluginDiscovery<SeaTunnelSource> {
- JobMode getJobMode();
+ public SeaTunnelSourcePluginDiscovery() {
+ super("seatunnel");
+ }
- void registerPlugin(Set<URL> pluginPaths);
+ public SeaTunnelSourcePluginDiscovery(BiConsumer<ClassLoader, URL> addURLToClassLoader) {
+ super("seatunnel", addURLToClassLoader);
+ }
+ @Override
+ protected Class<SeaTunnelSource> getPluginBaseClass() {
+ return SeaTunnelSource.class;
+ }
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelSparkTransformPluginDiscovery.java
similarity index 62%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelSparkTransformPluginDiscovery.java
index f5f308154..bfc5358e7 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelSparkTransformPluginDiscovery.java
@@ -15,18 +15,18 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.plugin.discovery.seatunnel;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery;
+import org.apache.seatunnel.spark.BaseSparkTransform;
- private final String type;
-
- PluginType(String type) {
- this.type = type;
+public class SeaTunnelSparkTransformPluginDiscovery extends AbstractPluginDiscovery<BaseSparkTransform> {
+ public SeaTunnelSparkTransformPluginDiscovery() {
+ super("seatunnel");
}
- public String getType() {
- return type;
+ @Override
+ protected Class<BaseSparkTransform> getPluginBaseClass() {
+ return BaseSparkTransform.class;
}
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/spark/SparkSinkPluginDiscovery.java
similarity index 65%
copy from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
copy to seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/spark/SparkSinkPluginDiscovery.java
index f5f308154..8a30c68ed 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/spark/SparkSinkPluginDiscovery.java
@@ -15,18 +15,19 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.plugin.discovery.spark;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery;
+import org.apache.seatunnel.spark.BaseSparkSink;
- private final String type;
+public class SparkSinkPluginDiscovery extends AbstractPluginDiscovery<BaseSparkSink> {
- PluginType(String type) {
- this.type = type;
+ public SparkSinkPluginDiscovery() {
+ super("spark");
}
- public String getType() {
- return type;
+ @Override
+ protected Class<BaseSparkSink> getPluginBaseClass() {
+ return BaseSparkSink.class;
}
}
diff --git a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/spark/SparkSourcePluginDiscovery.java
similarity index 64%
rename from seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
rename to seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/spark/SparkSourcePluginDiscovery.java
index f5f308154..ee961bf98 100644
--- a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/core/base/config/PluginType.java
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/spark/SparkSourcePluginDiscovery.java
@@ -15,18 +15,19 @@
* limitations under the License.
*/
-package org.apache.seatunnel.core.base.config;
+package org.apache.seatunnel.plugin.discovery.spark;
-public enum PluginType {
- SOURCE("source"), TRANSFORM("transform"), SINK("sink");
+import org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery;
+import org.apache.seatunnel.spark.BaseSparkSource;
- private final String type;
+public class SparkSourcePluginDiscovery extends AbstractPluginDiscovery<BaseSparkSource> {
- PluginType(String type) {
- this.type = type;
+ public SparkSourcePluginDiscovery() {
+ super("spark");
}
- public String getType() {
- return type;
+ @Override
+ protected Class<BaseSparkSource> getPluginBaseClass() {
+ return BaseSparkSource.class;
}
}
diff --git a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/spark/SparkTransformPluginDiscovery.java
similarity index 51%
copy from seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
copy to seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/spark/SparkTransformPluginDiscovery.java
index f0cf54447..cc125e26d 100644
--- a/seatunnel-apis/seatunnel-api-base/src/main/java/org/apache/seatunnel/apis/base/env/RuntimeEnv.java
+++ b/seatunnel-plugin-discovery/src/main/java/org/apache/seatunnel/plugin/discovery/spark/SparkTransformPluginDiscovery.java
@@ -15,33 +15,32 @@
* limitations under the License.
*/
-package org.apache.seatunnel.apis.base.env;
+package org.apache.seatunnel.plugin.discovery.spark;
-import org.apache.seatunnel.common.config.CheckResult;
-import org.apache.seatunnel.common.constants.JobMode;
-
-import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery;
+import org.apache.seatunnel.plugin.discovery.PluginIdentifier;
+import org.apache.seatunnel.spark.BaseSparkTransform;
import java.net.URL;
-import java.util.Set;
+import java.util.Collections;
+import java.util.List;
/**
- * engine related runtime environment
+ * Transform plugin will load from the classpath.
*/
-public interface RuntimeEnv {
-
- RuntimeEnv setConfig(Config config);
-
- Config getConfig();
-
- CheckResult checkConfig();
-
- RuntimeEnv prepare();
-
- RuntimeEnv setJobMode(JobMode mode);
+public class SparkTransformPluginDiscovery extends AbstractPluginDiscovery<BaseSparkTransform> {
- JobMode getJobMode();
+ public SparkTransformPluginDiscovery() {
+ super("spark");
+ }
- void registerPlugin(Set<URL> pluginPaths);
+ @Override
+ public List<URL> getPluginJarPaths(List<PluginIdentifier> pluginIdentifiers) {
+ return Collections.emptyList();
+ }
+ @Override
+ protected Class<BaseSparkTransform> getPluginBaseClass() {
+ return BaseSparkTransform.class;
+ }
}
diff --git a/tools/checkstyle/checkStyle.xml b/tools/checkstyle/checkStyle.xml
index ebbc1f258..ddff7c20f 100755
--- a/tools/checkstyle/checkStyle.xml
+++ b/tools/checkstyle/checkStyle.xml
@@ -85,6 +85,11 @@
<property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
</module>
+ <module name="SuppressionSingleFilter">
+ <property name="files" value=".*(IT|Test).java"/>
+ <property name="checks" value="MagicNumber"/>
+ </module>
+
<module name="SuppressWarningsFilter"/>
<module name="TreeWalker">
<module name="SuppressWarningsHolder"/>
@@ -207,7 +212,9 @@
<module name="OneStatementPerLine"/>
- <module name="MagicNumber"/>
+ <module name="MagicNumber">
+ <property name="ignoreHashCodeMethod" value="true"/>
+ </module>
<module name="MultipleVariableDeclarations"/>
diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt
index 066afbd87..59af35d51 100755
--- a/tools/dependencies/known-dependencies.txt
+++ b/tools/dependencies/known-dependencies.txt
@@ -67,13 +67,8 @@ commons-beanutils-core-1.8.0.jar
commons-cli-1.2.jar
commons-cli-1.3.1.jar
commons-cli-1.4.jar
-commons-codec-1.10.jar
-commons-codec-1.11.jar
commons-codec-1.13.jar
-commons-codec-1.7.jar
-commons-codec-1.9.jar
commons-collections-3.2.2.jar
-commons-collections4-4.2.jar
commons-collections4-4.4.jar
commons-compiler-3.0.9.jar
commons-compiler-3.1.6.jar
@@ -574,6 +569,7 @@ okio-1.4.0.jar
okio-2.8.0.jar
opencsv-4.6.jar
orc-core-1.5.6.jar
+orc-shims-1.5.2.jar
orc-shims-1.5.6.jar
osgi-resource-locator-1.0.1.jar
paranamer-2.3.jar