You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ki...@apache.org on 2022/03/24 14:49:03 UTC
[dolphinscheduler] branch dev updated: [Feat]Restore presto and spark sql datasource in datasource plugin module (#7619)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 934d0ad [Feat]Restore presto and spark sql datasource in datasource plugin module (#7619)
934d0ad is described below
commit 934d0ad0633e3ec04e54fd6527662f2674194ad3
Author: sparklezzz <zh...@gmail.com>
AuthorDate: Thu Mar 24 22:48:55 2022 +0800
[Feat]Restore presto and spark sql datasource in datasource plugin module (#7619)
* feat(datasource-plugin): restore presto and spark sql datasource
* feat(datasource-plugin): restore presto and spark sql datasource
* feat(datasource for spark/presto): fix pom and simplify code
Co-authored-by: xudong.zhang <xu...@nio.com>
---
.../dolphinscheduler-datasource-all/pom.xml | 8 ++++
.../datasource/hive/HiveDataSourceClient.java | 2 +-
.../pom.xml | 41 +++++++++-----------
.../datasource/presto/PrestoDataSourceChannel.java | 31 +++++++++++++++
.../presto/PrestoDataSourceChannelFactory.java | 36 ++++++++++++++++++
.../datasource/presto/PrestoDataSourceClient.java | 30 +++++++++++++++
.../presto/PrestoDataSourceChannelFactoryTest.java | 33 ++++++++++++++++
.../presto/PrestoDataSourceChannelTest.java | 44 ++++++++++++++++++++++
.../pom.xml | 36 +++++++++---------
.../datasource/spark/SparkDataSourceChannel.java | 31 +++++++++++++++
.../spark/SparkDataSourceChannelFactory.java | 36 ++++++++++++++++++
.../datasource/spark/SparkDataSourceClient.java | 35 +++++++++++++++++
.../spark/SparkDataSourceChannelFactoryTest.java | 33 ++++++++++++++++
.../spark/SparkDataSourceChannelTest.java | 44 ++++++++++++++++++++++
dolphinscheduler-datasource-plugin/pom.xml | 2 +
pom.xml | 10 +++++
16 files changed, 410 insertions(+), 42 deletions(-)
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
index ff20614..05a5845 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
@@ -60,5 +60,13 @@
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-redshift</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-datasource-spark</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-datasource-presto</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceClient.java
index 68edf02..642e06a 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceClient.java
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceClient.java
@@ -158,7 +158,7 @@ public class HiveDataSourceClient extends CommonDataSourceClient {
public void close() {
super.close();
- logger.info("close HiveDataSourceClient.");
+ logger.info("close {}.", this.getClass().getSimpleName());
kerberosRenewalService.shutdown();
this.ugi = null;
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/pom.xml
similarity index 58%
copy from dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
copy to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/pom.xml
index ff20614..4c45815 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/pom.xml
@@ -25,40 +25,35 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>dolphinscheduler-datasource-all</artifactId>
+ <artifactId>dolphinscheduler-datasource-presto</artifactId>
+ <name>${project.artifactId}</name>
+ <packaging>jar</packaging>
<dependencies>
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-clickhouse</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-db2</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-hive</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-mysql</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-oracle</artifactId>
+ <artifactId>dolphinscheduler-spi</artifactId>
+ <scope>provided</scope>
</dependency>
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-postgresql</artifactId>
+ <artifactId>dolphinscheduler-datasource-api</artifactId>
+ <version>${project.version}</version>
</dependency>
+
<dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-sqlserver</artifactId>
+ <groupId>com.facebook.presto</groupId>
+ <artifactId>presto-jdbc</artifactId>
+ <version>${presto.jdbc.version}</version>
</dependency>
+
<dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-redshift</artifactId>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <type>jar</type>
+ <scope>test</scope>
</dependency>
</dependencies>
</project>
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannel.java
new file mode 100644
index 0000000..1ea69fe
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannel.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.presto;
+
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
+import org.apache.dolphinscheduler.spi.datasource.DataSourceClient;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
+public class PrestoDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
+ return new PrestoDataSourceClient(baseConnectionParam, dbType);
+ }
+}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java
new file mode 100644
index 0000000..95d0f25
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.presto;
+
+import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
+import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(DataSourceChannelFactory.class)
+public class PrestoDataSourceChannelFactory implements DataSourceChannelFactory {
+ @Override
+ public String getName() {
+ return "presto";
+ }
+
+ @Override
+ public DataSourceChannel create() {
+ return new PrestoDataSourceChannel();
+ }
+}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceClient.java
new file mode 100644
index 0000000..bb59171
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceClient.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.presto;
+
+import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient;
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
+public class PrestoDataSourceClient extends CommonDataSourceClient {
+
+ public PrestoDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
+ super(baseConnectionParam, dbType);
+ }
+
+}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/test/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactoryTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/test/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactoryTest.java
new file mode 100644
index 0000000..c8052bc
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/test/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactoryTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.presto;
+
+import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class PrestoDataSourceChannelFactoryTest {
+
+ @Test
+ public void testCreate() {
+ PrestoDataSourceChannelFactory sourceChannelFactory = new PrestoDataSourceChannelFactory();
+ DataSourceChannel dataSourceChannel = sourceChannelFactory.create();
+ Assert.assertNotNull(dataSourceChannel);
+ }
+}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/test/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/test/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelTest.java
new file mode 100644
index 0000000..e0f4de9
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/test/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.presto;
+
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoConnectionParam;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+@SuppressStaticInitializationFor("org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient")
+@PrepareForTest({PrestoDataSourceClient.class, PrestoDataSourceChannel.class})
+public class PrestoDataSourceChannelTest {
+
+ @Test
+ public void testCreateDataSourceClient() {
+ PrestoDataSourceChannel sourceChannel = PowerMockito.mock(PrestoDataSourceChannel.class);
+ PrestoDataSourceClient dataSourceClient = PowerMockito.mock(PrestoDataSourceClient.class);
+ PowerMockito.when(sourceChannel.createDataSourceClient(Mockito.any(), Mockito.any())).thenReturn(dataSourceClient);
+ Assert.assertNotNull(sourceChannel.createDataSourceClient(new PrestoConnectionParam(), DbType.PRESTO));
+ }
+}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/pom.xml
similarity index 66%
copy from dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
copy to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/pom.xml
index ff20614..8360237 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/pom.xml
@@ -25,40 +25,40 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>dolphinscheduler-datasource-all</artifactId>
+ <artifactId>dolphinscheduler-datasource-spark</artifactId>
+ <name>${project.artifactId}</name>
+ <packaging>jar</packaging>
<dependencies>
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-clickhouse</artifactId>
+ <artifactId>dolphinscheduler-spi</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-db2</artifactId>
+ <artifactId>dolphinscheduler-task-api</artifactId>
+ <scope>provided</scope>
</dependency>
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-hive</artifactId>
+ <scope>provided</scope>
</dependency>
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-mysql</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-oracle</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-postgresql</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-sqlserver</artifactId>
+ <artifactId>dolphinscheduler-datasource-api</artifactId>
+ <version>${project.version}</version>
</dependency>
+
<dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-redshift</artifactId>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <type>jar</type>
+ <scope>test</scope>
</dependency>
</dependencies>
</project>
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannel.java
new file mode 100644
index 0000000..d24e0d1
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannel.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.spark;
+
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
+import org.apache.dolphinscheduler.spi.datasource.DataSourceClient;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
+public class SparkDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
+ return new SparkDataSourceClient(baseConnectionParam, dbType);
+ }
+}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java
new file mode 100644
index 0000000..f49c0b3
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.spark;
+
+import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
+import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(DataSourceChannelFactory.class)
+public class SparkDataSourceChannelFactory implements DataSourceChannelFactory {
+ @Override
+ public String getName() {
+ return "spark";
+ }
+
+ @Override
+ public DataSourceChannel create() {
+ return new SparkDataSourceChannel();
+ }
+}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceClient.java
new file mode 100644
index 0000000..26ac9b3
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceClient.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.spark;
+
+import org.apache.dolphinscheduler.plugin.datasource.hive.HiveDataSourceClient;
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SparkDataSourceClient extends HiveDataSourceClient {
+
+ private static final Logger logger = LoggerFactory.getLogger(SparkDataSourceClient.class);
+
+ public SparkDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
+ super(baseConnectionParam, dbType);
+ }
+
+}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/test/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactoryTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/test/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactoryTest.java
new file mode 100644
index 0000000..9b4b4a3
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/test/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactoryTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.spark;
+
+import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SparkDataSourceChannelFactoryTest {
+
+ @Test
+ public void testCreate() {
+ SparkDataSourceChannelFactory sourceChannelFactory = new SparkDataSourceChannelFactory();
+ DataSourceChannel dataSourceChannel = sourceChannelFactory.create();
+ Assert.assertNotNull(dataSourceChannel);
+ }
+}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/test/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/test/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelTest.java
new file mode 100644
index 0000000..c8077ee
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/test/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.spark;
+
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkConnectionParam;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+@SuppressStaticInitializationFor("org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient")
+@PrepareForTest({SparkDataSourceChannel.class, SparkDataSourceClient.class})
+public class SparkDataSourceChannelTest {
+
+ @Test
+ public void testCreateDataSourceClient() {
+ SparkDataSourceChannel sourceChannel = PowerMockito.mock(SparkDataSourceChannel.class);
+ SparkDataSourceClient dataSourceClient = PowerMockito.mock(SparkDataSourceClient.class);
+ PowerMockito.when(sourceChannel.createDataSourceClient(Mockito.any(), Mockito.any())).thenReturn(dataSourceClient);
+ Assert.assertNotNull(sourceChannel.createDataSourceClient(new SparkConnectionParam(), DbType.SPARK));
+ }
+}
diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/pom.xml
index a8c018e..3c281fc 100644
--- a/dolphinscheduler-datasource-plugin/pom.xml
+++ b/dolphinscheduler-datasource-plugin/pom.xml
@@ -36,6 +36,8 @@
<module>dolphinscheduler-datasource-mysql</module>
<module>dolphinscheduler-datasource-oracle</module>
<module>dolphinscheduler-datasource-postgresql</module>
+ <module>dolphinscheduler-datasource-spark</module>
+ <module>dolphinscheduler-datasource-presto</module>
<module>dolphinscheduler-datasource-api</module>
<module>dolphinscheduler-datasource-all</module>
<module>dolphinscheduler-datasource-redshift</module>
diff --git a/pom.xml b/pom.xml
index f02898d..2db6a4a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -442,6 +442,16 @@
<artifactId>dolphinscheduler-datasource-redshift</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-datasource-spark</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-datasource-presto</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>