You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2022/03/01 11:30:37 UTC
[dolphinscheduler] branch dev updated: [Feature-8527][Datasource] Add dataSource for Amazon Redshift (#8564)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 7b66b66 [Feature-8527][Datasource] Add dataSource for Amazon Redshift (#8564)
7b66b66 is described below
commit 7b66b66e8ffbadeadc0b7f3081e7e218b776932c
Author: ronyang1985 <yr...@gmail.com>
AuthorDate: Tue Mar 1 19:29:52 2022 +0800
[Feature-8527][Datasource] Add dataSource for Amazon Redshift (#8564)
Co-authored-by: Wenjun Ruan <we...@apache.org>
---
.../dolphinscheduler-datasource-all/pom.xml | 4 +
.../api/datasource/BaseDataSourceParamDTO.java | 3 +
.../redshift/RedshiftConnectionParam.java | 37 ++++++
.../redshift/RedshiftDataSourceParamDTO.java | 43 +++++++
.../redshift/RedshiftDataSourceProcessor.java | 140 +++++++++++++++++++++
.../datasource/api/utils/DataSourceUtils.java | 4 +
.../redshift/RedshiftDataSourceProcessorTest.java | 98 +++++++++++++++
.../pom.xml | 34 ++---
.../redshift/RedshiftDataSourceChannel.java | 61 ++-------
.../redshift/RedshiftDataSourceChannelFactory.java | 59 +++------
.../redshift/RedshiftDataSourceClient.java | 55 ++------
dolphinscheduler-datasource-plugin/pom.xml | 1 +
.../apache/dolphinscheduler/spi/enums/DbType.java | 4 +-
.../dolphinscheduler/spi/utils/Constants.java | 23 +++-
.../src/service/modules/data-source/types.ts | 1 +
.../src/views/datasource/list/use-form.ts | 5 +
.../components/node/fields/use-datasource-type.ts | 5 +
.../pages/list/_source/createDataSource.vue | 7 ++
.../src/js/conf/home/store/dag/state.js | 5 +
.../src/js/conf/home/store/datasource/actions.js | 4 +-
pom.xml | 5 +
21 files changed, 436 insertions(+), 162 deletions(-)
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
index 8da9b91..ff20614 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
@@ -56,5 +56,9 @@
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-sqlserver</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-datasource-redshift</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java
index e825285..7d9cbb2 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java
@@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLD
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSQLDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoDataSourceParamDTO;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift.RedshiftDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SQLServerDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;
@@ -46,6 +47,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
* see {@link SQLServerDataSourceParamDTO}
* see {@link Db2DataSourceParamDTO}
* see {@link PrestoDataSourceParamDTO}
+ * see {@link RedshiftDataSourceParamDTO}
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
@JsonSubTypes(value = {
@@ -58,6 +60,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
@JsonSubTypes.Type(value = SQLServerDataSourceParamDTO.class, name = "SQLSERVER"),
@JsonSubTypes.Type(value = Db2DataSourceParamDTO.class, name = "DB2"),
@JsonSubTypes.Type(value = PrestoDataSourceParamDTO.class, name = "PRESTO"),
+ @JsonSubTypes.Type(value = RedshiftDataSourceParamDTO.class, name = "REDSHIFT"),
})
public abstract class BaseDataSourceParamDTO implements Serializable {
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftConnectionParam.java
new file mode 100644
index 0000000..a266a2e
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftConnectionParam.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift;
+
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+
+public class RedshiftConnectionParam extends BaseConnectionParam {
+ @Override
+ public String toString() {
+ return "RedshiftConnectionParam{"
+ + "user='" + user + '\''
+ + ", password='" + password + '\''
+ + ", address='" + address + '\''
+ + ", database='" + database + '\''
+ + ", jdbcUrl='" + jdbcUrl + '\''
+ + ", driverLocation='" + driverLocation + '\''
+ + ", driverClassName='" + driverClassName + '\''
+ + ", validationQuery='" + validationQuery + '\''
+ + ", other='" + other + '\''
+ + '}';
+ }
+}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceParamDTO.java
new file mode 100644
index 0000000..16d3281
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceParamDTO.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift;
+
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
+public class RedshiftDataSourceParamDTO extends BaseDataSourceParamDTO {
+
+ @Override
+ public String toString() {
+ return "RedshiftDataSourceParamDTO{"
+ + "name='" + name + '\''
+ + ", note='" + note + '\''
+ + ", host='" + host + '\''
+ + ", port=" + port
+ + ", database='" + database + '\''
+ + ", userName='" + userName + '\''
+ + ", password='" + password + '\''
+ + ", other='" + other + '\''
+ + '}';
+ }
+
+ @Override
+ public DbType getType() {
+ return DbType.REDSHIFT;
+ }
+}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessor.java
new file mode 100644
index 0000000..5a9c521
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessor.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift;
+
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.Constants;
+import org.apache.dolphinscheduler.spi.utils.JSONUtils;
+
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor {
+
+ @Override
+ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
+ RedshiftConnectionParam
+ connectionParams = (RedshiftConnectionParam) createConnectionParams(connectionJson);
+
+ String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH);
+ String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA);
+
+ RedshiftDataSourceParamDTO
+ redshiftDatasourceParamDTO = new RedshiftDataSourceParamDTO();
+ redshiftDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1]));
+ redshiftDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]);
+ redshiftDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
+ redshiftDatasourceParamDTO.setUserName(connectionParams.getUser());
+ redshiftDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
+
+ return redshiftDatasourceParamDTO;
+ }
+
+ @Override
+ public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
+ RedshiftDataSourceParamDTO redshiftParam = (RedshiftDataSourceParamDTO) datasourceParam;
+ String address = String.format("%s%s:%s", Constants.JDBC_REDSHIFT, redshiftParam.getHost(), redshiftParam.getPort());
+ String jdbcUrl = address + Constants.SLASH + redshiftParam.getDatabase();
+
+ RedshiftConnectionParam
+ redshiftConnectionParam = new RedshiftConnectionParam();
+ redshiftConnectionParam.setUser(redshiftParam.getUserName());
+ redshiftConnectionParam.setPassword(PasswordUtils.encodePassword(redshiftParam.getPassword()));
+ redshiftConnectionParam.setOther(transformOther(redshiftParam.getOther()));
+ redshiftConnectionParam.setAddress(address);
+ redshiftConnectionParam.setJdbcUrl(jdbcUrl);
+ redshiftConnectionParam.setDatabase(redshiftParam.getDatabase());
+ redshiftConnectionParam.setDriverClassName(getDatasourceDriver());
+ redshiftConnectionParam.setValidationQuery(getValidationQuery());
+ redshiftConnectionParam.setProps(redshiftParam.getOther());
+
+ return redshiftConnectionParam;
+ }
+
+ @Override
+ public ConnectionParam createConnectionParams(String connectionJson) {
+ return JSONUtils.parseObject(connectionJson, RedshiftConnectionParam.class);
+ }
+
+ @Override
+ public String getDatasourceDriver() {
+ return Constants.COM_REDSHIFT_JDBC_DRIVER;
+ }
+
+ @Override
+ public String getValidationQuery() {
+ return Constants.REDHIFT_VALIDATION_QUERY;
+ }
+
+ @Override
+ public String getJdbcUrl(ConnectionParam connectionParam) {
+ RedshiftConnectionParam
+ redshiftConnectionParam = (RedshiftConnectionParam) connectionParam;
+ if (!StringUtils.isEmpty(redshiftConnectionParam.getOther())) {
+ return String.format("%s?%s", redshiftConnectionParam.getJdbcUrl(), redshiftConnectionParam.getOther());
+ }
+ return redshiftConnectionParam.getJdbcUrl();
+ }
+
+ @Override
+ public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException {
+ RedshiftConnectionParam redshiftConnectionParam = (RedshiftConnectionParam) connectionParam;
+ Class.forName(getDatasourceDriver());
+ return DriverManager.getConnection(getJdbcUrl(connectionParam),
+ redshiftConnectionParam.getUser(), PasswordUtils.decodePassword(redshiftConnectionParam.getPassword()));
+ }
+
+ @Override
+ public DbType getDbType() {
+ return DbType.REDSHIFT;
+ }
+
+ private String transformOther(Map<String, String> otherMap) {
+ if (MapUtils.isNotEmpty(otherMap)) {
+ List<String> list = new ArrayList<>(otherMap.size());
+ otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value)));
+ return String.join(Constants.SEMICOLON, list);
+ }
+ return null;
+ }
+
+ private Map<String, String> parseOther(String other) {
+ Map<String, String> otherMap = new LinkedHashMap<>();
+ if (StringUtils.isEmpty(other)) {
+ return otherMap;
+ }
+ String[] configs = other.split(Constants.SEMICOLON);
+ for (String config : configs) {
+ otherMap.put(config.split(Constants.EQUAL_SIGN)[0], config.split(Constants.EQUAL_SIGN)[1]);
+ }
+ return otherMap;
+ }
+}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java
index cf41bc2..edcd1a3 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java
@@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLD
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSQLDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoDataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift.RedshiftDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SQLServerDataSourceProcessor;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
@@ -52,6 +53,7 @@ public class DataSourceUtils {
private static final DataSourceProcessor sqlServerProcessor = new SQLServerDataSourceProcessor();
private static final DataSourceProcessor db2PROCESSOR = new Db2DataSourceProcessor();
private static final DataSourceProcessor prestoPROCESSOR = new PrestoDataSourceProcessor();
+ private static final DataSourceProcessor redshiftProcessor = new RedshiftDataSourceProcessor();
/**
* check datasource param
@@ -120,6 +122,8 @@ public class DataSourceUtils {
return db2PROCESSOR;
case PRESTO:
return prestoPROCESSOR;
+ case REDSHIFT:
+ return redshiftProcessor;
default:
throw new IllegalArgumentException("datasource type illegal:" + dbType);
}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessorTest.java
new file mode 100644
index 0000000..a1756c3
--- /dev/null
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessorTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift;
+
+import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.Constants;
+
+import java.sql.DriverManager;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({Class.class, DriverManager.class, DataSourceUtils.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
+public class RedshiftDataSourceProcessorTest {
+
+ private RedshiftDataSourceProcessor redshiftDatasourceProcessor = new RedshiftDataSourceProcessor();
+
+ @Test
+ public void testCreateConnectionParams() {
+ Map<String, String> props = new HashMap<>();
+ props.put("serverTimezone", "utc");
+ RedshiftDataSourceParamDTO redshiftDatasourceParamDTO = new RedshiftDataSourceParamDTO();
+ redshiftDatasourceParamDTO.setHost("localhost");
+ redshiftDatasourceParamDTO.setPort(5439);
+ redshiftDatasourceParamDTO.setDatabase("dev");
+ redshiftDatasourceParamDTO.setUserName("awsuser");
+ redshiftDatasourceParamDTO.setPassword("123456");
+ redshiftDatasourceParamDTO.setOther(props);
+ PowerMockito.mockStatic(PasswordUtils.class);
+ PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
+ RedshiftConnectionParam connectionParams = (RedshiftConnectionParam) redshiftDatasourceProcessor
+ .createConnectionParams(redshiftDatasourceParamDTO);
+ Assert.assertEquals("jdbc:redshift://localhost:5439", connectionParams.getAddress());
+ Assert.assertEquals("jdbc:redshift://localhost:5439/dev", connectionParams.getJdbcUrl());
+ }
+
+ @Test
+ public void testCreateConnectionParams2() {
+ String connectionJson = "{\"user\":\"awsuser\",\"password\":\"123456\",\"address\":\"jdbc:redshift://localhost:5439\""
+ + ",\"database\":\"dev\",\"jdbcUrl\":\"jdbc:redshift://localhost:5439/dev\"}";
+ RedshiftConnectionParam connectionParams = (RedshiftConnectionParam) redshiftDatasourceProcessor
+ .createConnectionParams(connectionJson);
+ Assert.assertNotNull(connectionParams);
+ Assert.assertEquals("awsuser", connectionParams.getUser());
+ }
+
+ @Test
+ public void testGetDatasourceDriver() {
+ Assert.assertEquals(Constants.COM_REDSHIFT_JDBC_DRIVER, redshiftDatasourceProcessor.getDatasourceDriver());
+ }
+
+ @Test
+ public void testGetJdbcUrl() {
+ RedshiftConnectionParam redshiftConnectionParam = new RedshiftConnectionParam();
+ redshiftConnectionParam.setJdbcUrl("jdbc:redshift://localhost:5439/default");
+ redshiftConnectionParam.setOther("DSILogLevel=6;defaultRowFetchSize=100");
+ Assert.assertEquals("jdbc:redshift://localhost:5439/default?DSILogLevel=6;defaultRowFetchSize=100",
+ redshiftDatasourceProcessor.getJdbcUrl(redshiftConnectionParam));
+
+ }
+
+ @Test
+ public void testGetDbType() {
+ Assert.assertEquals(DbType.REDSHIFT, redshiftDatasourceProcessor.getDbType());
+ }
+
+ @Test
+ public void testGetValidationQuery() {
+ Assert.assertEquals(Constants.REDHIFT_VALIDATION_QUERY, redshiftDatasourceProcessor.getValidationQuery());
+ }
+}
\ No newline at end of file
diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml
similarity index 62%
copy from dolphinscheduler-datasource-plugin/pom.xml
copy to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml
index c167dab..d837b0d 100644
--- a/dolphinscheduler-datasource-plugin/pom.xml
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml
@@ -19,24 +19,26 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
- <artifactId>dolphinscheduler</artifactId>
+ <artifactId>dolphinscheduler-datasource-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<version>2.0.4-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>dolphinscheduler-datasource-plugin</artifactId>
- <name>${project.artifactId}</name>
- <packaging>pom</packaging>
- <modules>
- <module>dolphinscheduler-datasource-sqlserver</module>
- <module>dolphinscheduler-datasource-clickhouse</module>
- <module>dolphinscheduler-datasource-db2</module>
- <module>dolphinscheduler-datasource-hive</module>
- <module>dolphinscheduler-datasource-mysql</module>
- <module>dolphinscheduler-datasource-oracle</module>
- <module>dolphinscheduler-datasource-postgresql</module>
- <module>dolphinscheduler-datasource-api</module>
- <module>dolphinscheduler-datasource-all</module>
- </modules>
-</project>
+ <artifactId>dolphinscheduler-datasource-redshift</artifactId>
+ <packaging>jar</packaging>
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-spi</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-datasource-api</artifactId>
+ </dependency>
+
+ </dependencies>
+</project>
\ No newline at end of file
diff --git a/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java
similarity index 50%
copy from dolphinscheduler-ui-next/src/service/modules/data-source/types.ts
copy to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java
index e17c6a2..13e8172 100644
--- a/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java
@@ -15,53 +15,16 @@
* limitations under the License.
*/
-type IDataBase =
- | 'MYSQL'
- | 'POSTGRESQL'
- | 'HIVE'
- | 'SPARK'
- | 'CLICKHOUSE'
- | 'ORACLE'
- | 'SQLSERVER'
- | 'DB2'
- | 'PRESTO'
-
-interface IDataSource {
- id?: number
- type?: IDataBase
- name?: string
- note?: string
- host?: string
- port?: number
- principal?: string
- javaSecurityKrb5Conf?: string
- loginUserKeytabUsername?: string
- loginUserKeytabPath?: string
- userName?: string
- password?: string
- database?: string
- connectType?: string
- other?: object
-}
-
-interface ListReq {
- pageNo: number
- pageSize: number
- searchVal?: string
+package org.apache.dolphinscheduler.plugin.datasource.redshift;
+
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
+import org.apache.dolphinscheduler.spi.datasource.DataSourceClient;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
+public class RedshiftDataSourceChannel implements DataSourceChannel {
+ @Override
+ public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
+ return new RedshiftDataSourceClient(baseConnectionParam,dbType);
+ }
}
-
-interface UserIdReq {
- userId: number
-}
-
-interface TypeReq {
- type: IDataBase
-}
-
-interface NameReq {
- name: string
-}
-
-type IdReq = number
-
-export { ListReq, IDataBase, IDataSource, UserIdReq, TypeReq, NameReq, IdReq }
diff --git a/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java
similarity index 50%
copy from dolphinscheduler-ui-next/src/service/modules/data-source/types.ts
copy to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java
index e17c6a2..576a519 100644
--- a/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java
@@ -15,53 +15,22 @@
* limitations under the License.
*/
-type IDataBase =
- | 'MYSQL'
- | 'POSTGRESQL'
- | 'HIVE'
- | 'SPARK'
- | 'CLICKHOUSE'
- | 'ORACLE'
- | 'SQLSERVER'
- | 'DB2'
- | 'PRESTO'
+package org.apache.dolphinscheduler.plugin.datasource.redshift;
-interface IDataSource {
- id?: number
- type?: IDataBase
- name?: string
- note?: string
- host?: string
- port?: number
- principal?: string
- javaSecurityKrb5Conf?: string
- loginUserKeytabUsername?: string
- loginUserKeytabPath?: string
- userName?: string
- password?: string
- database?: string
- connectType?: string
- other?: object
-}
+import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
+import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
-interface ListReq {
- pageNo: number
- pageSize: number
- searchVal?: string
-}
+import com.google.auto.service.AutoService;
-interface UserIdReq {
- userId: number
-}
+@AutoService(DataSourceChannelFactory.class)
+public class RedshiftDataSourceChannelFactory implements DataSourceChannelFactory {
+ @Override
+ public DataSourceChannel create() {
+ return new RedshiftDataSourceChannel();
+ }
-interface TypeReq {
- type: IDataBase
+ @Override
+ public String getName() {
+ return "redshift";
+ }
}
-
-interface NameReq {
- name: string
-}
-
-type IdReq = number
-
-export { ListReq, IDataBase, IDataSource, UserIdReq, TypeReq, NameReq, IdReq }
diff --git a/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java
similarity index 50%
copy from dolphinscheduler-ui-next/src/service/modules/data-source/types.ts
copy to dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java
index e17c6a2..f10f7e1 100644
--- a/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java
@@ -15,53 +15,14 @@
* limitations under the License.
*/
-type IDataBase =
- | 'MYSQL'
- | 'POSTGRESQL'
- | 'HIVE'
- | 'SPARK'
- | 'CLICKHOUSE'
- | 'ORACLE'
- | 'SQLSERVER'
- | 'DB2'
- | 'PRESTO'
+package org.apache.dolphinscheduler.plugin.datasource.redshift;
-interface IDataSource {
- id?: number
- type?: IDataBase
- name?: string
- note?: string
- host?: string
- port?: number
- principal?: string
- javaSecurityKrb5Conf?: string
- loginUserKeytabUsername?: string
- loginUserKeytabPath?: string
- userName?: string
- password?: string
- database?: string
- connectType?: string
- other?: object
-}
-
-interface ListReq {
- pageNo: number
- pageSize: number
- searchVal?: string
-}
-
-interface UserIdReq {
- userId: number
-}
+import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient;
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.enums.DbType;
-interface TypeReq {
- type: IDataBase
+public class RedshiftDataSourceClient extends CommonDataSourceClient {
+ public RedshiftDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
+ super(baseConnectionParam, dbType);
+ }
}
-
-interface NameReq {
- name: string
-}
-
-type IdReq = number
-
-export { ListReq, IDataBase, IDataSource, UserIdReq, TypeReq, NameReq, IdReq }
diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/pom.xml
index c167dab..a8c018e 100644
--- a/dolphinscheduler-datasource-plugin/pom.xml
+++ b/dolphinscheduler-datasource-plugin/pom.xml
@@ -38,5 +38,6 @@
<module>dolphinscheduler-datasource-postgresql</module>
<module>dolphinscheduler-datasource-api</module>
<module>dolphinscheduler-datasource-all</module>
+ <module>dolphinscheduler-datasource-redshift</module>
</modules>
</project>
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java
index 2055249..099ec7f 100644
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java
+++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java
@@ -35,7 +35,9 @@ public enum DbType {
SQLSERVER(6, "sqlserver"),
DB2(7, "db2"),
PRESTO(8, "presto"),
- H2(9, "h2");
+ H2(9, "h2"),
+ REDSHIFT(10,"redshift"),
+ ;
@EnumValue
private final int code;
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java
index 8effe9a..cd05241 100644
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java
+++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java
@@ -161,6 +161,7 @@ public class Constants {
public static final String COM_SQLSERVER_JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver";
public static final String COM_PRESTO_JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver";
+ public static final String COM_REDSHIFT_JDBC_DRIVER = "com.amazon.redshift.jdbc42.Driver";
/**
@@ -174,6 +175,7 @@ public class Constants {
public static final String SQLSERVER_VALIDATION_QUERY = "select 1";
public static final String DB2_VALIDATION_QUERY = "select 1 from sysibm.sysdummy1";
public static final String PRESTO_VALIDATION_QUERY = "select 1";
+ public static final String REDHIFT_VALIDATION_QUERY = "select 1";
/**
* jdbc url
@@ -187,7 +189,7 @@ public class Constants {
public static final String JDBC_SQLSERVER = "jdbc:sqlserver://";
public static final String JDBC_DB2 = "jdbc:db2://";
public static final String JDBC_PRESTO = "jdbc:presto://";
-
+ public static final String JDBC_REDSHIFT = "jdbc:redshift://";
public static final String ADDRESS = "address";
public static final String DATABASE = "database";
@@ -205,6 +207,11 @@ public class Constants {
public static final String DOUBLE_SLASH = "//";
/**
+ * SLASH /
+ */
+ public static final String SLASH = "/";
+
+ /**
* comma ,
*/
public static final String COMMA = ",";
@@ -214,12 +221,24 @@ public class Constants {
*/
public static final String COLON = ":";
+
/**
- * AT SIGN
+ * AT SIGN @
*/
public static final String AT_SIGN = "@";
/**
+ * SEMICOLON ;
+ */
+ public static final String SEMICOLON = ";";
+
+
+ /**
+ * EQUAL_SIGN =
+ */
+ public static final String EQUAL_SIGN = "=";
+
+ /**
* datasource encryption salt
*/
public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*";
diff --git a/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts b/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts
index e17c6a2..ebfec18 100644
--- a/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts
+++ b/dolphinscheduler-ui-next/src/service/modules/data-source/types.ts
@@ -25,6 +25,7 @@ type IDataBase =
| 'SQLSERVER'
| 'DB2'
| 'PRESTO'
+ | 'REDSHIFT'
interface IDataSource {
id?: number
diff --git a/dolphinscheduler-ui-next/src/views/datasource/list/use-form.ts b/dolphinscheduler-ui-next/src/views/datasource/list/use-form.ts
index 1f091be..9ea39ad 100644
--- a/dolphinscheduler-ui-next/src/views/datasource/list/use-form.ts
+++ b/dolphinscheduler-ui-next/src/views/datasource/list/use-form.ts
@@ -202,6 +202,11 @@ const datasourceType: IDataBaseOptionKeys = {
value: 'PRESTO',
label: 'PRESTO',
defaultPort: 8080
+ },
+ REDSHIFT: {
+ value: 'REDSHIFT',
+ label: 'REDSHIFT',
+ defaultPort: 5439
}
}
diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource-type.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource-type.ts
index c082784..1336802 100644
--- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource-type.ts
+++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource-type.ts
@@ -75,6 +75,11 @@ export function useDatasourceType(
id: 8,
code: 'PRESTO',
disabled: false
+ },
+ {
+ id: 9,
+ code: 'REDSHIFT',
+ disabled: false
}
]
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue
index 08f95fa..d26c175 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue
@@ -279,6 +279,10 @@
{
value: 'PRESTO',
label: 'PRESTO'
+ },
+ {
+ value: 'REDSHIFT',
+ label: 'REDSHIFT'
}
]
}
@@ -490,6 +494,9 @@
case 'PRESTO':
defaultPort = '8080'
break
+ case 'REDSHIFT':
+ defaultPort = '5439'
+ break
default:
break
}
diff --git a/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js b/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js
index 6d8011f..416dccb 100644
--- a/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js
+++ b/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js
@@ -113,6 +113,11 @@ export default {
id: 8,
code: 'PRESTO',
disabled: false
+ },
+ {
+ id: 9,
+ code: 'REDSHIFT',
+ disabled: false
}
],
// Alarm interface
diff --git a/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js
index 754eb66..a8b4529 100644
--- a/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js
+++ b/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js
@@ -20,7 +20,7 @@ import io from '@/module/io'
export default {
/**
* Data source creation
- * @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER, PRESTO
+ * @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER, PRESTO, REDSHIFT
* @param "name": string,
* @param "desc": string,
* @param "parameter":string //{"address":"jdbc:hive2://192.168.220.189:10000","autoReconnect":"true","characterEncoding":"utf8","database":"default","initialTimeout":3000,"jdbcUrl":"jdbc:hive2://192.168.220.189:10000/default","maxReconnect":10,"password":"","useUnicode":true,"user":"hive"}
@@ -53,7 +53,7 @@ export default {
},
/**
* Query data source list - no paging
- * @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER, PRESTO
+ * @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER, PRESTO, REDSHIFT
*/
getDatasourcesList ({ state }, payload) {
return new Promise((resolve, reject) => {
diff --git a/pom.xml b/pom.xml
index 673d8d3..0647e3f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -435,6 +435,11 @@
<artifactId>dolphinscheduler-datasource-sqlserver</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-datasource-redshift</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>