You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by fa...@apache.org on 2023/03/07 03:44:48 UTC
[incubator-seatunnel] branch dev updated: [Fix] [Bug] Fix S3RedShift is not correct with S3 (#4291)
This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 7b72dd95a [Fix] [Bug] Fix S3RedShift is not correct with S3 (#4291)
7b72dd95a is described below
commit 7b72dd95a228aad9ebb2f51c2f5755ab2ea03d04
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Mar 7 11:44:41 2023 +0800
[Fix] [Bug] Fix S3RedShift is not correct with S3 (#4291)
---
.../seatunnel/redshift/sink/S3RedshiftFactory.java | 23 +++++++++++++++++-----
.../seatunnel/redshift/sink/S3RedshiftSink.java | 1 +
2 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/seatunnel-connectors-v2/connector-s3-redshift/src/main/java/org/apache/seatunnel/connectors/seatunnel/redshift/sink/S3RedshiftFactory.java b/seatunnel-connectors-v2/connector-s3-redshift/src/main/java/org/apache/seatunnel/connectors/seatunnel/redshift/sink/S3RedshiftFactory.java
index 0211cadbe..1f372f232 100644
--- a/seatunnel-connectors-v2/connector-s3-redshift/src/main/java/org/apache/seatunnel/connectors/seatunnel/redshift/sink/S3RedshiftFactory.java
+++ b/seatunnel-connectors-v2/connector-s3-redshift/src/main/java/org/apache/seatunnel/connectors/seatunnel/redshift/sink/S3RedshiftFactory.java
@@ -22,6 +22,7 @@ import org.apache.seatunnel.api.table.factory.Factory;
import org.apache.seatunnel.api.table.factory.TableSinkFactory;
import org.apache.seatunnel.connectors.seatunnel.file.config.BaseSinkConfig;
import org.apache.seatunnel.connectors.seatunnel.file.config.BaseSourceConfig;
+import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat;
import org.apache.seatunnel.connectors.seatunnel.file.s3.config.S3Config;
import org.apache.seatunnel.connectors.seatunnel.redshift.config.S3RedshiftConfig;
@@ -44,12 +45,24 @@ public class S3RedshiftFactory implements TableSinkFactory {
S3RedshiftConfig.JDBC_USER,
S3RedshiftConfig.JDBC_PASSWORD,
S3RedshiftConfig.EXECUTE_SQL,
- BaseSourceConfig.FILE_PATH)
- .optional(S3Config.S3_ACCESS_KEY, S3Config.S3_SECRET_KEY)
+ BaseSourceConfig.FILE_PATH,
+ S3Config.S3A_AWS_CREDENTIALS_PROVIDER)
+ .conditional(
+ S3Config.S3A_AWS_CREDENTIALS_PROVIDER,
+ S3Config.S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider,
+ S3Config.S3_ACCESS_KEY,
+ S3Config.S3_SECRET_KEY)
+ .optional(S3Config.S3_PROPERTIES)
.optional(BaseSinkConfig.FILE_FORMAT_TYPE)
- .optional(BaseSinkConfig.FILENAME_TIME_FORMAT)
- .optional(BaseSinkConfig.FIELD_DELIMITER)
- .optional(BaseSinkConfig.ROW_DELIMITER)
+ .conditional(
+ BaseSinkConfig.FILE_FORMAT_TYPE,
+ FileFormat.TEXT,
+ BaseSinkConfig.FIELD_DELIMITER,
+ BaseSinkConfig.ROW_DELIMITER)
+ .conditional(
+ BaseSinkConfig.FILE_FORMAT_TYPE,
+ FileFormat.CSV,
+ BaseSinkConfig.ROW_DELIMITER)
.optional(BaseSinkConfig.PARTITION_BY)
.optional(BaseSinkConfig.PARTITION_DIR_EXPRESSION)
.optional(BaseSinkConfig.IS_PARTITION_FIELD_WRITE_IN_FILE)
diff --git a/seatunnel-connectors-v2/connector-s3-redshift/src/main/java/org/apache/seatunnel/connectors/seatunnel/redshift/sink/S3RedshiftSink.java b/seatunnel-connectors-v2/connector-s3-redshift/src/main/java/org/apache/seatunnel/connectors/seatunnel/redshift/sink/S3RedshiftSink.java
index 97d594afa..4ccf1f8f3 100644
--- a/seatunnel-connectors-v2/connector-s3-redshift/src/main/java/org/apache/seatunnel/connectors/seatunnel/redshift/sink/S3RedshiftSink.java
+++ b/seatunnel-connectors-v2/connector-s3-redshift/src/main/java/org/apache/seatunnel/connectors/seatunnel/redshift/sink/S3RedshiftSink.java
@@ -53,6 +53,7 @@ public class S3RedshiftSink extends BaseHdfsFileSink {
CheckConfigUtil.checkAllExists(
pluginConfig,
S3Config.S3_BUCKET.key(),
+ S3Config.S3A_AWS_CREDENTIALS_PROVIDER.key(),
S3RedshiftConfig.JDBC_URL.key(),
S3RedshiftConfig.JDBC_USER.key(),
S3RedshiftConfig.JDBC_PASSWORD.key(),