You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2024/07/12 08:19:33 UTC
(flink) 01/08: [hotfix] Create ConfigOptions for s3 access/secret keys and endpoint
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 957804a95916d86623c67d79bfca011b6835cc78
Author: Piotr Nowojski <pi...@gmail.com>
AuthorDate: Sat Jan 27 10:06:24 2024 -0500
[hotfix] Create ConfigOptions for s3 access/secret keys and endpoint
---
.../fs/s3/common/AbstractS3FileSystemFactory.java | 18 ++++++++++++++++++
.../flink/fs/s3presto/PrestoS3FileSystemITCase.java | 5 +++--
.../flink/fs/s3presto/PrestoS3FileSystemTest.java | 6 ++++--
3 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/AbstractS3FileSystemFactory.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/AbstractS3FileSystemFactory.java
index 7a396d286fb..d5dd4975421 100644
--- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/AbstractS3FileSystemFactory.java
+++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/AbstractS3FileSystemFactory.java
@@ -41,6 +41,24 @@ import java.net.URI;
/** Base class for file system factories that create S3 file systems. */
public abstract class AbstractS3FileSystemFactory implements FileSystemFactory {
+ public static final ConfigOption<String> ACCESS_KEY =
+ ConfigOptions.key("s3.access-key")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("This optionally defines S3 access key.");
+
+ public static final ConfigOption<String> SECRET_KEY =
+ ConfigOptions.key("s3.secret-key")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("This optionally defines S3 secret key.");
+
+ public static final ConfigOption<String> ENDPOINT =
+ ConfigOptions.key("s3.endpoint")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("This optionally defines S3 endpoint.");
+
public static final ConfigOption<Long> PART_UPLOAD_MIN_SIZE =
ConfigOptions.key("s3.upload.min.part.size")
.longType()
diff --git a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java
index 72d093a9407..dbebf0bb392 100644
--- a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java
+++ b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java
@@ -21,6 +21,7 @@ package org.apache.flink.fs.s3presto;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.fs.s3.common.AbstractS3FileSystemFactory;
import org.apache.flink.runtime.fs.hdfs.AbstractHadoopFileSystemITTest;
import org.apache.flink.testutils.s3.S3TestCredentials;
@@ -110,8 +111,8 @@ public class PrestoS3FileSystemITCase extends AbstractHadoopFileSystemITTest {
{
Configuration conf = new Configuration();
conf.setString(S3_USE_INSTANCE_CREDENTIALS, "false");
- conf.setString("s3.access-key", S3TestCredentials.getS3AccessKey());
- conf.setString("s3.secret-key", S3TestCredentials.getS3SecretKey());
+ conf.set(AbstractS3FileSystemFactory.ACCESS_KEY, S3TestCredentials.getS3AccessKey());
+ conf.set(AbstractS3FileSystemFactory.SECRET_KEY, S3TestCredentials.getS3SecretKey());
FileSystem.initialize(conf);
path.getFileSystem().exists(path);
diff --git a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemTest.java b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemTest.java
index c6609588476..b91e2b7c515 100644
--- a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemTest.java
+++ b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemTest.java
@@ -20,6 +20,7 @@ package org.apache.flink.fs.s3presto;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.fs.s3.common.AbstractS3FileSystemFactory;
import org.apache.flink.fs.s3.common.FlinkS3FileSystem;
import org.apache.flink.fs.s3.common.token.DynamicTemporaryAWSCredentialsProvider;
import org.apache.flink.runtime.util.HadoopConfigLoader;
@@ -46,8 +47,9 @@ public class PrestoS3FileSystemTest {
@Test
public void testConfigPropagation() throws Exception {
final Configuration conf = new Configuration();
- conf.setString("s3.access-key", "test_access_key_id");
- conf.setString("s3.secret-key", "test_secret_access_key");
+
+ conf.set(AbstractS3FileSystemFactory.ACCESS_KEY, "test_access_key_id");
+ conf.set(AbstractS3FileSystemFactory.SECRET_KEY, "test_secret_access_key");
FileSystem.initialize(conf);