You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/04/21 00:35:15 UTC
(doris) branch branch-4.1 updated: branch-4.1: [opt](test) Support IAM Role for s3 client ut test #62303 (#62618)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 99c8b3820fa branch-4.1: [opt](test) Support IAM Role for s3 client ut test #62303 (#62618)
99c8b3820fa is described below
commit 99c8b3820fae08c7e19a2953b3da43e725741cd4
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Apr 21 08:35:08 2026 +0800
branch-4.1: [opt](test) Support IAM Role for s3 client ut test #62303 (#62618)
Cherry-picked from #62303
Co-authored-by: Yixuan Wang <wa...@selectdb.com>
---
be/test/io/client/s3_file_system_test.cpp | 52 ++++++++++++++++++++++++++-----
cloud/test/s3_accessor_client_test.cpp | 52 ++++++++++++++++++++++++++-----
2 files changed, 90 insertions(+), 14 deletions(-)
diff --git a/be/test/io/client/s3_file_system_test.cpp b/be/test/io/client/s3_file_system_test.cpp
index d33240485af..71034020120 100644
--- a/be/test/io/client/s3_file_system_test.cpp
+++ b/be/test/io/client/s3_file_system_test.cpp
@@ -46,6 +46,15 @@ namespace doris {
static const char* MOCK_S3_FS_ID = "mock_s3_fs_id_6";
+static std::string get_env_with_fallback(const char* primary, const char* fallback) {
+ const char* value = std::getenv(primary);
+ if (value != nullptr) {
+ return value;
+ }
+ value = std::getenv(fallback);
+ return value != nullptr ? value : "";
+}
+
class S3TestConfig {
public:
S3TestConfig() {
@@ -58,18 +67,25 @@ public:
access_key = GET_ENV_IF_DEFINED(S3_AK);
secret_key = GET_ENV_IF_DEFINED(S3_SK);
- endpoint = GET_ENV_IF_DEFINED(S3_ENDPOINT);
+ role_arn = GET_ENV_IF_DEFINED(AWS_ROLE_ARN);
+ external_id = GET_ENV_IF_DEFINED(AWS_EXTERNAL_ID);
+ endpoint = get_env_with_fallback("S3_ENDPOINT", "AWS_ENDPOINT");
provider = GET_ENV_IF_DEFINED(S3_PROVIDER);
- bucket = GET_ENV_IF_DEFINED(S3_BUCKET);
- region = GET_ENV_IF_DEFINED(S3_REGION);
- prefix = GET_ENV_IF_DEFINED(S3_PREFIX);
+ bucket = get_env_with_fallback("S3_BUCKET", "AWS_BUCKET");
+ region = get_env_with_fallback("S3_REGION", "AWS_REGION");
+ prefix = get_env_with_fallback("S3_PREFIX", "AWS_PREFIX");
}
bool is_enabled() const { return enabled; }
bool is_valid() const {
- return enabled && !access_key.empty() && !secret_key.empty() && !endpoint.empty() &&
- !region.empty() && !bucket.empty();
+ if (!enabled || endpoint.empty() || region.empty() || bucket.empty() || prefix.empty()) {
+ return false;
+ }
+ if (has_partial_static_credentials()) {
+ return false;
+ }
+ return has_static_credentials() || use_instance_profile();
}
std::string get_access_key() const { return access_key; }
@@ -79,15 +95,23 @@ public:
std::string get_bucket() const { return bucket; }
std::string get_prefix() const { return prefix; }
std::string get_region() const { return region; }
+ std::string get_role_arn() const { return role_arn; }
+ std::string get_external_id() const { return external_id; }
+ bool use_instance_profile() const { return !has_static_credentials(); }
void print_config() const {
std::cout << "S3 Test Configuration:" << std::endl;
std::cout << " Enabled: " << (enabled ? "Yes" : "No") << std::endl;
if (enabled) {
+ std::cout << " Auth Mode: " << (has_static_credentials() ? "AK/SK" : "IAM Role")
+ << std::endl;
std::cout << " Access Key: " << (access_key.empty() ? "<empty>" : "<set>")
<< std::endl;
std::cout << " Secret Key: " << (secret_key.empty() ? "<empty>" : "<hidden>")
<< std::endl;
+ std::cout << " Role Arn: " << (role_arn.empty() ? "<empty>" : role_arn) << std::endl;
+ std::cout << " External Id: " << (external_id.empty() ? "<empty>" : "<hidden>")
+ << std::endl;
std::cout << " Endpoint: " << endpoint << std::endl;
std::cout << " Provider: " << provider << std::endl;
std::cout << " Bucket: " << bucket << std::endl;
@@ -97,9 +121,17 @@ public:
}
private:
+ bool has_static_credentials() const { return !access_key.empty() && !secret_key.empty(); }
+
+ bool has_partial_static_credentials() const {
+ return !has_static_credentials() && (!access_key.empty() || !secret_key.empty());
+ }
+
bool enabled = false;
std::string access_key;
std::string secret_key;
+ std::string role_arn;
+ std::string external_id;
std::string endpoint;
std::string provider;
std::string bucket;
@@ -121,7 +153,8 @@ protected:
}
if (!config_->is_valid()) {
- GTEST_SKIP() << "S3 configuration is incomplete. Required: AK, SK, ENDPOINT, BUCKET.";
+ GTEST_SKIP() << "S3 configuration is incomplete. Required: ENDPOINT, REGION, BUCKET, "
+ "PREFIX and either AK/SK or iam role credentials.";
}
// Attempt to create S3 client
@@ -159,6 +192,11 @@ protected:
s3_conf.client_conf.bucket = config_->get_bucket();
s3_conf.client_conf.region = config_->get_region();
s3_conf.client_conf.provider = convert_provider(config_->get_provider());
+ if (config_->use_instance_profile()) {
+ s3_conf.client_conf.cred_provider_type = CredProviderType::InstanceProfile;
+ s3_conf.client_conf.role_arn = config_->get_role_arn();
+ s3_conf.client_conf.external_id = config_->get_external_id();
+ }
s3_fs_ = DORIS_TRY(io::S3FileSystem::create(s3_conf, MOCK_S3_FS_ID));
return Status::OK();
diff --git a/cloud/test/s3_accessor_client_test.cpp b/cloud/test/s3_accessor_client_test.cpp
index 9805ae2b488..f6a8802cf22 100644
--- a/cloud/test/s3_accessor_client_test.cpp
+++ b/cloud/test/s3_accessor_client_test.cpp
@@ -60,6 +60,15 @@ namespace doris::cloud {
return val ? std::string(val) : ""; \
}())
+static std::string get_env_with_fallback(const char* primary, const char* fallback) {
+ const char* value = std::getenv(primary);
+ if (value != nullptr) {
+ return value;
+ }
+ value = std::getenv(fallback);
+ return value != nullptr ? value : "";
+}
+
class S3TestConfig {
public:
S3TestConfig() {
@@ -72,18 +81,25 @@ public:
access_key = GET_ENV_IF_DEFINED(S3_AK);
secret_key = GET_ENV_IF_DEFINED(S3_SK);
- endpoint = GET_ENV_IF_DEFINED(S3_ENDPOINT);
+ role_arn = GET_ENV_IF_DEFINED(AWS_ROLE_ARN);
+ external_id = GET_ENV_IF_DEFINED(AWS_EXTERNAL_ID);
+ endpoint = get_env_with_fallback("S3_ENDPOINT", "AWS_ENDPOINT");
provider = GET_ENV_IF_DEFINED(S3_PROVIDER);
- bucket = GET_ENV_IF_DEFINED(S3_BUCKET);
- region = GET_ENV_IF_DEFINED(S3_REGION);
- prefix = GET_ENV_IF_DEFINED(S3_PREFIX);
+ bucket = get_env_with_fallback("S3_BUCKET", "AWS_BUCKET");
+ region = get_env_with_fallback("S3_REGION", "AWS_REGION");
+ prefix = get_env_with_fallback("S3_PREFIX", "AWS_PREFIX");
}
bool is_enabled() const { return enabled; }
bool is_valid() const {
- return enabled && !access_key.empty() && !secret_key.empty() && !endpoint.empty() &&
- !region.empty() && !bucket.empty();
+ if (!enabled || endpoint.empty() || region.empty() || bucket.empty() || prefix.empty()) {
+ return false;
+ }
+ if (has_partial_static_credentials()) {
+ return false;
+ }
+ return has_static_credentials() || use_instance_profile();
}
std::string get_access_key() const { return access_key; }
@@ -93,15 +109,23 @@ public:
std::string get_bucket() const { return bucket; }
std::string get_prefix() const { return prefix; }
std::string get_region() const { return region; }
+ std::string get_role_arn() const { return role_arn; }
+ std::string get_external_id() const { return external_id; }
+ bool use_instance_profile() const { return !has_static_credentials(); }
void print_config() const {
std::cout << "S3 Test Configuration:" << std::endl;
std::cout << " Enabled: " << (enabled ? "Yes" : "No") << std::endl;
if (enabled) {
+ std::cout << " Auth Mode: " << (has_static_credentials() ? "AK/SK" : "IAM Role")
+ << std::endl;
std::cout << " Access Key: " << (access_key.empty() ? "<empty>" : "<set>")
<< std::endl;
std::cout << " Secret Key: " << (secret_key.empty() ? "<empty>" : "<hidden>")
<< std::endl;
+ std::cout << " Role Arn: " << (role_arn.empty() ? "<empty>" : role_arn) << std::endl;
+ std::cout << " External Id: " << (external_id.empty() ? "<empty>" : "<hidden>")
+ << std::endl;
std::cout << " Endpoint: " << endpoint << std::endl;
std::cout << " Provider: " << provider << std::endl;
std::cout << " Bucket: " << bucket << std::endl;
@@ -111,9 +135,17 @@ public:
}
private:
+ bool has_static_credentials() const { return !access_key.empty() && !secret_key.empty(); }
+
+ bool has_partial_static_credentials() const {
+ return !has_static_credentials() && (!access_key.empty() || !secret_key.empty());
+ }
+
bool enabled = false;
std::string access_key;
std::string secret_key;
+ std::string role_arn;
+ std::string external_id;
std::string endpoint;
std::string provider;
std::string bucket;
@@ -135,7 +167,8 @@ protected:
}
if (!config_->is_valid()) {
- GTEST_SKIP() << "S3 configuration is incomplete. Required: AK, SK, ENDPOINT, BUCKET.";
+ GTEST_SKIP() << "S3 configuration is incomplete. Required: ENDPOINT, REGION, BUCKET, "
+ "PREFIX and either AK/SK or iam role credentials.";
}
// Attempt to create S3 accessor
@@ -169,6 +202,11 @@ protected:
s3_conf.bucket = config_->get_bucket();
s3_conf.prefix = config_->get_prefix();
s3_conf.provider = convert_provider(config_->get_provider());
+ if (config_->use_instance_profile()) {
+ s3_conf.cred_provider_type = CredProviderType::InstanceProfile;
+ s3_conf.role_arn = config_->get_role_arn();
+ s3_conf.external_id = config_->get_external_id();
+ }
std::shared_ptr<S3Accessor> accessor;
int ret = S3Accessor::create(s3_conf, &accessor);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org