You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/10/21 10:25:33 UTC
(doris) branch branch-4.2 updated: branch-4.2: [feat](storage) Support GCP IAM (#69293)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 3e82c4b0f34 branch-4.2: [feat](storage) Support GCP IAM (#69293)
3e82c4b0f34 is described below
commit 3e82c4b0f34805b832625bab0c6cee1363a57152
Author: hui lai <la...@selectdb.com>
AuthorDate: Wed Oct 21 18:25:23 2026 +0800
branch-4.2: [feat](storage) Support GCP IAM (#69293)
pick https://github.com/apache/doris/pull/68540
---
be/CMakeLists.txt | 10 +-
be/cmake/thirdparty.cmake | 3 +
be/src/io/fs/err_utils.cpp | 6 +-
be/src/io/fs/gcs_signed_url_provider.cpp | 211 ++++++
be/src/io/fs/gcs_signed_url_provider.h | 48 ++
be/src/io/fs/s3_file_system.cpp | 1 +
be/src/io/fs/s3_obj_storage_client.cpp | 18 +-
be/src/io/fs/s3_obj_storage_client.h | 11 +-
be/src/runtime/runtime_state.cpp | 5 +
be/src/util/obj_credential.cpp | 76 +++
be/src/util/s3_util.cpp | 83 ++-
be/src/util/s3_util.h | 8 +-
be/test/common/auth/gcp/gcs_signed_url_test.cpp | 79 +++
be/test/io/s3_client_factory_test.cpp | 295 ++++++++
cloud/CMakeLists.txt | 16 +
cloud/cmake/thirdparty.cmake | 4 +
cloud/src/common/CMakeLists.txt | 1 +
cloud/src/common/auth/obj_credential.cpp | 239 +++++++
cloud/src/common/auth/obj_credential.h | 65 ++
cloud/src/meta-service/meta_service_resource.cpp | 104 ++-
cloud/src/recycler/s3_accessor.cpp | 18 +-
cloud/src/recycler/s3_accessor.h | 3 +
cloud/test/CMakeLists.txt | 4 +
cloud/test/meta_service_test.cpp | 267 ++++++++
cloud/test/obj_credential_test.cpp | 149 ++++
cloud/test/s3_accessor_test.cpp | 39 ++
common/cpp/obj-client/auth/gcp/gcp_auth.cpp | 99 +++
common/cpp/obj-client/auth/gcp/gcp_auth.h | 60 ++
common/cpp/obj-client/auth/gcp/gcp_s3_client.cpp | 51 ++
common/cpp/obj-client/auth/gcp/gcp_s3_client.h | 55 ++
.../cpp/obj-client/auth/gcp/gcp_token_provider.cpp | 93 +++
.../cpp/obj-client/auth/gcp/gcp_token_provider.h | 57 ++
common/cpp/obj-client/auth/gcp/gcs_signed_url.cpp | 194 ++++++
common/cpp/obj-client/auth/gcp/gcs_signed_url.h | 62 ++
common/cpp/obj-client/auth/obj_credential.cpp | 134 ++++
common/cpp/obj-client/auth/obj_credential.h | 70 ++
common/cpp/obj-client/auth/obj_credential_common.h | 88 +++
.../cpp/obj-client/auth/obj_s3_client_factory.cpp | 54 ++
common/cpp/obj-client/auth/obj_s3_client_factory.h | 50 ++
fe/be-java-extensions/hadoop-deps/pom.xml | 5 +
fe/fe-core/pom.xml | 9 +
.../java/org/apache/doris/analysis/BrokerDesc.java | 14 +
.../org/apache/doris/analysis/OutFileClause.java | 5 +
.../java/org/apache/doris/catalog/S3Resource.java | 142 ++--
.../org/apache/doris/catalog/S3StorageVault.java | 7 +-
.../org/apache/doris/catalog/StorageVaultMgr.java | 72 +-
.../org/apache/doris/cloud/storage/RemoteBase.java | 28 +-
.../org/apache/doris/cloud/storage/S3Remote.java | 4 +
.../java/org/apache/doris/common/util/S3Util.java | 51 ++
.../fileio/GcpS3FileIOAwsClientFactory.java | 64 ++
.../metastore/AbstractIcebergProperties.java | 41 ++
.../storage/AbstractS3CompatibleProperties.java | 13 +
.../datasource/property/storage/GCSProperties.java | 180 ++++-
.../datasource/property/storage/S3Properties.java | 49 ++
.../datasource/property/storage/auth/GcpAuth.java | 88 +++
.../property/storage/auth/GcpCredential.java | 98 +++
.../storage/auth/GcpCredentialProviderType.java | 33 +
.../datasource/property/storage/auth/GcsAuth.java | 50 ++
.../property/storage/auth/GcsAuthResolver.java | 161 +++++
.../property/storage/auth/ObjCredential.java | 34 +
.../storage/auth/ObjCredentialFactory.java | 63 ++
.../java/org/apache/doris/fs/obj/S3ObjStorage.java | 3 +-
.../trees/plans/commands/ExportCommand.java | 6 +-
.../doris/analysis/StorageDescPersistTest.java | 21 +
.../org/apache/doris/catalog/S3ResourceTest.java | 439 ++++++++++++
.../doris/cloud/catalog/S3StorageVaultTest.java | 253 +++++++
.../doris/cloud/storage/DefaultRemoteTest.java | 18 +
.../org/apache/doris/common/util/S3UtilTest.java | 20 +
.../IcebergFileSystemMetaStorePropertiesTest.java | 117 ++++
.../property/storage/GCSPropertiesTest.java | 752 +++++++++++++++++++++
.../property/storage/S3PropertiesTest.java | 25 +
.../property/storage/auth/GcpAuthTest.java | 89 +++
.../property/storage/auth/GcsAuthResolverTest.java | 126 ++++
.../doris/nereids/trees/plans/OutFileTest.java | 104 +++
.../trees/plans/commands/ExportCommandTest.java | 116 +++-
.../trees/plans/commands/LoadCommandTest.java | 41 ++
.../ExternalFileTableValuedFunctionTest.java | 49 ++
fe/pom.xml | 20 +
gensrc/proto/cloud.proto | 20 +
gensrc/thrift/AgentService.thrift | 18 +
regression-test/conf/regression-conf.groovy | 22 +
.../apache/doris/regression/suite/Syncer.groovy | 16 +-
.../util/ObjectStorageIamTestUtils.groovy | 193 ++++++
.../test_alter_resource_with_role.groovy | 234 +++++++
.../test_alter_storage_vault_iam.groovy | 158 +++++
.../test_backup_restore_iam.groovy | 101 +++
.../test_catalog_instance_profile.groovy | 134 ++++
.../test_catalog_with_role.groovy | 123 ++++
.../object_storage_iam_p0/test_export_iam.groovy | 108 +++
.../test_external_catalog_iam.groovy | 78 +++
.../test_gcp_iam_property_validation.groovy | 131 ++++
.../test_gcp_tvf_anonymous.groovy | 51 ++
.../test_gcp_vault_anonymous_validation.groovy | 42 ++
...rg_s3tables_catalog_credentials_provider.groovy | 110 +++
.../object_storage_iam_p0/test_load_iam.groovy | 106 +++
.../object_storage_iam_p0/test_resource_iam.groovy | 134 ++++
.../test_select_into_outfile_iam.groovy | 80 +++
.../test_storage_vault_iam.groovy | 80 +++
.../test_tvf_anonymous.groovy | 77 +++
.../object_storage_iam_p0/test_tvf_iam.groovy | 66 ++
100 files changed, 8179 insertions(+), 143 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index d165c3e57b8..52ba734065c 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -580,7 +580,7 @@ set(DORIS_LINK_LIBS
)
set(absl_DIR ${THIRDPARTY_DIR}/lib/cmake/absl)
-find_package(absl)
+find_package(absl CONFIG REQUIRED)
# COMMON_THIRDPARTY are thirdparty dependencies that can run on all platform
# When adding new dependencies, If you don’t know if it can run on all platforms,
@@ -618,6 +618,14 @@ endif()
if (absl_FOUND)
set(COMMON_THIRDPARTY
${COMMON_THIRDPARTY}
+ # Public dependencies of the statically linked google-cloud-cpp common library.
+ absl::base
+ absl::memory
+ absl::optional
+ absl::span
+ absl::str_format
+ absl::time
+ absl::variant
absl::cord
absl::cord_internal
absl::cordz_functions
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index b67c8e9d0aa..19b5c1ecc1d 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -69,6 +69,9 @@ add_thirdparty(lz4)
add_thirdparty(lzo2)
add_thirdparty(thrift)
add_thirdparty(thriftnb)
+add_thirdparty(google_cloud_cpp_oauth2)
+add_thirdparty(google_cloud_cpp_rest_internal)
+add_thirdparty(google_cloud_cpp_common)
add_thirdparty(crc32c)
add_thirdparty(libevent_core LIBNAME "lib/libevent_core.a")
diff --git a/be/src/io/fs/err_utils.cpp b/be/src/io/fs/err_utils.cpp
index 0a9514605d4..3547f7b57f4 100644
--- a/be/src/io/fs/err_utils.cpp
+++ b/be/src/io/fs/err_utils.cpp
@@ -132,10 +132,12 @@ Status s3fs_error(const Aws::S3::S3Error& err, std::string_view msg) {
return Status::Error<NOT_FOUND, false>("{}: {} {} code=NOT_FOUND, type={}, request_id={}",
msg, err.GetExceptionName(), err.GetMessage(),
err.GetErrorType(), request_id);
+ case HttpResponseCode::UNAUTHORIZED:
case HttpResponseCode::FORBIDDEN:
return Status::Error<PERMISSION_DENIED, false>(
- "{}: {} {} code=FORBIDDEN, type={}, request_id={}", msg, err.GetExceptionName(),
- err.GetMessage(), err.GetErrorType(), request_id);
+ "{}: {} {} code=PERMISSION_DENIED, http_code={}, type={}, request_id={}", msg,
+ err.GetExceptionName(), err.GetMessage(), err.GetResponseCode(), err.GetErrorType(),
+ request_id);
default:
return Status::Error<ErrorCode::INTERNAL_ERROR, false>(
"{}: {} {} code={} type={}, request_id={}", msg, err.GetExceptionName(),
diff --git a/be/src/io/fs/gcs_signed_url_provider.cpp b/be/src/io/fs/gcs_signed_url_provider.cpp
new file mode 100644
index 00000000000..d69e0fa46e5
--- /dev/null
+++ b/be/src/io/fs/gcs_signed_url_provider.cpp
@@ -0,0 +1,211 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/gcs_signed_url_provider.h"
+
+#include <fmt/format.h>
+#include <rapidjson/document.h>
+
+#include <algorithm>
+#include <cstdlib>
+#include <exception>
+#include <string_view>
+#include <utility>
+
+#include "cpp/obj-client/auth/gcp/gcp_token_provider.h"
+#include "cpp/obj-client/auth/gcp/gcs_signed_url.h"
+#include "service/http/http_client.h"
+#include "util/string_util.h"
+#include "util/url_coding.h"
+
+namespace doris::io {
+namespace {
+
+constexpr std::string_view IAM_CREDENTIALS_ENDPOINT =
+ "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/";
+constexpr std::string_view DEFAULT_METADATA_HOST = "metadata.google.internal";
+constexpr std::string_view METADATA_SERVICE_ACCOUNT_EMAIL_PATH =
+ "/computeMetadata/v1/instance/service-accounts/default/email";
+constexpr int64_t METADATA_REQUEST_TIMEOUT_MS = 1000;
+
+std::string iam_error_message(const rapidjson::Document& document) {
+ if (!document.IsObject() || !document.HasMember("error") || !document["error"].IsObject()) {
+ return "unparseable error response";
+ }
+ const auto& error = document["error"];
+ std::string message;
+ if (error.HasMember("status") && error["status"].IsString()) {
+ message = error["status"].GetString();
+ }
+ if (error.HasMember("message") && error["message"].IsString()) {
+ if (!message.empty()) {
+ message.append(": ");
+ }
+ message.append(error["message"].GetString());
+ }
+ if (message.empty()) {
+ return "error response did not contain status or message";
+ }
+ constexpr size_t MAX_ERROR_MESSAGE_SIZE = 1024;
+ if (message.size() > MAX_ERROR_MESSAGE_SIZE) {
+ message.resize(MAX_ERROR_MESSAGE_SIZE);
+ message.append("...");
+ }
+ return message;
+}
+
+bool is_unreserved(unsigned char c) {
+ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' ||
+ c == '_' || c == '.' || c == '~';
+}
+
+std::string percent_encode(std::string_view value) {
+ constexpr char HEX[] = "0123456789ABCDEF";
+ std::string encoded;
+ encoded.reserve(value.size());
+ for (unsigned char c : value) {
+ if (is_unreserved(c)) {
+ encoded.push_back(static_cast<char>(c));
+ continue;
+ }
+ encoded.push_back('%');
+ encoded.push_back(HEX[c >> 4]);
+ encoded.push_back(HEX[c & 0x0F]);
+ }
+ return encoded;
+}
+
+Status call_iam_sign_blob(std::string_view access_token, std::string_view service_account,
+ std::string_view string_to_sign, int64_t request_timeout_ms,
+ std::string* signature) {
+ std::string encoded_payload;
+ base64_encode(std::string(string_to_sign), &encoded_payload);
+ std::string request_body = fmt::format(R"({{"payload":"{}"}})", encoded_payload);
+ std::string endpoint =
+ std::string(IAM_CREDENTIALS_ENDPOINT) + percent_encode(service_account) + ":signBlob";
+
+ HttpClient client;
+ // Keep the response body for non-2xx replies so IAM permission and
+ // service-account errors are actionable to operators.
+ RETURN_IF_ERROR(client.init(endpoint, false));
+ client.set_authorization("Bearer " + std::string(access_token));
+ client.set_content_type("application/json");
+ client.set_timeout_ms(request_timeout_ms > 0 ? request_timeout_ms : 10000);
+
+ std::string response;
+ RETURN_IF_ERROR(client.execute_post_request(request_body, &response));
+
+ rapidjson::Document document;
+ document.Parse(response.data(), response.size());
+ const auto http_status = client.get_http_status();
+ if (http_status < 200 || http_status >= 300) {
+ return Status::HttpError("IAM signBlob failed with HTTP {}: {}", http_status,
+ iam_error_message(document));
+ }
+ if (document.HasParseError() || !document.IsObject() || !document.HasMember("signedBlob") ||
+ !document["signedBlob"].IsString()) {
+ return Status::InternalError("IAM signBlob returned an invalid response");
+ }
+ if (!base64_decode(document["signedBlob"].GetString(), signature)) {
+ return Status::InternalError("IAM signBlob returned an invalid base64 signature");
+ }
+ return Status::OK();
+}
+
+Status fetch_metadata_service_account_email(int64_t request_timeout_ms, std::string* email) {
+ const char* configured_host = std::getenv("GCE_METADATA_HOST");
+ std::string_view metadata_host = configured_host != nullptr && configured_host[0] != '\0'
+ ? configured_host
+ : DEFAULT_METADATA_HOST;
+ HttpClient client;
+ RETURN_IF_ERROR(client.init(
+ fmt::format("http://{}{}", metadata_host, METADATA_SERVICE_ACCOUNT_EMAIL_PATH)));
+ client.set_header("Metadata-Flavor", "Google");
+ client.set_timeout_ms(request_timeout_ms > 0
+ ? std::min(request_timeout_ms, METADATA_REQUEST_TIMEOUT_MS)
+ : METADATA_REQUEST_TIMEOUT_MS);
+
+ std::string response;
+ RETURN_IF_ERROR(client.execute(&response));
+ auto resolved_email = trim(response);
+ if (!is_valid_gcp_service_account_email(resolved_email)) {
+ return Status::InternalError(
+ "GCP metadata server returned an invalid service account email");
+ }
+ email->assign(resolved_email);
+ return Status::OK();
+}
+
+} // namespace
+
+Status generate_gcs_v4_signed_url(const GcsV4SignedUrlProviderOptions& options,
+ const GcpCredentialConfig& credential,
+ const std::shared_ptr<GcpTokenProvider>& token_provider,
+ std::string* signed_url) {
+ if (signed_url == nullptr) {
+ return Status::InvalidArgument("signed_url output must not be null");
+ }
+ signed_url->clear();
+
+ std::string signer_email;
+ if (!credential.impersonation_service_account.empty()) {
+ signer_email = credential.impersonation_service_account;
+ } else if (credential.provider_type == GcpCredentialProviderType::ComputeEngine) {
+ RETURN_IF_ERROR(
+ fetch_metadata_service_account_email(options.request_timeout_ms, &signer_email));
+ } else {
+ return Status::InvalidArgument(
+ "GCS V4 signing with DEFAULT credentials requires "
+ "gs.impersonation_service_account; use COMPUTE_ENGINE to resolve the VM service "
+ "account from metadata");
+ }
+ if (token_provider == nullptr) {
+ return Status::InternalError("GCS V4 signing token provider is not initialized");
+ }
+
+ try {
+ auto token = token_provider->get_token();
+ if (!token.has_value()) {
+ return Status::InternalError("failed to obtain OAuth token for IAM signBlob");
+ }
+
+ auto result = build_gcs_v4_signed_url(
+ {.endpoint = options.endpoint,
+ .bucket = options.bucket,
+ .key = options.key,
+ .signer_email = signer_email,
+ .expiration_secs = options.expiration_secs},
+ std::chrono::system_clock::now(), [&](std::string_view string_to_sign) {
+ std::string signature;
+ auto status = call_iam_sign_blob(*token, signer_email, string_to_sign,
+ options.request_timeout_ms, &signature);
+ if (!status.ok()) {
+ return GcsSignBlobResult {.error = status.to_string()};
+ }
+ return GcsSignBlobResult {.signature = std::move(signature)};
+ });
+ if (!result.ok()) {
+ return Status::InternalError("failed to build GCS V4 signed URL: {}", result.error);
+ }
+ *signed_url = std::move(result.signed_url);
+ return Status::OK();
+ } catch (const std::exception& e) {
+ return Status::InternalError("failed to initialize GCS URL signer: {}", e.what());
+ }
+}
+
+} // namespace doris::io
diff --git a/be/src/io/fs/gcs_signed_url_provider.h b/be/src/io/fs/gcs_signed_url_provider.h
new file mode 100644
index 00000000000..771a247bf2d
--- /dev/null
+++ b/be/src/io/fs/gcs_signed_url_provider.h
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "common/status.h"
+#include "cpp/obj-client/auth/gcp/gcp_auth.h"
+
+namespace doris {
+class GcpTokenProvider;
+}
+
+namespace doris::io {
+
+struct GcsV4SignedUrlProviderOptions {
+ std::string endpoint;
+ std::string bucket;
+ std::string key;
+ int64_t expiration_secs = 0;
+ int64_t request_timeout_ms = 10000;
+};
+
+// Resolves the signing service account, obtains an OAuth token, and invokes
+// IAM Credentials signBlob before delegating URL construction to common auth.
+Status generate_gcs_v4_signed_url(const GcsV4SignedUrlProviderOptions& options,
+ const GcpCredentialConfig& credential,
+ const std::shared_ptr<GcpTokenProvider>& token_provider,
+ std::string* signed_url);
+
+} // namespace doris::io
diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp
index 45251eab6e9..62dea5f8c2a 100644
--- a/be/src/io/fs/s3_file_system.cpp
+++ b/be/src/io/fs/s3_file_system.cpp
@@ -102,6 +102,7 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) {
reset_conf.role_arn = conf.role_arn;
reset_conf.external_id = conf.external_id;
reset_conf.cred_provider_type = conf.cred_provider_type;
+ reset_conf.credential = conf.credential;
// Compare full-field equality of the merged conf, not get_hash(): the hash is
// an XOR of crc32s and distinct configurations can collide, which would skip a
diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp
index f553de8ca48..92b56b7b278 100644
--- a/be/src/io/fs/s3_obj_storage_client.cpp
+++ b/be/src/io/fs/s3_obj_storage_client.cpp
@@ -69,6 +69,7 @@
#include "cpp/obj_retry_strategy.h"
#include "cpp/sync_point.h"
#include "io/fs/err_utils.h"
+#include "io/fs/gcs_signed_url_provider.h"
#include "io/fs/s3_common.h"
#include "util/bvar_helper.h"
@@ -489,7 +490,22 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively(
std::string S3ObjStorageClient::generate_presigned_url(const ObjectStoragePathOptions& opts,
int64_t expiration_secs,
- const S3ClientConf&) {
+ const S3ClientConf& conf) {
+ if (const auto* gcp_credential = std::get_if<GcpCredentialConfig>(&conf.credential)) {
+ std::string signed_url;
+ auto status = generate_gcs_v4_signed_url({.endpoint = conf.endpoint,
+ .bucket = opts.bucket,
+ .key = opts.key,
+ .expiration_secs = expiration_secs,
+ .request_timeout_ms = conf.request_timeout_ms},
+ *gcp_credential, _gcs_signing_token_provider,
+ &signed_url);
+ if (!status.ok()) {
+ LOG(WARNING) << "failed to generate GCS V4 signed URL: " << status;
+ return {};
+ }
+ return signed_url;
+ }
return _client->GeneratePresignedUrl(opts.bucket, opts.key, Aws::Http::HttpMethod::HTTP_GET,
expiration_secs);
}
diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h
index 45294226594..532ef7058de 100644
--- a/be/src/io/fs/s3_obj_storage_client.h
+++ b/be/src/io/fs/s3_obj_storage_client.h
@@ -27,12 +27,20 @@ class CompletedPart;
}
} // namespace Aws::S3
+namespace doris {
+class GcpTokenProvider;
+}
+
namespace doris::io {
class ObjClientHolder;
class S3ObjStorageClient final : public ObjStorageClient {
public:
- S3ObjStorageClient(std::shared_ptr<Aws::S3::S3Client> client) : _client(std::move(client)) {}
+ explicit S3ObjStorageClient(
+ std::shared_ptr<Aws::S3::S3Client> client,
+ std::shared_ptr<GcpTokenProvider> gcs_signing_token_provider = nullptr)
+ : _client(std::move(client)),
+ _gcs_signing_token_provider(std::move(gcs_signing_token_provider)) {}
~S3ObjStorageClient() override = default;
ObjectStorageUploadResponse create_multipart_upload(
const ObjectStoragePathOptions& opts) override;
@@ -58,6 +66,7 @@ public:
private:
std::shared_ptr<Aws::S3::S3Client> _client;
+ std::shared_ptr<GcpTokenProvider> _gcs_signing_token_provider;
};
} // namespace doris::io
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index 0054dd0aa82..48bbdd06231 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -485,6 +485,11 @@ std::string RuntimeState::get_error_log_file_path() {
auto presigned_url =
s3_error_fs->generate_presigned_url(remote_error_log_file_path, EXPIRATION_SECONDS,
config::use_public_endpoint_for_error_log);
+ if (presigned_url.empty()) {
+ LOG(WARNING) << "Fail to generate presigned URL for S3 error log; retain local path "
+ << local_error_log_file_path;
+ return local_error_log_file_path;
+ }
std::lock_guard<std::mutex> load_lock(_load_error_log_lock);
_error_log_file_path = std::move(presigned_url);
return _error_log_file_path;
diff --git a/be/src/util/obj_credential.cpp b/be/src/util/obj_credential.cpp
new file mode 100644
index 00000000000..6308350c0e4
--- /dev/null
+++ b/be/src/util/obj_credential.cpp
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cpp/obj-client/auth/obj_credential.h"
+
+#include <gen_cpp/AgentService_types.h>
+#include <gen_cpp/cloud.pb.h>
+
+namespace doris {
+namespace {
+
+GcpCredentialConfig to_gcp_credential_config(const cloud::GcpCredentialPB& credential) {
+ GcpCredentialConfig config;
+ switch (credential.credential_provider_type()) {
+ case cloud::GcpCredentialPB::DEFAULT:
+ config.provider_type = GcpCredentialProviderType::Default;
+ break;
+ case cloud::GcpCredentialPB::COMPUTE_ENGINE:
+ config.provider_type = GcpCredentialProviderType::ComputeEngine;
+ break;
+ }
+ config.impersonation_service_account = credential.impersonation_service_account();
+ return config;
+}
+
+GcpCredentialConfig to_gcp_credential_config(const TGcpCredential& credential) {
+ GcpCredentialConfig config;
+ if (credential.__isset.credential_provider_type) {
+ switch (credential.credential_provider_type) {
+ case TGcpCredentialProviderType::DEFAULT:
+ config.provider_type = GcpCredentialProviderType::Default;
+ break;
+ case TGcpCredentialProviderType::COMPUTE_ENGINE:
+ config.provider_type = GcpCredentialProviderType::ComputeEngine;
+ break;
+ }
+ }
+ if (credential.__isset.impersonation_service_account) {
+ config.impersonation_service_account = credential.impersonation_service_account;
+ }
+ return config;
+}
+
+} // namespace
+
+void convert_obj_credential(const cloud::ObjectStoreInfoPB& source, CredentialConfig* credential) {
+ if (!source.has_credential() || !source.credential().has_gcp_credential()) {
+ *credential = std::monostate {};
+ return;
+ }
+ *credential = to_gcp_credential_config(source.credential().gcp_credential());
+}
+
+void convert_obj_credential(const TCredential& source, CredentialConfig* credential) {
+ if (!source.__isset.gcp_credential) {
+ *credential = std::monostate {};
+ return;
+ }
+ *credential = to_gcp_credential_config(source.gcp_credential);
+}
+
+} // namespace doris
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index 0cae6f74cc2..f069308ee78 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -53,6 +53,9 @@
#include "common/status.h"
#include "cpp/aws_logger.h"
#include "cpp/custom_aws_credentials_provider_chain.h"
+#include "cpp/obj-client/auth/gcp/gcp_token_provider.h"
+#include "cpp/obj-client/auth/obj_credential.h"
+#include "cpp/obj-client/auth/obj_s3_client_factory.h"
#include "cpp/obj_retry_strategy.h"
#include "cpp/sync_point.h"
#include "cpp/util.h"
@@ -83,6 +86,32 @@ bvar::LatencyRecorder s3_copy_object_latency("s3_copy_object");
namespace {
+// Adapter between the IO storage-domain enum and the credential layer. Keep it
+// local: neither layer should own or depend on the other's enum definition.
+ObjCredentialProvider to_obj_credential_provider(io::ObjStorageType provider) {
+ switch (provider) {
+ case io::ObjStorageType::AWS:
+ return ObjCredentialProvider::Aws;
+ case io::ObjStorageType::AZURE:
+ return ObjCredentialProvider::Azure;
+ case io::ObjStorageType::BOS:
+ return ObjCredentialProvider::Bos;
+ case io::ObjStorageType::COS:
+ return ObjCredentialProvider::Cos;
+ case io::ObjStorageType::GCP:
+ return ObjCredentialProvider::Gcp;
+ case io::ObjStorageType::OBS:
+ return ObjCredentialProvider::Obs;
+ case io::ObjStorageType::OSS:
+ return ObjCredentialProvider::Oss;
+ case io::ObjStorageType::TOS:
+ return ObjCredentialProvider::Tos;
+ case io::ObjStorageType::UNKNOWN:
+ return ObjCredentialProvider::Unknown;
+ }
+ return ObjCredentialProvider::Unknown;
+}
+
doris::Status is_s3_conf_valid(const S3ClientConf& conf) {
if (conf.endpoint.empty()) {
return Status::InvalidArgument<false>("Invalid s3 conf, empty endpoint");
@@ -91,6 +120,20 @@ doris::Status is_s3_conf_valid(const S3ClientConf& conf) {
return Status::InvalidArgument<false>("Invalid s3 conf, empty region");
}
+ ObjCredentialValidationContext credential_context {
+ .provider = to_obj_credential_provider(conf.provider),
+ .ak = conf.ak,
+ .sk = conf.sk,
+ .token = conf.token,
+ .has_aws_role_arn = !conf.role_arn.empty(),
+ .has_aws_external_id = !conf.external_id.empty(),
+ .has_aws_credential_provider = conf.cred_provider_type != CredProviderType::Default,
+ };
+ if (auto error = validate_obj_credential_config(conf.credential, credential_context);
+ error.has_value()) {
+ return Status::InvalidArgument<false>(*error);
+ }
+
if (conf.role_arn.empty()) {
// Allow anonymous access when both ak and sk are empty
bool hasAk = !conf.ak.empty();
@@ -143,6 +186,7 @@ std::string build_azure_tls_debug_context(const std::string& selected_ca_file) {
constexpr char USE_PATH_STYLE[] = "use_path_style";
constexpr char AZURE_PROVIDER_STRING[] = "AZURE";
+constexpr char GCP_PROVIDER_STRING[] = "GCP";
constexpr char S3_PROVIDER[] = "provider";
constexpr char S3_AK[] = "AWS_ACCESS_KEY";
constexpr char S3_SK[] = "AWS_SECRET_KEY";
@@ -449,12 +493,30 @@ std::shared_ptr<io::ObjStorageClient> S3ClientFactory::_create_s3_client(
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
config::max_s3_client_retry /*scaleFactor = 25*/, /*retry_slow_down=*/true);
- std::shared_ptr<Aws::S3::S3Client> new_client = std::make_shared<Aws::S3::S3Client>(
- get_aws_credentials_provider(s3_conf), std::move(aws_config),
- Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
- s3_conf.use_virtual_addressing);
+ auto new_client = make_s3_client(
+ s3_conf.credential,
+ S3ClientBuildContext {
+ .fallback_provider = get_aws_credentials_provider(s3_conf),
+ .config = std::move(aws_config),
+ .payload_signing_policy =
+ Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
+ .use_virtual_addressing = s3_conf.use_virtual_addressing,
+ .ca_cert_path = _ca_cert_file_path,
+ });
- auto obj_client = std::make_shared<io::S3ObjStorageClient>(std::move(new_client));
+ std::shared_ptr<GcpTokenProvider> gcs_signing_token_provider;
+ if (const auto* gcp_credential = std::get_if<GcpCredentialConfig>(&s3_conf.credential)) {
+ // signBlob is authorized by the source identity, not by the optional
+ // impersonated target identity. Keep one refreshable cloud-platform
+ // token provider for the lifetime of the S3 client.
+ auto signing_caller_credential = *gcp_credential;
+ signing_caller_credential.impersonation_service_account.clear();
+ gcs_signing_token_provider = std::make_shared<GcpTokenProvider>(
+ signing_caller_credential, _ca_cert_file_path, GcpTokenScope::CloudPlatform);
+ }
+
+ auto obj_client = std::make_shared<io::S3ObjStorageClient>(
+ std::move(new_client), std::move(gcs_signing_token_provider));
LOG_INFO("create one s3 client with {}", s3_conf.to_string());
return obj_client;
}
@@ -501,6 +563,8 @@ Status S3ClientFactory::convert_properties_to_s3_conf(
// S3 Provider properties should be case insensitive.
if (0 == strcasecmp(it->second.c_str(), AZURE_PROVIDER_STRING)) {
s3_conf->client_conf.provider = io::ObjStorageType::AZURE;
+ } else if (0 == strcasecmp(it->second.c_str(), GCP_PROVIDER_STRING)) {
+ s3_conf->client_conf.provider = io::ObjStorageType::GCP;
}
}
@@ -533,6 +597,11 @@ Status S3ClientFactory::convert_properties_to_s3_conf(
s3_conf->client_conf.cred_provider_type = cred_provider_type_from_string(it->second);
}
+ if (auto error = parse_obj_credential_properties(properties, &s3_conf->client_conf.credential);
+ error.has_value()) {
+ return Status::InvalidArgument(*error);
+ }
+
if (auto st = is_s3_conf_valid(s3_conf->client_conf); !st.ok()) {
return st;
}
@@ -592,6 +661,7 @@ S3Conf S3Conf::get_s3_conf(const cloud::ObjectStoreInfoPB& info) {
if (info.has_cred_provider_type()) {
ret.client_conf.cred_provider_type = cred_provider_type_from_pb(info.cred_provider_type());
}
+ convert_obj_credential(info, &ret.client_conf.credential);
io::ObjStorageType type = io::ObjStorageType::AWS;
switch (info.provider()) {
@@ -653,6 +723,9 @@ S3Conf S3Conf::get_s3_conf(const TS3StorageParam& param) {
ret.client_conf.cred_provider_type =
cred_provider_type_from_thrift(param.cred_provider_type);
}
+ if (param.__isset.credential) {
+ convert_obj_credential(param.credential, &ret.client_conf.credential);
+ }
io::ObjStorageType type = io::ObjStorageType::AWS;
switch (param.provider) {
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index 9132632c8b0..ba836569c01 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -35,6 +35,7 @@
#include "common/status.h"
#include "core/string_ref.h"
#include "cpp/aws_common.h"
+#include "cpp/obj-client/auth/obj_credential.h"
#include "cpp/token_bucket_rate_limiter.h"
#include "io/fs/obj_storage_client.h"
@@ -84,6 +85,7 @@ struct S3ClientConf {
CredProviderType cred_provider_type = CredProviderType::Default;
std::string role_arn;
std::string external_id;
+ CredentialConfig credential {};
// True when this client is bound to a Doris internal object storage bucket
// (a storage vault in cloud mode). S3ClientFactory wraps such clients with the
// shared rate limiter; external buckets (S3 load, TVF, external catalogs) are
@@ -112,6 +114,7 @@ struct S3ClientConf {
hash_code ^= static_cast<int>(cred_provider_type);
hash_code ^= crc32_hash(role_arn);
hash_code ^= crc32_hash(external_id);
+ hash_combine_credential(&hash_code, credential);
hash_code ^= is_internal_bucket;
return hash_code;
}
@@ -120,10 +123,11 @@ struct S3ClientConf {
return fmt::format(
"(ak={}, token={}, endpoint={}, region={}, bucket={}, max_connections={}, "
"request_timeout_ms={}, connect_timeout_ms={}, use_virtual_addressing={}, "
- "cred_provider_type={},role_arn={}, external_id={}, is_internal_bucket={}",
+ "cred_provider_type={},role_arn={}, external_id={}, credential_type={}, "
+ "is_internal_bucket={}",
hide_access_key(ak), token, endpoint, region, bucket, max_connections,
request_timeout_ms, connect_timeout_ms, use_virtual_addressing, cred_provider_type,
- role_arn, external_id, is_internal_bucket);
+ role_arn, external_id, credential_type_name(credential), is_internal_bucket);
}
};
diff --git a/be/test/common/auth/gcp/gcs_signed_url_test.cpp b/be/test/common/auth/gcp/gcs_signed_url_test.cpp
new file mode 100644
index 00000000000..f4a6292ae23
--- /dev/null
+++ b/be/test/common/auth/gcp/gcs_signed_url_test.cpp
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cpp/obj-client/auth/gcp/gcs_signed_url.h"
+
+#include <gtest/gtest.h>
+
+#include <chrono>
+#include <string>
+
+namespace doris {
+namespace {
+
+TEST(GcsSignedUrlTest, BuildsV4UrlAndPassesCanonicalStringToSigner) {
+ GcsV4SignedUrlOptions options {
+ .endpoint = "storage.googleapis.com/",
+ .bucket = "test-bucket",
+ .key = "error_log/a b+%.csv",
+ .signer_email = "signer@my-project.iam.gserviceaccount.com",
+ .expiration_secs = 60,
+ };
+ std::string captured_string_to_sign;
+ auto result = build_gcs_v4_signed_url(
+ options, std::chrono::system_clock::time_point {},
+ [&](std::string_view string_to_sign) {
+ captured_string_to_sign = string_to_sign;
+ return GcsSignBlobResult {.signature = std::string("\x00\xab\xff", 3)};
+ });
+
+ ASSERT_TRUE(result.ok()) << result.error;
+ EXPECT_EQ(result.signed_url,
+ "https://storage.googleapis.com/test-bucket/error_log/a%20b%2B%25.csv?"
+ "X-Goog-Algorithm=GOOG4-RSA-SHA256&"
+ "X-Goog-Credential=signer%40my-project.iam.gserviceaccount.com%2F19700101%2Fauto%"
+ "2Fstorage%2Fgoog4_request&X-Goog-Date=19700101T000000Z&X-Goog-Expires=60&"
+ "X-Goog-SignedHeaders=host&X-Goog-Signature=00abff");
+
+ EXPECT_EQ(captured_string_to_sign,
+ "GOOG4-RSA-SHA256\n19700101T000000Z\n"
+ "19700101/auto/storage/goog4_request\n"
+ "f741db2e699530357ae34622ae689c7ec43e10e418af0ff7f1487beb1f144ce5");
+}
+
+TEST(GcsSignedUrlTest, RejectsInvalidExpirationBeforeSigning) {
+ GcsV4SignedUrlOptions options {
+ .endpoint = "storage.googleapis.com",
+ .bucket = "test-bucket",
+ .key = "error_log/file",
+ .signer_email = "signer@my-project.iam.gserviceaccount.com",
+ .expiration_secs = 7 * 24 * 60 * 60 + 1,
+ };
+ bool signer_called = false;
+ auto result = build_gcs_v4_signed_url(options, std::chrono::system_clock::time_point {},
+ [&](std::string_view) {
+ signer_called = true;
+ return GcsSignBlobResult {};
+ });
+
+ EXPECT_FALSE(result.ok());
+ EXPECT_FALSE(signer_called);
+ EXPECT_TRUE(result.signed_url.empty());
+}
+
+} // namespace
+} // namespace doris
diff --git a/be/test/io/s3_client_factory_test.cpp b/be/test/io/s3_client_factory_test.cpp
index fc6413a0f04..74299eea30a 100644
--- a/be/test/io/s3_client_factory_test.cpp
+++ b/be/test/io/s3_client_factory_test.cpp
@@ -18,20 +18,28 @@
#include <aws/core/auth/AWSCredentialsProviderChain.h>
#include <aws/core/auth/GeneralHTTPCredentialsProvider.h>
#include <aws/core/auth/STSCredentialsProvider.h>
+#include <aws/core/auth/signer/AWSAuthV4Signer.h>
#include <aws/core/client/ClientConfiguration.h>
+#include <aws/core/http/HttpClientFactory.h>
+#include <aws/core/utils/stream/ResponseStream.h>
#include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h>
+#include <aws/s3/model/GetObjectRequest.h>
#include <gtest/gtest.h>
#include <cstdio>
#include <cstdlib>
#include <memory>
#include <string>
+#include <string_view>
#include <utility>
#include <vector>
#include "cloud/config.h"
#include "cpp/aws_common.h"
#include "cpp/custom_aws_credentials_provider_chain.h"
+#include "cpp/obj-client/auth/gcp/gcp_auth.h"
+#include "cpp/obj-client/auth/gcp/gcp_s3_client.h"
+#include "io/fs/err_utils.h"
#include "io/fs/rate_limited_obj_storage_client.h"
#include "io/fs/s3_obj_storage_client.h"
#include "testutil/container_credentials_endpoint.h"
@@ -82,6 +90,30 @@ S3ClientConf make_hash_collision_conf(std::string endpoint, bool is_internal_buc
return conf;
}
+S3ClientConf make_gcp_native_conf(std::string impersonation_service_account = {}) {
+ auto conf = make_factory_conf("storage.googleapis.com", true);
+ conf.provider = io::ObjStorageType::GCP;
+ conf.region = "us-east1";
+ conf.cred_provider_type = CredProviderType::Default;
+ conf.credential = GcpCredentialConfig {
+ .provider_type = GcpCredentialProviderType::Default,
+ .impersonation_service_account = std::move(impersonation_service_account),
+ };
+ return conf;
+}
+
+class FixedTokenGcpS3Client final : public GcpS3Client {
+public:
+ FixedTokenGcpS3Client()
+ : GcpS3Client(GcpCredentialConfig {}, "", Aws::Client::ClientConfiguration {},
+ Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, true) {}
+
+ using GcpS3Client::BuildHttpRequest;
+
+protected:
+ std::optional<std::string> fetch_token() const override { return "test-gcp-token"; }
+};
+
} // namespace
TEST_F(S3ClientFactoryTest, WrapsAllClientsInNonCloudMode) {
@@ -150,6 +182,119 @@ TEST_F(S3ClientFactoryTest, ObjClientHolderResetDistinguishesHashCollisions) {
EXPECT_EQ(holder.s3_client_conf(), internal_conf);
}
+TEST_F(S3ClientFactoryTest, ObjClientHolderResetRefreshesNativeCredential) {
+ auto initial_conf = make_gcp_native_conf();
+ auto rotated_conf = make_gcp_native_conf("rotated@my-project.iam.gserviceaccount.com");
+ std::vector<S3ClientConf> created_confs;
+ auto client = std::make_shared<io::S3ObjStorageClient>(std::shared_ptr<Aws::S3::S3Client> {});
+ S3ClientFactory::instance().set_client_creator_for_test(
+ [&](const S3ClientConf& conf) -> std::shared_ptr<io::ObjStorageClient> {
+ created_confs.push_back(conf);
+ return client;
+ });
+
+ io::ObjClientHolder holder(initial_conf);
+ ASSERT_TRUE(holder.init().ok());
+ ASSERT_TRUE(holder.reset(rotated_conf).ok());
+
+ ASSERT_EQ(created_confs.size(), 2);
+ EXPECT_EQ(created_confs.back(), rotated_conf);
+ EXPECT_EQ(holder.s3_client_conf(), rotated_conf);
+}
+
+TEST_F(S3ClientFactoryTest, ObjClientHolderResetSwitchesNativeAndStaticCredentials) {
+ auto native_conf = make_gcp_native_conf();
+ auto static_conf = native_conf;
+ static_conf.credential = std::monostate {};
+ static_conf.ak = "access-key";
+ static_conf.sk = "secret-key";
+
+ std::vector<S3ClientConf> created_confs;
+ auto client = std::make_shared<io::S3ObjStorageClient>(std::shared_ptr<Aws::S3::S3Client> {});
+ S3ClientFactory::instance().set_client_creator_for_test(
+ [&](const S3ClientConf& conf) -> std::shared_ptr<io::ObjStorageClient> {
+ created_confs.push_back(conf);
+ return client;
+ });
+
+ io::ObjClientHolder holder(native_conf);
+ ASSERT_TRUE(holder.init().ok());
+ ASSERT_TRUE(holder.reset(static_conf).ok());
+ ASSERT_TRUE(std::holds_alternative<std::monostate>(holder.s3_client_conf().credential));
+ EXPECT_EQ(holder.s3_client_conf().ak, "access-key");
+
+ ASSERT_TRUE(holder.reset(native_conf).ok());
+ ASSERT_EQ(created_confs.size(), 3);
+ EXPECT_EQ(created_confs[1], static_conf);
+ EXPECT_EQ(created_confs[2], native_conf);
+ EXPECT_EQ(holder.s3_client_conf(), native_conf);
+}
+
+TEST_F(S3ClientFactoryTest, CredentialTypeNamesAreExplicit) {
+ CredentialConfig none;
+ CredentialConfig gcp = GcpCredentialConfig {};
+
+ EXPECT_EQ(credential_type_name(none), "none");
+ EXPECT_EQ(credential_type_name(gcp), "gcp");
+}
+
+TEST_F(S3ClientFactoryTest, GcpBearerHeaderSurvivesAnonymousAwsSigning) {
+ FixedTokenGcpS3Client client;
+ Aws::S3::Model::GetObjectRequest request;
+ request.SetBucket("bucket");
+ request.SetKey("key");
+ auto http_request = Aws::Http::CreateHttpRequest(
+ Aws::Http::URI("https://storage.googleapis.com/bucket/key"),
+ Aws::Http::HttpMethod::HTTP_GET,
+ Aws::Utils::Stream::DefaultResponseStreamFactoryMethod);
+
+ client.BuildHttpRequest(request, http_request);
+ EXPECT_EQ(http_request->GetHeaderValue("Authorization"), "Bearer test-gcp-token");
+
+ // AWSAuthV4Signer reads this before it checks for anonymous credentials.
+ http_request->SetServiceSpecificParameters(
+ std::make_shared<Aws::Http::ServiceSpecificParameters>());
+ Aws::Client::AWSAuthV4Signer signer(
+ std::make_shared<Aws::Auth::AnonymousAWSCredentialsProvider>(), "s3", "us-east-1");
+ EXPECT_TRUE(signer.SignRequest(*http_request));
+ EXPECT_EQ(http_request->GetHeaderValue("Authorization"), "Bearer test-gcp-token");
+}
+
+TEST_F(S3ClientFactoryTest, UnauthorizedObjectStoreResponseIsPermissionDenied) {
+ Aws::S3::S3Error error;
+ error.SetResponseCode(Aws::Http::HttpResponseCode::UNAUTHORIZED);
+ error.SetExceptionName("InvalidAuthenticationCredentials");
+ error.SetMessage("missing or invalid bearer token");
+
+ auto status = io::s3fs_error(error, "GCS request failed");
+ EXPECT_EQ(status.code(), ErrorCode::PERMISSION_DENIED);
+}
+
+TEST_F(S3ClientFactoryTest, GcpServiceAccountEmailValidation) {
+ const std::vector<std::string_view> valid_emails = {
+ "target@my-project.iam.gserviceaccount.com",
+ "123456789-compute@developer.gserviceaccount.com",
+ "my-project@appspot.gserviceaccount.com",
+ };
+ for (auto email : valid_emails) {
+ EXPECT_TRUE(is_valid_gcp_service_account_email(email)) << email;
+ }
+
+ const std::vector<std::string_view> invalid_emails = {
+ "",
+ "@my-project.iam.gserviceaccount.com",
+ "target@.iam.gserviceaccount.com",
+ "target@@my-project.iam.gserviceaccount.com",
+ "target@evil.developer.gserviceaccount.com",
+ "target@evil.appspot.gserviceaccount.com",
+ "target@my-project.iam.gserviceaccount.com.evil.example",
+ "target account@my-project.iam.gserviceaccount.com",
+ };
+ for (auto email : invalid_emails) {
+ EXPECT_FALSE(is_valid_gcp_service_account_email(email)) << email;
+ }
+}
+
TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) {
S3ClientFactory& factory = S3ClientFactory::instance();
S3ClientConf anonymous_conf;
@@ -320,6 +465,156 @@ TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfProviderTypeMatrix) {
}
}
+TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfGcpCredentialModes) {
+ S3URI s3_uri("s3://test-bucket/test-prefix");
+ ASSERT_TRUE(s3_uri.parse().ok());
+
+ std::map<std::string, std::string> base_properties {
+ {"AWS_ENDPOINT", "storage.googleapis.com"},
+ {"AWS_REGION", "us-east1"},
+ {"provider", "GCP"},
+ };
+
+ struct TestCase {
+ const char* provider_type;
+ const char* impersonation_service_account;
+ GcpCredentialProviderType expected_provider_type;
+ };
+
+ // Both ADC and Compute Engine metadata credentials may be used directly or
+ // as the source credential for service-account impersonation.
+ std::vector<TestCase> cases = {
+ {"DEFAULT", "", GcpCredentialProviderType::Default},
+ {"default", "", GcpCredentialProviderType::Default},
+ {"DEFAULT", "target@my-project.iam.gserviceaccount.com",
+ GcpCredentialProviderType::Default},
+ {"COMPUTE_ENGINE", "", GcpCredentialProviderType::ComputeEngine},
+ {" compute_engine ", "", GcpCredentialProviderType::ComputeEngine},
+ {"COMPUTE_ENGINE", "target@my-project.iam.gserviceaccount.com",
+ GcpCredentialProviderType::ComputeEngine},
+ };
+
+ for (const auto& test_case : cases) {
+ auto properties = base_properties;
+ properties["gs.credential_provider_type"] = test_case.provider_type;
+ if (test_case.impersonation_service_account[0] != '\0') {
+ properties["gs.impersonation_service_account"] =
+ test_case.impersonation_service_account;
+ }
+
+ S3Conf s3_conf;
+ ASSERT_TRUE(
+ S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok())
+ << "provider_type=" << test_case.provider_type;
+ ASSERT_EQ(s3_conf.client_conf.provider, io::ObjStorageType::GCP);
+ ASSERT_TRUE(std::holds_alternative<GcpCredentialConfig>(s3_conf.client_conf.credential));
+ const auto& credential = std::get<GcpCredentialConfig>(s3_conf.client_conf.credential);
+ EXPECT_EQ(credential.provider_type, test_case.expected_provider_type);
+ EXPECT_EQ(credential.impersonation_service_account,
+ test_case.impersonation_service_account);
+ }
+}
+
+TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfGcpDefaultImpersonation) {
+ S3URI s3_uri("s3://test-bucket/test-prefix");
+ ASSERT_TRUE(s3_uri.parse().ok());
+ std::map<std::string, std::string> properties {
+ {"AWS_ENDPOINT", "storage.googleapis.com"},
+ {"AWS_REGION", "us-east1"},
+ {"provider", "GCP"},
+ {"gs.impersonation_service_account", "target@my-project.iam.gserviceaccount.com"},
+ };
+ S3Conf s3_conf;
+ ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok());
+ const auto& credential = std::get<GcpCredentialConfig>(s3_conf.client_conf.credential);
+ EXPECT_EQ(credential.provider_type, GcpCredentialProviderType::Default);
+ EXPECT_EQ(credential.impersonation_service_account,
+ "target@my-project.iam.gserviceaccount.com");
+
+ properties["gs.impersonation_service_account"] = "";
+ ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok());
+ const auto& direct_credential = std::get<GcpCredentialConfig>(s3_conf.client_conf.credential);
+ EXPECT_EQ(direct_credential.provider_type, GcpCredentialProviderType::Default);
+ EXPECT_TRUE(direct_credential.impersonation_service_account.empty());
+}
+
+TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfRejectsConflictingGcpCredentials) {
+ S3URI s3_uri("s3://test-bucket/test-prefix");
+ ASSERT_TRUE(s3_uri.parse().ok());
+
+ std::map<std::string, std::string> base_properties {
+ {"AWS_ENDPOINT", "storage.googleapis.com"},
+ {"AWS_REGION", "us-east1"},
+ {"provider", "GCP"},
+ {"gs.credential_provider_type", "DEFAULT"},
+ };
+
+ const std::vector<std::pair<std::string, std::string>> conflicts = {
+ {"AWS_ACCESS_KEY", "ak"},
+ {"AWS_SECRET_KEY", "sk"},
+ {"AWS_TOKEN", "token"},
+ {"AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/test-role"},
+ {"AWS_EXTERNAL_ID", "external-id"},
+ {"AWS_CREDENTIALS_PROVIDER_TYPE", "INSTANCE_PROFILE"},
+ };
+ for (const auto& [key, value] : conflicts) {
+ auto properties = base_properties;
+ properties[key] = value;
+ S3Conf s3_conf;
+ auto status = S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf);
+ ASSERT_FALSE(status.ok()) << key;
+ EXPECT_NE(status.to_string().find("cannot be combined"), std::string::npos) << key;
+ }
+}
+
+TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfRejectsInvalidGcpCredentialModes) {
+ S3URI s3_uri("s3://test-bucket/test-prefix");
+ ASSERT_TRUE(s3_uri.parse().ok());
+
+ std::map<std::string, std::string> base_properties {
+ {"AWS_ENDPOINT", "storage.googleapis.com"},
+ {"AWS_REGION", "us-east1"},
+ {"provider", "GCP"},
+ };
+ for (const auto& provider_type : {"ADC", "", "unknown"}) {
+ auto properties = base_properties;
+ properties["gs.credential_provider_type"] = provider_type;
+ S3Conf s3_conf;
+ auto status = S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf);
+ ASSERT_FALSE(status.ok()) << provider_type;
+ EXPECT_NE(status.to_string().find("gs.credential_provider_type"), std::string::npos)
+ << provider_type;
+ EXPECT_NE(status.to_string().find("DEFAULT"), std::string::npos) << provider_type;
+ EXPECT_NE(status.to_string().find("COMPUTE_ENGINE"), std::string::npos) << provider_type;
+ }
+
+ for (const auto& account : {" ", "invalid-account"}) {
+ auto properties = base_properties;
+ properties["gs.impersonation_service_account"] = account;
+ S3Conf s3_conf;
+ ASSERT_FALSE(
+ S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok());
+ }
+}
+
+TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfRejectsGcpCredentialWithAwsProvider) {
+ S3URI s3_uri("s3://test-bucket/test-prefix");
+ ASSERT_TRUE(s3_uri.parse().ok());
+
+ for (const auto& provider : {"S3", "AWS"}) {
+ std::map<std::string, std::string> properties {
+ {"AWS_ENDPOINT", "storage.googleapis.com"},
+ {"AWS_REGION", "us-east1"},
+ {"provider", provider},
+ {"gs.credential_provider_type", "DEFAULT"},
+ };
+ S3Conf s3_conf;
+ auto status = S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf);
+ ASSERT_FALSE(status.ok()) << provider;
+ EXPECT_NE(status.to_string().find("provider=GCP"), std::string::npos) << provider;
+ }
+}
+
TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfCredentialValidation) {
S3URI s3_uri("s3://test-bucket/test-prefix");
ASSERT_TRUE(s3_uri.parse().ok());
diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt
index d4e0b346659..0f5891a5a9f 100644
--- a/cloud/CMakeLists.txt
+++ b/cloud/CMakeLists.txt
@@ -139,6 +139,9 @@ set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools")
# Set all libraries
include(thirdparty)
+set(absl_DIR ${THIRDPARTY_DIR}/lib/cmake/absl)
+find_package(absl CONFIG REQUIRED)
+
find_program(THRIFT_COMPILER thrift ${CMAKE_SOURCE_DIR}/bin)
# Check if functions are supported in this platform. All flags will generated
@@ -321,6 +324,19 @@ else()
add_definitions(-DUSE_LIBHDFS3)
endif()
+# google-cloud-cpp is linked as static archives in thirdparty.cmake, so its
+# public Abseil dependencies must be present in the final link group as well.
+set(COMMON_THIRDPARTY
+ ${COMMON_THIRDPARTY}
+ absl::base
+ absl::memory
+ absl::optional
+ absl::span
+ absl::str_format
+ absl::time
+ absl::variant
+)
+
set(DORIS_DEPENDENCIES
${DORIS_DEPENDENCIES}
${WL_START_GROUP}
diff --git a/cloud/cmake/thirdparty.cmake b/cloud/cmake/thirdparty.cmake
index a383b263b9c..d945ceaa45e 100644
--- a/cloud/cmake/thirdparty.cmake
+++ b/cloud/cmake/thirdparty.cmake
@@ -72,6 +72,10 @@ add_thirdparty(brpc LIB64)
add_thirdparty(rocksdb) # For local storage mocking
add_thirdparty(libz LIBNAME "lib/libz.a") # Required by google::protobuf
add_thirdparty(curl)
+add_thirdparty(google_cloud_cpp_oauth2)
+add_thirdparty(google_cloud_cpp_rest_internal)
+add_thirdparty(google_cloud_cpp_common)
+add_thirdparty(crc32c)
add_thirdparty(zstd LIB64)
add_thirdparty(fmt)
# begin aws libs
diff --git a/cloud/src/common/CMakeLists.txt b/cloud/src/common/CMakeLists.txt
index 9f29e1a183b..e356861dd72 100644
--- a/cloud/src/common/CMakeLists.txt
+++ b/cloud/src/common/CMakeLists.txt
@@ -32,6 +32,7 @@ set(COMMON_FILES
version_metrics.cpp
kms.cpp
network_util.cpp
+ auth/obj_credential.cpp
../server/oss/cloud_server_starter_factory.cpp
)
diff --git a/cloud/src/common/auth/obj_credential.cpp b/cloud/src/common/auth/obj_credential.cpp
new file mode 100644
index 00000000000..acb7116c1bc
--- /dev/null
+++ b/cloud/src/common/auth/obj_credential.cpp
@@ -0,0 +1,239 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "common/auth/obj_credential.h"
+
+#include <gen_cpp/cloud.pb.h>
+
+#include <string_view>
+
+#include "cpp/obj-client/auth/obj_credential.h"
+
+namespace doris {
+namespace {
+
+GcpCredentialConfig to_gcp_credential_config(const cloud::GcpCredentialPB& credential) {
+ GcpCredentialConfig config;
+ switch (credential.credential_provider_type()) {
+ case cloud::GcpCredentialPB::DEFAULT:
+ config.provider_type = GcpCredentialProviderType::Default;
+ break;
+ case cloud::GcpCredentialPB::COMPUTE_ENGINE:
+ config.provider_type = GcpCredentialProviderType::ComputeEngine;
+ break;
+ }
+ config.impersonation_service_account = credential.impersonation_service_account();
+ return config;
+}
+
+} // namespace
+
+void convert_obj_credential(const cloud::ObjectStoreInfoPB& source, CredentialConfig* credential) {
+ if (!source.has_credential() || !source.credential().has_gcp_credential()) {
+ *credential = std::monostate {};
+ return;
+ }
+ *credential = to_gcp_credential_config(source.credential().gcp_credential());
+}
+
+} // namespace doris
+
+namespace doris::cloud {
+namespace {
+
+// Extension checklist for a new passwordless provider:
+// 1. add a provider-specific validate_<provider>_obj_credential();
+// 2. dispatch it from validate_obj_credential(); and
+// 3. reject requests that set more than one provider credential.
+// Generic create/alter normalization does not need provider-specific branches.
+ObjCredentialProvider to_obj_credential_provider(const ObjectStoreInfoPB& obj) {
+ if (!obj.has_provider()) {
+ return ObjCredentialProvider::Unknown;
+ }
+ switch (obj.provider()) {
+ case ObjectStoreInfoPB::S3:
+ return ObjCredentialProvider::Aws;
+ case ObjectStoreInfoPB::AZURE:
+ return ObjCredentialProvider::Azure;
+ case ObjectStoreInfoPB::BOS:
+ return ObjCredentialProvider::Bos;
+ case ObjectStoreInfoPB::COS:
+ return ObjCredentialProvider::Cos;
+ case ObjectStoreInfoPB::GCP:
+ return ObjCredentialProvider::Gcp;
+ case ObjectStoreInfoPB::OBS:
+ return ObjCredentialProvider::Obs;
+ case ObjectStoreInfoPB::OSS:
+ return ObjCredentialProvider::Oss;
+ case ObjectStoreInfoPB::TOS:
+ return ObjCredentialProvider::Tos;
+ case ObjectStoreInfoPB::UNKONWN:
+ return ObjCredentialProvider::Unknown;
+ }
+ return ObjCredentialProvider::Unknown;
+}
+
+ObjCredentialValidationContext credential_validation_context(const ObjectStoreInfoPB& obj) {
+ return ObjCredentialValidationContext {
+ .provider = to_obj_credential_provider(obj),
+ .ak = obj.ak(),
+ .sk = obj.sk(),
+ .has_aws_role_arn = !obj.role_arn().empty(),
+ .has_aws_external_id = !obj.external_id().empty(),
+ .has_aws_credential_provider = obj.has_cred_provider_type(),
+ .has_encrypted_access_keys = obj.has_encryption_info(),
+ };
+}
+
+// A provider-native credential supersedes both common static credentials and
+// provider-specific authentication fields outside the credential envelope.
+void clear_legacy_credentials(ObjectStoreInfoPB* obj) {
+ obj->clear_ak();
+ obj->clear_sk();
+ obj->clear_encryption_info();
+ obj->clear_cred_provider_type();
+ obj->clear_role_arn();
+ obj->clear_external_id();
+}
+
+std::optional<std::string> validate_required_storage_fields(const ObjectStoreInfoPB& obj,
+ std::string_view provider) {
+ if (obj.bucket().empty() || obj.endpoint().empty() || obj.region().empty()) {
+ return std::string(provider) + " storage conf requires bucket, endpoint and region";
+ }
+ return std::nullopt;
+}
+
+std::optional<std::string> validate_gcp_obj_credential(const ObjectStoreInfoPB& obj) {
+ const auto& credential = obj.credential().gcp_credential();
+ if (!credential.has_credential_provider_type()) {
+ return "GCP credential provider type is required";
+ }
+ if (auto error = validate_required_storage_fields(obj, "GCP"); error.has_value()) {
+ return error;
+ }
+
+ CredentialConfig config;
+ convert_obj_credential(obj, &config);
+ return validate_obj_credential_config(config, credential_validation_context(obj));
+}
+
+} // namespace
+
+bool has_obj_credential(const ObjectStoreInfoPB& obj) {
+ return obj.has_credential();
+}
+
+std::optional<std::string> validate_obj_credential(const ObjectStoreInfoPB& obj) {
+ if (!obj.has_credential()) {
+ return "credential is not set";
+ }
+
+ // Add one provider-specific branch for every new field in
+ // ObjectStoreCredentialPB. Reject multiple fields before dispatch when a
+ // second provider is introduced. The common visitor validates only the
+ // normalized credential itself.
+ if (obj.credential().has_gcp_credential()) {
+ return validate_gcp_obj_credential(obj);
+ }
+ return "unsupported object storage credential";
+}
+
+std::optional<std::string> validate_obj_authentication(const ObjectStoreInfoPB& obj) {
+ if (has_obj_credential(obj)) {
+ return validate_obj_credential(obj);
+ }
+ if (obj.has_role_arn() && obj.role_arn().empty()) {
+ return "AWS role ARN cannot be empty";
+ }
+
+ const auto context = credential_validation_context(obj);
+ if (context.has_static_credentials() && context.has_aws_authentication()) {
+ return "access keys cannot be combined with AWS role or credential provider";
+ }
+ if (!context.has_aws_authentication()) {
+ return std::nullopt;
+ }
+ if (!obj.has_provider() || obj.provider() != ObjectStoreInfoPB::S3) {
+ return "AWS role and credential provider require provider=S3";
+ }
+ if (obj.has_external_id() && obj.external_id().size() > 0 && obj.role_arn().empty()) {
+ return "AWS external ID requires a role ARN";
+ }
+ return std::nullopt;
+}
+
+std::optional<std::string> validate_and_normalize_obj_credential(ObjectStoreInfoPB* obj) {
+ if (auto error = validate_obj_credential(*obj); error.has_value()) {
+ return error;
+ }
+ clear_legacy_credentials(obj);
+ return std::nullopt;
+}
+
+std::optional<std::string> apply_obj_credential(const ObjectStoreInfoPB& update,
+ ObjectStoreInfoPB* target) {
+ if (!has_obj_credential(update)) {
+ return "credential is not set";
+ }
+ if (credential_validation_context(update).has_conflicting_credentials()) {
+ return "credentials cannot be combined with other credentials";
+ }
+
+ // Merge into a copy first so a rejected alter never partially mutates target.
+ ObjectStoreInfoPB candidate = *target;
+ copy_obj_credential(update, &candidate);
+ if (update.credential().has_gcp_credential()) {
+ const auto& patch = update.credential().gcp_credential();
+ if (!patch.has_credential_provider_type() && !patch.has_impersonation_service_account()) {
+ return "GCP credential update must specify a provider type or service account";
+ }
+ auto* credential = candidate.mutable_credential()->mutable_gcp_credential();
+ if (target->credential().has_gcp_credential()) {
+ credential->CopyFrom(target->credential().gcp_credential());
+ credential->MergeFrom(patch);
+ }
+ // Only a new native credential needs a default; an existing source is
+ // preserved when the patch changes just the impersonation target.
+ if (!credential->has_credential_provider_type()) {
+ credential->set_credential_provider_type(GcpCredentialPB::DEFAULT);
+ }
+ if (credential->impersonation_service_account().empty()) {
+ credential->clear_impersonation_service_account();
+ }
+ }
+ clear_legacy_credentials(&candidate);
+ if (auto error = validate_obj_credential(candidate); error.has_value()) {
+ return error;
+ }
+
+ clear_legacy_credentials(target);
+ copy_obj_credential(candidate, target);
+ return std::nullopt;
+}
+
+void copy_obj_credential(const ObjectStoreInfoPB& source, ObjectStoreInfoPB* target) {
+ if (&source == target) {
+ return;
+ }
+ target->clear_credential();
+ if (source.has_credential()) {
+ target->mutable_credential()->CopyFrom(source.credential());
+ }
+}
+
+} // namespace doris::cloud
diff --git a/cloud/src/common/auth/obj_credential.h b/cloud/src/common/auth/obj_credential.h
new file mode 100644
index 00000000000..a1ad7714e14
--- /dev/null
+++ b/cloud/src/common/auth/obj_credential.h
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <optional>
+#include <string>
+
+namespace doris::cloud {
+
+// This helper keeps provider-specific credential branches out of MetaService.
+// Provider authentication implementations belong in common/cpp/obj-client/auth; this
+// Cloud layer owns vault-field validation, compatibility cleanup, and protobuf
+// persistence mutations.
+
+class ObjectStoreInfoPB;
+
+// Whether a provider-native credential envelope is present.
+bool has_obj_credential(const ObjectStoreInfoPB& obj);
+
+// TODO: Move AWS authentication fields into ObjectStoreInfoPB.credential so AWS
+// follows the same validation and mutation path. Keep cred_provider_type,
+// role_arn, and external_id in their current fields for now to preserve
+// compatibility with stored vault metadata and existing clients. AK/SK remain
+// the common static-credential representation for S3-compatible providers.
+
+// Validates the active ObjectStoreInfoPB.credential case without modifying obj.
+// Add a provider-specific validation case when extending the credential envelope.
+std::optional<std::string> validate_obj_credential(const ObjectStoreInfoPB& obj);
+
+// Validates the complete authentication state after create/alter mutations. In
+// addition to provider-native credentials, this rejects cross-provider AWS
+// authentication and mutually exclusive authentication modes.
+std::optional<std::string> validate_obj_authentication(const ObjectStoreInfoPB& obj);
+
+// Used by create paths: validates first, then removes common static credentials
+// and provider-specific authentication fields that must not coexist with the
+// selected provider-native credential.
+std::optional<std::string> validate_and_normalize_obj_credential(ObjectStoreInfoPB* obj);
+
+// Used by alter paths: merges explicitly present GCP fields into the stored
+// credential, with an empty service account clearing impersonation. Validates
+// the result against target's storage fields before changing authentication.
+std::optional<std::string> apply_obj_credential(const ObjectStoreInfoPB& update,
+ ObjectStoreInfoPB* target);
+
+// Replaces only the credential envelope. An unset source clears target's credential;
+// all non-credential fields are preserved.
+void copy_obj_credential(const ObjectStoreInfoPB& source, ObjectStoreInfoPB* target);
+
+} // namespace doris::cloud
diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp
index 0284d4246ef..fbdaae334fd 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -31,6 +31,7 @@
#include <tuple>
#include <unordered_set>
+#include "common/auth/obj_credential.h"
#include "common/bvars.h"
#include "common/config.h"
#include "common/encryption_util.h"
@@ -765,7 +766,7 @@ static bool has_non_empty_role_arn(const ObjectStoreInfoPB& obj) {
}
static bool use_credential_provider(const ObjectStoreInfoPB& obj) {
- return obj.has_cred_provider_type() || has_non_empty_role_arn(obj);
+ return obj.has_cred_provider_type() || has_non_empty_role_arn(obj) || has_obj_credential(obj);
}
static void create_object_info_with_encrypt(const InstanceInfoPB& instance, ObjectStoreInfoPB* obj,
@@ -781,14 +782,23 @@ static void create_object_info_with_encrypt(const InstanceInfoPB& instance, Obje
std::string external_endpoint = obj->has_external_endpoint() ? obj->external_endpoint() : "";
std::string region = obj->has_region() ? obj->region() : "";
- if (obj->has_role_arn()) {
- if (obj->role_arn().empty() || !obj->has_cred_provider_type() || !obj->has_provider() ||
+ if (has_obj_credential(*obj)) {
+ if (auto error = validate_and_normalize_obj_credential(obj); error.has_value()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = *error;
+ return;
+ }
+ } else if (obj->has_role_arn()) {
+ if (obj->role_arn().empty() || !obj->has_provider() ||
obj->provider() != ObjectStoreInfoPB::S3 || bucket.empty() || endpoint.empty() ||
region.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "s3 conf info err with role_arn or cred provider, please check it";
return;
}
+ if (!obj->has_cred_provider_type()) {
+ obj->set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE);
+ }
} else {
// ATTN: prefix may be empty
if (plain_ak.empty() || plain_sk.empty() || bucket.empty() || endpoint.empty() ||
@@ -1071,7 +1081,11 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt
msg = ss.str();
return -1;
}
- auto* storage_vault_names = instance.mutable_storage_vault_names();
+ // Apply all mutations to a candidate. The caller can safely abandon the
+ // transaction on any validation/serialization error without leaking a
+ // partial rename into the InstanceInfoPB it later persists.
+ InstanceInfoPB candidate_instance = instance;
+ auto* storage_vault_names = candidate_instance.mutable_storage_vault_names();
auto* name_ptr = storage_vault_names->Mutable(pos);
DCHECK(name_ptr != nullptr);
const std::string old_name = *name_ptr;
@@ -1081,8 +1095,8 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt
target_vault_id, name, old_name);
return -1;
}
- std::string vault_id = instance.resource_ids().Get(pos);
- auto vault_key = storage_vault_key({instance.instance_id(), vault_id});
+ std::string vault_id = candidate_instance.resource_ids().Get(pos);
+ auto vault_key = storage_vault_key({candidate_instance.instance_id(), vault_id});
std::string val;
auto err = txn->get(vault_key, &val);
@@ -1118,7 +1132,7 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt
return -1;
}
- if (vault_exist(instance, vault.alter_name())) {
+ if (vault_exist(candidate_instance, vault.alter_name())) {
code = MetaServiceCode::ALREADY_EXISTED;
msg = fmt::format("vault_name={} already existed", vault.alter_name());
return -1;
@@ -1126,14 +1140,16 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt
new_vault.set_name(vault.alter_name());
*name_ptr = vault.alter_name();
- if (instance.default_storage_vault_id() == vault_id) {
- instance.set_default_storage_vault_name(vault.alter_name());
+ if (candidate_instance.default_storage_vault_id() == vault_id) {
+ candidate_instance.set_default_storage_vault_name(vault.alter_name());
}
}
- if (obj_info.has_role_arn() && (obj_info.has_ak() || obj_info.has_sk())) {
+ if ((obj_info.has_role_arn() || obj_info.has_cred_provider_type() ||
+ has_obj_credential(obj_info)) &&
+ (obj_info.has_ak() || obj_info.has_sk())) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "invaild argument, both set ak/sk and role_arn is not allowed";
+ msg = "invalid argument, access keys cannot be combined with role or credentials";
LOG(WARNING) << msg;
return -1;
}
@@ -1162,28 +1178,68 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt
new_vault.mutable_obj_info()->clear_role_arn();
new_vault.mutable_obj_info()->clear_external_id();
new_vault.mutable_obj_info()->clear_cred_provider_type();
+ new_vault.mutable_obj_info()->clear_credential();
new_vault.mutable_obj_info()->set_ak(new_ak_sk_pair.first);
new_vault.mutable_obj_info()->set_sk(new_ak_sk_pair.second);
new_vault.mutable_obj_info()->mutable_encryption_info()->CopyFrom(encryption_info);
}
- if (obj_info.has_role_arn()) {
+ if (obj_info.has_role_arn() || obj_info.has_cred_provider_type()) {
new_vault.mutable_obj_info()->clear_ak();
new_vault.mutable_obj_info()->clear_sk();
new_vault.mutable_obj_info()->clear_encryption_info();
+ new_vault.mutable_obj_info()->clear_credential();
+
+ if (obj_info.has_role_arn()) {
+ if (obj_info.role_arn().empty()) {
+ new_vault.mutable_obj_info()->clear_role_arn();
+ // An external ID has no meaning without a role. Clearing the role
+ // explicitly clears its dependent value as well.
+ new_vault.mutable_obj_info()->clear_external_id();
+ } else {
+ new_vault.mutable_obj_info()->set_role_arn(obj_info.role_arn());
+ if (!new_vault.obj_info().has_cred_provider_type() &&
+ !obj_info.has_cred_provider_type()) {
+ new_vault.mutable_obj_info()->set_cred_provider_type(
+ CredProviderTypePB::INSTANCE_PROFILE);
+ }
+ }
+ }
+ if (obj_info.has_cred_provider_type()) {
+ new_vault.mutable_obj_info()->set_cred_provider_type(obj_info.cred_provider_type());
+ }
+ }
- new_vault.mutable_obj_info()->set_role_arn(obj_info.role_arn());
- new_vault.mutable_obj_info()->set_cred_provider_type(get_cred_provider_type(obj_info));
- if (obj_info.has_external_id()) {
+ // external_id is an independent ALTER field. It may be changed without
+ // requiring FE to resend role_arn and cred_provider_type.
+ if (obj_info.has_external_id()) {
+ if (obj_info.external_id().empty()) {
+ new_vault.mutable_obj_info()->clear_external_id();
+ } else {
new_vault.mutable_obj_info()->set_external_id(obj_info.external_id());
}
}
+ if (has_obj_credential(obj_info)) {
+ if (auto error = apply_obj_credential(obj_info, new_vault.mutable_obj_info());
+ error.has_value()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = *error;
+ return -1;
+ }
+ }
+
if (obj_info.has_use_path_style()) {
new_vault.mutable_obj_info()->set_use_path_style(obj_info.use_path_style());
}
+ if (auto error = validate_obj_authentication(new_vault.obj_info()); error.has_value()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = *error;
+ return -1;
+ }
+
auto now_time = std::chrono::system_clock::now();
uint64_t time =
std::chrono::duration_cast<std::chrono::seconds>(now_time.time_since_epoch()).count();
@@ -1198,6 +1254,7 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt
}
txn->put(vault_key, val);
+ instance.Swap(&candidate_instance);
LOG(INFO) << "put vault_id=" << vault_id << ", vault_key=" << hex(vault_key)
<< ", origin vault=" << encryt_sk(hide_ak(origin_vault_info))
<< ", new vault=" << encryt_sk(hide_ak(new_vault_info));
@@ -1259,7 +1316,7 @@ static int extract_object_storage_info(const AlterObjStoreInfoRequest* request,
external_id] = obj_desc;
bool use_credential_provider_for_add_vault =
- request->op() == AlterObjStoreInfoRequest::ADD_S3_VAULT && obj.has_cred_provider_type();
+ request->op() == AlterObjStoreInfoRequest::ADD_S3_VAULT && use_credential_provider(obj);
if (!obj.has_role_arn() && !use_credential_provider_for_add_vault) {
if (!obj.has_ak() || !obj.has_sk()) {
code = MetaServiceCode::INVALID_ARGUMENT;
@@ -1325,7 +1382,10 @@ static ObjectStoreInfoPB object_info_pb_factory(ObjectStorageDesc& obj_desc,
last_item.set_role_arn(role_arn);
last_item.set_external_id(external_id);
}
- last_item.set_cred_provider_type(get_cred_provider_type(obj));
+ if (obj.has_cred_provider_type() || has_non_empty_role_arn(obj)) {
+ last_item.set_cred_provider_type(get_cred_provider_type(obj));
+ }
+ copy_obj_credential(obj, &last_item);
}
last_item.set_bucket(bucket);
// format prefix, such as `/aa/bb/`, `aa/bb//`, `//aa/bb`, ` /aa/bb` -> `aa/bb`
@@ -1496,7 +1556,11 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
}
if (use_credential_provider(obj)) {
- if (!obj.has_provider() || obj.provider() != ObjectStoreInfoPB::S3) {
+ bool valid_aws = obj.has_cred_provider_type() && !has_obj_credential(obj) &&
+ obj.has_provider() && obj.provider() == ObjectStoreInfoPB::S3;
+ bool valid_credential =
+ has_obj_credential(obj) && !validate_obj_credential(obj).has_value();
+ if (!valid_aws && !valid_credential) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "s3 conf info err with credentials_provider_type, please check it";
return;
@@ -1620,7 +1684,9 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
break;
}
case AlterObjStoreInfoRequest::ALTER_S3_VAULT: {
- alter_s3_storage_vault(instance, txn, request->vault(), code, msg, response);
+ if (alter_s3_storage_vault(instance, txn, request->vault(), code, msg, response) != 0) {
+ return;
+ }
break;
}
case AlterObjStoreInfoRequest::ALTER_HDFS_VAULT: {
diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp
index e5fb445a45d..56b2338cd84 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -47,6 +47,8 @@
#include "common/util.h"
#include "cpp/aws_logger.h"
#include "cpp/custom_aws_credentials_provider_chain.h"
+#include "cpp/obj-client/auth/obj_credential.h"
+#include "cpp/obj-client/auth/obj_s3_client_factory.h"
#include "cpp/obj_retry_strategy.h"
#include "cpp/sync_point.h"
#include "cpp/token_bucket_rate_limiter.h"
@@ -242,6 +244,8 @@ std::optional<S3Conf> S3Conf::from_obj_store_info(const ObjectStoreInfoPB& obj_i
}
}
+ convert_obj_credential(obj_info, &s3_conf.credential);
+
s3_conf.endpoint = obj_info.endpoint();
s3_conf.region = obj_info.region();
s3_conf.bucket = obj_info.bucket();
@@ -443,10 +447,16 @@ int S3Accessor::init() {
// curl error 28 on slow/large S3 DeleteObjects (OVH cold vault).
aws_config.requestTimeoutMs = 30000;
aws_config.connectTimeoutMs = 5000;
- auto s3_client = std::make_shared<Aws::S3::S3Client>(
- get_aws_credentials_provider(conf_), std::move(aws_config),
- Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
- conf_.use_virtual_addressing /* useVirtualAddressing */);
+ auto s3_client = doris::make_s3_client(
+ conf_.credential,
+ doris::S3ClientBuildContext {
+ .fallback_provider = get_aws_credentials_provider(conf_),
+ .config = std::move(aws_config),
+ .payload_signing_policy =
+ Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
+ .use_virtual_addressing = conf_.use_virtual_addressing,
+ .ca_cert_path = _ca_cert_file_path,
+ });
obj_client_ = std::make_shared<S3ObjClient>(std::move(s3_client), conf_.endpoint);
return 0;
}
diff --git a/cloud/src/recycler/s3_accessor.h b/cloud/src/recycler/s3_accessor.h
index 28d12c7f980..3e1d57e3e2a 100644
--- a/cloud/src/recycler/s3_accessor.h
+++ b/cloud/src/recycler/s3_accessor.h
@@ -23,8 +23,10 @@
#include <array>
#include <cstdint>
#include <memory>
+#include <optional>
#include "cpp/aws_common.h"
+#include "cpp/obj-client/auth/obj_credential.h"
#include "recycler/obj_storage_client.h"
#include "recycler/storage_vault_accessor.h"
@@ -100,6 +102,7 @@ struct S3Conf {
CredProviderType cred_provider_type = CredProviderType::Default;
std::string role_arn;
std::string external_id;
+ doris::CredentialConfig credential {};
enum Provider : uint8_t {
S3,
diff --git a/cloud/test/CMakeLists.txt b/cloud/test/CMakeLists.txt
index 057b0132356..2bf79b7d3cf 100644
--- a/cloud/test/CMakeLists.txt
+++ b/cloud/test/CMakeLists.txt
@@ -78,6 +78,8 @@ add_executable(log_test log_test.cpp)
add_executable(resource_test resource_test.cpp)
+add_executable(obj_credential_test obj_credential_test.cpp)
+
add_executable(rpc_kv_bvar_test rpc_kv_bvar_test.cpp)
add_executable(http_encode_key_test http_encode_key_test.cpp)
@@ -171,6 +173,8 @@ target_link_libraries(log_test ${TEST_LINK_LIBS})
target_link_libraries(resource_test ${TEST_LINK_LIBS})
+target_link_libraries(obj_credential_test ${TEST_LINK_LIBS})
+
target_link_libraries(rpc_kv_bvar_test ${TEST_LINK_LIBS})
target_link_libraries(http_encode_key_test ${TEST_LINK_LIBS})
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index 1c5fc0c06c1..7bba23793f3 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -34,6 +34,7 @@
#include <memory>
#include <random>
#include <string>
+#include <string_view>
#include <thread>
#include "common/config.h"
@@ -733,10 +734,28 @@ TEST(MetaServiceTest, AlterS3StorageVaultTest) {
InstanceInfoPB instance;
instance.add_storage_vault_names(vault.name());
instance.add_resource_ids(vault.id());
+
+ constexpr char gcp_vault_name[] = "test_alter_gcp_vault";
+ StorageVaultPB gcp_vault;
+ gcp_vault.set_name(gcp_vault_name);
+ gcp_vault.set_id("3");
+ auto* gcp_obj_info = gcp_vault.mutable_obj_info();
+ gcp_obj_info->set_id("3");
+ gcp_obj_info->set_provider(ObjectStoreInfoPB::GCP);
+ gcp_obj_info->set_bucket("gcp-bucket");
+ gcp_obj_info->set_endpoint("storage.googleapis.com");
+ gcp_obj_info->set_region("us-east1");
+ gcp_obj_info->mutable_credential()->mutable_gcp_credential()->set_credential_provider_type(
+ GcpCredentialPB::COMPUTE_ENGINE);
+ gcp_obj_info->mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account(
+ "original@my-project.iam.gserviceaccount.com");
+ instance.add_storage_vault_names(gcp_vault.name());
+ instance.add_resource_ids(gcp_vault.id());
instance.set_instance_id("GetObjStoreInfoTestInstance");
val = instance.SerializeAsString();
txn->put(key, val);
txn->put(storage_vault_key({instance.instance_id(), "2"}), vault.SerializeAsString());
+ txn->put(storage_vault_key({instance.instance_id(), "3"}), gcp_vault.SerializeAsString());
ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
txn = nullptr;
@@ -751,6 +770,173 @@ TEST(MetaServiceTest, AlterS3StorageVaultTest) {
i.ParseFromString(val);
};
+ auto get_test_vault = [&](std::string_view vault_id, StorageVaultPB& stored_vault) {
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(), vault_id}), &val),
+ TxnErrorCode::TXN_OK);
+ ASSERT_TRUE(stored_vault.ParseFromString(val));
+ };
+
+ // A rejected credential update must not commit a rename that was applied
+ // earlier in the request.
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ req.mutable_vault()->set_name(vault_name);
+ req.mutable_vault()->set_alter_name("rename_must_not_commit");
+ req.mutable_vault()
+ ->mutable_obj_info()
+ ->mutable_credential()
+ ->mutable_gcp_credential()
+ ->set_credential_provider_type(GcpCredentialPB::DEFAULT);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) << res.status().msg();
+
+ InstanceInfoPB stored_instance;
+ get_test_instance(stored_instance);
+ ASSERT_EQ(stored_instance.storage_vault_names(0), vault_name);
+ StorageVaultPB stored_vault;
+ get_test_vault("2", stored_vault);
+ ASSERT_EQ(stored_vault.name(), vault_name);
+ }
+
+ // A GCP-native vault cannot be changed to either AWS role or AWS
+ // credential-provider authentication while its provider remains GCP.
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ req.mutable_vault()->set_name(gcp_vault_name);
+ auto* update = req.mutable_vault()->mutable_obj_info();
+ update->set_role_arn("arn:aws:iam::123456789012:role/test-role");
+ update->set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) << res.status().msg();
+ ASSERT_NE(res.status().msg().find("provider=S3"), std::string::npos);
+ }
+
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ req.mutable_vault()->set_name(gcp_vault_name);
+ req.mutable_vault()->mutable_obj_info()->set_cred_provider_type(
+ CredProviderTypePB::CONTAINER);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) << res.status().msg();
+ ASSERT_NE(res.status().msg().find("provider=S3"), std::string::npos);
+
+ InstanceInfoPB stored_instance;
+ get_test_instance(stored_instance);
+ ASSERT_EQ(stored_instance.storage_vault_names(1), gcp_vault_name);
+ StorageVaultPB stored_vault;
+ get_test_vault("3", stored_vault);
+ ASSERT_TRUE(stored_vault.obj_info().has_credential());
+ ASSERT_TRUE(stored_vault.obj_info().credential().has_gcp_credential());
+ ASSERT_FALSE(stored_vault.obj_info().has_role_arn());
+ ASSERT_FALSE(stored_vault.obj_info().has_cred_provider_type());
+ }
+
+ // Verify partial GCP updates at the RPC and persistence boundary.
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ req.mutable_vault()->set_name(gcp_vault_name);
+ auto alter_gcp = [&]() {
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res,
+ nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ };
+ auto* update = req.mutable_vault()->mutable_obj_info();
+ auto* patch = update->mutable_credential()->mutable_gcp_credential();
+ patch->set_impersonation_service_account("updated@my-project.iam.gserviceaccount.com");
+ alter_gcp();
+ StorageVaultPB stored;
+ get_test_vault("3", stored);
+ ASSERT_EQ(stored.obj_info().credential().gcp_credential().credential_provider_type(),
+ GcpCredentialPB::COMPUTE_ENGINE);
+ ASSERT_EQ(stored.obj_info().credential().gcp_credential().impersonation_service_account(),
+ "updated@my-project.iam.gserviceaccount.com");
+
+ patch->Clear();
+ patch->set_credential_provider_type(GcpCredentialPB::DEFAULT);
+ alter_gcp();
+ get_test_vault("3", stored);
+ ASSERT_EQ(stored.obj_info().credential().gcp_credential().credential_provider_type(),
+ GcpCredentialPB::DEFAULT);
+ ASSERT_EQ(stored.obj_info().credential().gcp_credential().impersonation_service_account(),
+ "updated@my-project.iam.gserviceaccount.com");
+
+ patch->set_credential_provider_type(GcpCredentialPB::COMPUTE_ENGINE);
+ alter_gcp();
+ patch->Clear();
+ patch->set_impersonation_service_account("");
+ alter_gcp();
+ get_test_vault("3", stored);
+ ASSERT_EQ(stored.obj_info().credential().gcp_credential().credential_provider_type(),
+ GcpCredentialPB::COMPUTE_ENGINE);
+ ASSERT_FALSE(stored.obj_info()
+ .credential()
+ .gcp_credential()
+ .has_impersonation_service_account());
+
+ // Rejected patches must preserve both the stored vault and its name.
+ const auto original = stored.SerializeAsString();
+ patch->set_impersonation_service_account("invalid-account");
+ req.mutable_vault()->set_alter_name("invalid_gcp_rename");
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ get_test_vault("3", stored);
+ ASSERT_EQ(stored.SerializeAsString(), original);
+ InstanceInfoPB stored_instance;
+ get_test_instance(stored_instance);
+ ASSERT_EQ(stored_instance.storage_vault_names(1), gcp_vault_name);
+ req.mutable_vault()->clear_alter_name();
+
+ // Switching from encrypted static keys creates a new native credential.
+ update->Clear();
+ update->set_ak("gcp-ak");
+ update->set_sk("gcp-sk");
+ alter_gcp();
+ get_test_vault("3", stored);
+ ASSERT_FALSE(stored.obj_info().has_credential());
+ ASSERT_TRUE(stored.obj_info().has_encryption_info());
+ update->Clear();
+ update->mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account(
+ "updated@my-project.iam.gserviceaccount.com");
+ alter_gcp();
+ get_test_vault("3", stored);
+ ASSERT_EQ(stored.obj_info().credential().gcp_credential().credential_provider_type(),
+ GcpCredentialPB::DEFAULT);
+ ASSERT_EQ(stored.obj_info().credential().gcp_credential().impersonation_service_account(),
+ "updated@my-project.iam.gserviceaccount.com");
+ ASSERT_FALSE(stored.obj_info().has_ak());
+ ASSERT_FALSE(stored.obj_info().has_sk());
+ ASSERT_FALSE(stored.obj_info().has_encryption_info());
+ }
+
{
AlterObjStoreInfoRequest req;
req.set_cloud_unique_id("test_cloud_unique_id");
@@ -13110,6 +13296,87 @@ TEST(MetaServiceTest, AlterS3StorageVaultWithRoleArnTest) {
ASSERT_EQ(get_obj.name(), new_vault_name) << get_obj.obj_info().ShortDebugString();
}
+ // Role, external ID, and credential provider are independent ALTER fields.
+ // Updating one must preserve the other two.
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ req.mutable_vault()->set_name("new_test_alter_s3_vault_111");
+ req.mutable_vault()->mutable_obj_info()->set_cred_provider_type(
+ CredProviderTypePB::CONTAINER);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({"GetObjStoreInfoTestInstance", "2"}), &val),
+ TxnErrorCode::TXN_OK);
+ StorageVaultPB stored_vault;
+ ASSERT_TRUE(stored_vault.ParseFromString(val));
+ ASSERT_EQ(stored_vault.obj_info().role_arn(), "arn:aws:iam::12311321:role/test-alter-role");
+ ASSERT_EQ(stored_vault.obj_info().external_id(), "external_id_123123");
+ ASSERT_EQ(stored_vault.obj_info().cred_provider_type(), CredProviderTypePB::CONTAINER);
+ }
+
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ req.mutable_vault()->set_name("new_test_alter_s3_vault_111");
+ req.mutable_vault()->mutable_obj_info()->set_role_arn(
+ "arn:aws:iam::12311321:role/replacement-role");
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({"GetObjStoreInfoTestInstance", "2"}), &val),
+ TxnErrorCode::TXN_OK);
+ StorageVaultPB stored_vault;
+ ASSERT_TRUE(stored_vault.ParseFromString(val));
+ ASSERT_EQ(stored_vault.obj_info().role_arn(),
+ "arn:aws:iam::12311321:role/replacement-role");
+ ASSERT_EQ(stored_vault.obj_info().external_id(), "external_id_123123");
+ ASSERT_EQ(stored_vault.obj_info().cred_provider_type(), CredProviderTypePB::CONTAINER);
+ }
+
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ req.mutable_vault()->set_name("new_test_alter_s3_vault_111");
+ req.mutable_vault()->mutable_obj_info()->set_external_id("replacement-external-id");
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({"GetObjStoreInfoTestInstance", "2"}), &val),
+ TxnErrorCode::TXN_OK);
+ StorageVaultPB stored_vault;
+ ASSERT_TRUE(stored_vault.ParseFromString(val));
+ ASSERT_EQ(stored_vault.obj_info().role_arn(),
+ "arn:aws:iam::12311321:role/replacement-role");
+ ASSERT_EQ(stored_vault.obj_info().external_id(), "replacement-external-id");
+ ASSERT_EQ(stored_vault.obj_info().cred_provider_type(), CredProviderTypePB::CONTAINER);
+ }
+
std::string cipher_sk = "JUkuTDctR+ckJtnPkLScWaQZRcOtWBhsLLpnCRxQLxr734qB8cs6gNLH6grE1FxO";
std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t";
diff --git a/cloud/test/obj_credential_test.cpp b/cloud/test/obj_credential_test.cpp
new file mode 100644
index 00000000000..50905efec6d
--- /dev/null
+++ b/cloud/test/obj_credential_test.cpp
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "common/auth/obj_credential.h"
+
+#include <gen_cpp/cloud.pb.h>
+#include <gtest/gtest.h>
+
+namespace doris::cloud {
+namespace {
+
+constexpr char ACCOUNT[] = "original@my-project.iam.gserviceaccount.com";
+constexpr char NEW_ACCOUNT[] = "updated@my-project.iam.gserviceaccount.com";
+
+ObjectStoreInfoPB make_gcp_vault() {
+ ObjectStoreInfoPB obj;
+ obj.set_provider(ObjectStoreInfoPB::GCP);
+ obj.set_bucket("test-bucket");
+ obj.set_endpoint("storage.googleapis.com");
+ obj.set_region("us-east1");
+ auto* credential = obj.mutable_credential()->mutable_gcp_credential();
+ credential->set_credential_provider_type(GcpCredentialPB::COMPUTE_ENGINE);
+ credential->set_impersonation_service_account(ACCOUNT);
+ return obj;
+}
+
+TEST(ObjCredentialTest, AlterTargetPreservesSourceAndStorageFields) {
+ auto stored = make_gcp_vault();
+ auto expected = stored;
+ expected.mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account(
+ NEW_ACCOUNT);
+ ObjectStoreInfoPB update;
+ update.mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account(
+ NEW_ACCOUNT);
+ auto error = apply_obj_credential(update, &stored);
+ ASSERT_FALSE(error.has_value()) << error.value_or("");
+ EXPECT_EQ(stored.SerializeAsString(), expected.SerializeAsString());
+}
+
+TEST(ObjCredentialTest, AlterSourcePreservesTarget) {
+ for (auto source : {GcpCredentialPB::COMPUTE_ENGINE, GcpCredentialPB::DEFAULT}) {
+ auto stored = make_gcp_vault();
+ ObjectStoreInfoPB update;
+ update.mutable_credential()->mutable_gcp_credential()->set_credential_provider_type(source);
+ auto error = apply_obj_credential(update, &stored);
+ ASSERT_FALSE(error.has_value()) << error.value_or("");
+ EXPECT_EQ(stored.credential().gcp_credential().credential_provider_type(), source);
+ EXPECT_EQ(stored.credential().gcp_credential().impersonation_service_account(), ACCOUNT);
+ }
+}
+
+TEST(ObjCredentialTest, ExplicitEmptyTargetClearsOnlyImpersonation) {
+ auto stored = make_gcp_vault();
+ ObjectStoreInfoPB update;
+ update.mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account("");
+ auto error = apply_obj_credential(update, &stored);
+ ASSERT_FALSE(error.has_value()) << error.value_or("");
+ EXPECT_EQ(stored.credential().gcp_credential().credential_provider_type(),
+ GcpCredentialPB::COMPUTE_ENGINE);
+ EXPECT_FALSE(stored.credential().gcp_credential().has_impersonation_service_account());
+}
+
+TEST(ObjCredentialTest, SwitchFromStaticDefaultsSourceAndRemovesLegacyCredentials) {
+ for (auto account : {NEW_ACCOUNT, ""}) {
+ for (bool explicit_source : {false, true}) {
+ auto stored = make_gcp_vault();
+ stored.clear_credential();
+ stored.set_ak("encrypted-ak");
+ stored.set_sk("encrypted-sk");
+ stored.mutable_encryption_info()->set_key_id(1);
+ ObjectStoreInfoPB update;
+ auto* credential = update.mutable_credential()->mutable_gcp_credential();
+ credential->set_impersonation_service_account(account);
+ if (explicit_source) {
+ credential->set_credential_provider_type(GcpCredentialPB::COMPUTE_ENGINE);
+ }
+ auto error = apply_obj_credential(update, &stored);
+ ASSERT_FALSE(error.has_value()) << error.value_or("");
+ EXPECT_TRUE(stored.credential().gcp_credential().has_credential_provider_type());
+ EXPECT_EQ(stored.credential().gcp_credential().credential_provider_type(),
+ explicit_source ? GcpCredentialPB::COMPUTE_ENGINE : GcpCredentialPB::DEFAULT);
+ EXPECT_EQ(stored.credential().gcp_credential().impersonation_service_account(),
+ account);
+ EXPECT_FALSE(stored.has_ak());
+ EXPECT_FALSE(stored.has_sk());
+ EXPECT_FALSE(stored.has_encryption_info());
+ EXPECT_FALSE(stored.has_cred_provider_type());
+ EXPECT_FALSE(stored.has_role_arn());
+ EXPECT_FALSE(stored.has_external_id());
+ }
+ }
+}
+
+TEST(ObjCredentialTest, RejectedPatchDoesNotMutateStoredCredential) {
+ for (bool native : {false, true}) {
+ auto stored = make_gcp_vault();
+ if (!native) {
+ stored.clear_credential();
+ stored.set_ak("encrypted-ak");
+ stored.set_sk("encrypted-sk");
+ stored.mutable_encryption_info()->set_key_id(1);
+ }
+ const auto original = stored.SerializeAsString();
+ ObjectStoreInfoPB update;
+ update.mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account(
+ "invalid-account");
+ EXPECT_TRUE(apply_obj_credential(update, &stored).has_value());
+ EXPECT_EQ(stored.SerializeAsString(), original);
+
+ update.mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account(
+ NEW_ACCOUNT);
+ update.set_ak("conflicting-ak");
+ EXPECT_TRUE(apply_obj_credential(update, &stored).has_value());
+ EXPECT_EQ(stored.SerializeAsString(), original);
+ }
+}
+
+TEST(ObjCredentialTest, RejectsEmptyPatchAndCrossProviderUpdate) {
+ auto stored = make_gcp_vault();
+ auto original = stored.SerializeAsString();
+ ObjectStoreInfoPB update;
+ update.mutable_credential()->mutable_gcp_credential();
+ EXPECT_TRUE(apply_obj_credential(update, &stored).has_value());
+ EXPECT_EQ(stored.SerializeAsString(), original);
+
+ stored.set_provider(ObjectStoreInfoPB::S3);
+ original = stored.SerializeAsString();
+ update.mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account(
+ NEW_ACCOUNT);
+ EXPECT_TRUE(apply_obj_credential(update, &stored).has_value());
+ EXPECT_EQ(stored.SerializeAsString(), original);
+}
+
+} // namespace
+} // namespace doris::cloud
diff --git a/cloud/test/s3_accessor_test.cpp b/cloud/test/s3_accessor_test.cpp
index d63baa2a7b5..62a606d768e 100644
--- a/cloud/test/s3_accessor_test.cpp
+++ b/cloud/test/s3_accessor_test.cpp
@@ -27,6 +27,7 @@
#include <chrono>
#include <unordered_set>
+#include "common/auth/obj_credential.h"
#include "common/config.h"
#include "common/configbase.h"
#include "common/logging.h"
@@ -393,6 +394,44 @@ TEST_F(S3AccessorTest, path_style_test) {
}
}
+TEST(S3AccessorConfigTest, GcpCredentialFromObjectStoreInfo) {
+ ObjectStoreInfoPB obj_info;
+ obj_info.set_provider(ObjectStoreInfoPB::GCP);
+ obj_info.set_endpoint("storage.googleapis.com");
+ obj_info.set_region("us-east1");
+ obj_info.set_bucket("test-bucket");
+ auto* gcp_credential = obj_info.mutable_credential()->mutable_gcp_credential();
+ gcp_credential->set_credential_provider_type(GcpCredentialPB::COMPUTE_ENGINE);
+ gcp_credential->set_impersonation_service_account("target@my-project.iam.gserviceaccount.com");
+
+ auto s3_conf = S3Conf::from_obj_store_info(obj_info);
+ ASSERT_TRUE(s3_conf.has_value());
+ EXPECT_EQ(s3_conf->provider, S3Conf::GCS);
+ ASSERT_TRUE(std::holds_alternative<doris::GcpCredentialConfig>(s3_conf->credential));
+ const auto& credential = std::get<doris::GcpCredentialConfig>(s3_conf->credential);
+ EXPECT_EQ(credential.provider_type, doris::GcpCredentialProviderType::ComputeEngine);
+ EXPECT_EQ(credential.impersonation_service_account,
+ "target@my-project.iam.gserviceaccount.com");
+}
+
+TEST(S3AccessorConfigTest, ValidateGcpStorageVaultCredential) {
+ ObjectStoreInfoPB obj_info;
+ obj_info.set_provider(ObjectStoreInfoPB::GCP);
+ obj_info.set_endpoint("storage.googleapis.com");
+ obj_info.set_region("us-east1");
+ obj_info.set_bucket("test-bucket");
+ obj_info.mutable_credential()->mutable_gcp_credential()->set_credential_provider_type(
+ GcpCredentialPB::DEFAULT);
+
+ EXPECT_FALSE(validate_obj_credential(obj_info).has_value());
+
+ obj_info.set_ak("ak");
+ obj_info.set_sk("sk");
+ auto error = validate_obj_credential(obj_info);
+ ASSERT_TRUE(error.has_value());
+ EXPECT_NE(error->find("cannot be combined"), std::string::npos);
+}
+
class S3AccessorRoleTest : public testing::Test {
static void SetUpTestSuite() {
if (!std::getenv("AWS_ROLE_ARN") || !std::getenv("AWS_EXTERNAL_ID") ||
diff --git a/common/cpp/obj-client/auth/gcp/gcp_auth.cpp b/common/cpp/obj-client/auth/gcp/gcp_auth.cpp
new file mode 100644
index 00000000000..801b7b5c7f6
--- /dev/null
+++ b/common/cpp/obj-client/auth/gcp/gcp_auth.cpp
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cpp/obj-client/auth/gcp/gcp_auth.h"
+
+#include <algorithm>
+#include <cctype>
+#include <utility>
+
+namespace doris {
+
+bool is_valid_gcp_service_account_email(std::string_view email) {
+ constexpr std::string_view user_managed_suffix = ".iam.gserviceaccount.com";
+ constexpr std::string_view compute_engine_domain = "developer.gserviceaccount.com";
+ constexpr std::string_view app_engine_domain = "appspot.gserviceaccount.com";
+ auto at = email.find('@');
+ if (at == std::string_view::npos || at == 0 ||
+ email.find('@', at + 1) != std::string_view::npos ||
+ std::any_of(email.begin(), email.end(), [](unsigned char c) { return std::isspace(c); })) {
+ return false;
+ }
+
+ auto domain = email.substr(at + 1);
+ return domain == compute_engine_domain || domain == app_engine_domain ||
+ (domain.size() > user_managed_suffix.size() && domain.ends_with(user_managed_suffix));
+}
+
+std::optional<GcpCredentialProviderType> parse_gcp_credential_provider_type(
+ std::string_view provider_type) {
+ std::string normalized(provider_type);
+ normalized.erase(normalized.begin(),
+ std::find_if(normalized.begin(), normalized.end(),
+ [](unsigned char c) { return !std::isspace(c); }));
+ normalized.erase(std::find_if(normalized.rbegin(), normalized.rend(),
+ [](unsigned char c) { return !std::isspace(c); })
+ .base(),
+ normalized.end());
+ std::transform(normalized.begin(), normalized.end(), normalized.begin(),
+ [](unsigned char c) { return static_cast<char>(std::toupper(c)); });
+ if (normalized == "DEFAULT") {
+ return GcpCredentialProviderType::Default;
+ }
+ if (normalized == "COMPUTE_ENGINE") {
+ return GcpCredentialProviderType::ComputeEngine;
+ }
+ return std::nullopt;
+}
+
+std::optional<std::string> validate_gcp_credential(const GcpCredentialConfig& credential) {
+ if (credential.impersonation_service_account.empty()) {
+ return std::nullopt;
+ }
+ if (!is_valid_gcp_service_account_email(credential.impersonation_service_account)) {
+ return "Invalid GCP service account email";
+ }
+ return std::nullopt;
+}
+
+GcpCredentialParseResult parse_gcp_credential_properties(
+ const ObjCredentialPropertyGetter& get_property) {
+ auto provider_type_value = get_property(GCP_CREDENTIAL_PROVIDER_TYPE);
+ auto impersonation_service_account = get_property(GCP_IMPERSONATION_SERVICE_ACCOUNT);
+ if (!provider_type_value.has_value() && !impersonation_service_account.has_value()) {
+ return {};
+ }
+ auto provider_type = provider_type_value.has_value()
+ ? parse_gcp_credential_provider_type(*provider_type_value)
+ : GcpCredentialProviderType::Default;
+ if (!provider_type.has_value()) {
+ return {.error = "invalid " + std::string(GCP_CREDENTIAL_PROVIDER_TYPE) + " value \"" +
+ *provider_type_value +
+ "\"; supported values are DEFAULT and COMPUTE_ENGINE"};
+ }
+
+ GcpCredentialConfig config {.provider_type = *provider_type};
+ if (impersonation_service_account.has_value()) {
+ config.impersonation_service_account = std::move(*impersonation_service_account);
+ }
+ if (auto error = validate_gcp_credential(config); error.has_value()) {
+ return {.error = std::move(error)};
+ }
+ return {.credential = std::move(config)};
+}
+
+} // namespace doris
diff --git a/common/cpp/obj-client/auth/gcp/gcp_auth.h b/common/cpp/obj-client/auth/gcp/gcp_auth.h
new file mode 100644
index 00000000000..99b35f7be50
--- /dev/null
+++ b/common/cpp/obj-client/auth/gcp/gcp_auth.h
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <optional>
+#include <string>
+#include <string_view>
+
+#include "cpp/obj-client/auth/obj_credential_common.h"
+
+namespace doris {
+
+// Provider-specific authentication lives under cpp/obj-client/auth/<provider>. Future
+// providers should use sibling packages and expose only their normalized config,
+// validation, and client-setup entry points there.
+
+inline constexpr std::string_view GCP_CREDENTIAL_PROVIDER_TYPE = "gs.credential_provider_type";
+inline constexpr std::string_view GCP_IMPERSONATION_SERVICE_ACCOUNT =
+ "gs.impersonation_service_account";
+
+enum class GcpCredentialProviderType {
+ Default,
+ ComputeEngine,
+};
+
+struct GcpCredentialConfig {
+ GcpCredentialProviderType provider_type = GcpCredentialProviderType::Default;
+ std::string impersonation_service_account {};
+
+ bool operator==(const GcpCredentialConfig&) const = default;
+};
+
+struct GcpCredentialParseResult {
+ std::optional<GcpCredentialConfig> credential = std::nullopt;
+ std::optional<std::string> error = std::nullopt;
+};
+
+bool is_valid_gcp_service_account_email(std::string_view email);
+std::optional<GcpCredentialProviderType> parse_gcp_credential_provider_type(
+ std::string_view provider_type);
+std::optional<std::string> validate_gcp_credential(const GcpCredentialConfig& credential);
+GcpCredentialParseResult parse_gcp_credential_properties(
+ const ObjCredentialPropertyGetter& get_property);
+
+} // namespace doris
diff --git a/common/cpp/obj-client/auth/gcp/gcp_s3_client.cpp b/common/cpp/obj-client/auth/gcp/gcp_s3_client.cpp
new file mode 100644
index 00000000000..0ad7ba19d2c
--- /dev/null
+++ b/common/cpp/obj-client/auth/gcp/gcp_s3_client.cpp
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cpp/obj-client/auth/gcp/gcp_s3_client.h"
+
+#include <aws/core/auth/AWSCredentialsProvider.h>
+
+#include <utility>
+
+namespace doris {
+
+GcpS3Client::GcpS3Client(const GcpCredentialConfig& credential, std::string ca_cert_path,
+ Aws::Client::ClientConfiguration aws_config,
+ Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy payload_signing_policy,
+ bool use_virtual_addressing)
+ : Aws::S3::S3Client(std::make_shared<Aws::Auth::AnonymousAWSCredentialsProvider>(),
+ std::move(aws_config), payload_signing_policy, use_virtual_addressing),
+ _token_provider(credential, ca_cert_path) {}
+
+void GcpS3Client::BuildHttpRequest(
+ const Aws::AmazonWebServiceRequest& request,
+ const std::shared_ptr<Aws::Http::HttpRequest>& http_request) const {
+ Aws::S3::S3Client::BuildHttpRequest(request, http_request);
+ auto token = fetch_token();
+ if (!token.has_value()) {
+ // GCS will return a useful 401 response. The token provider has already
+ // logged the underlying credential or metadata error.
+ return;
+ }
+ http_request->SetHeaderValue("Authorization", Aws::String("Bearer ") + token->c_str());
+}
+
+std::optional<std::string> GcpS3Client::fetch_token() const {
+ return _token_provider.get_token();
+}
+
+} // namespace doris
diff --git a/common/cpp/obj-client/auth/gcp/gcp_s3_client.h b/common/cpp/obj-client/auth/gcp/gcp_s3_client.h
new file mode 100644
index 00000000000..4e15f3255b3
--- /dev/null
+++ b/common/cpp/obj-client/auth/gcp/gcp_s3_client.h
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <aws/core/client/ClientConfiguration.h>
+#include <aws/core/http/HttpRequest.h>
+#include <aws/s3/S3Client.h>
+
+#include <memory>
+#include <optional>
+#include <string>
+
+#include "cpp/obj-client/auth/gcp/gcp_auth.h"
+#include "cpp/obj-client/auth/gcp/gcp_token_provider.h"
+
+namespace doris {
+
+// GCS accepts OAuth2 bearer tokens on its S3-compatible XML API. The client uses
+// anonymous AWS credentials so AWSAuthV4Signer returns without adding SigV4, and
+// BuildHttpRequest injects the bearer header immediately before that signing step.
+class GcpS3Client : public Aws::S3::S3Client {
+public:
+ GcpS3Client(const GcpCredentialConfig& credential, std::string ca_cert_path,
+ Aws::Client::ClientConfiguration aws_config,
+ Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy payload_signing_policy,
+ bool use_virtual_addressing);
+
+protected:
+ void BuildHttpRequest(
+ const Aws::AmazonWebServiceRequest& request,
+ const std::shared_ptr<Aws::Http::HttpRequest>& http_request) const override;
+
+ // Test seam for asserting the outgoing header without relying on ADC or metadata.
+ virtual std::optional<std::string> fetch_token() const;
+
+private:
+ GcpTokenProvider _token_provider;
+};
+
+} // namespace doris
diff --git a/common/cpp/obj-client/auth/gcp/gcp_token_provider.cpp b/common/cpp/obj-client/auth/gcp/gcp_token_provider.cpp
new file mode 100644
index 00000000000..596c4370dac
--- /dev/null
+++ b/common/cpp/obj-client/auth/gcp/gcp_token_provider.cpp
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cpp/obj-client/auth/gcp/gcp_token_provider.h"
+
+#include <glog/logging.h>
+#include <google/cloud/credentials.h>
+#include <google/cloud/oauth2/access_token_generator.h>
+
+#include <utility>
+
+namespace doris {
+namespace {
+
+constexpr char GCS_READ_WRITE_SCOPE[] = "https://www.googleapis.com/auth/devstorage.read_write";
+constexpr char CLOUD_PLATFORM_SCOPE[] = "https://www.googleapis.com/auth/cloud-platform";
+
+std::shared_ptr<google::cloud::Credentials> make_credentials(const GcpCredentialConfig& credential,
+ const std::string& ca_cert_path,
+ GcpTokenScope token_scope) {
+ google::cloud::Options options;
+ if (!ca_cert_path.empty()) {
+ options.set<google::cloud::CARootsFilePathOption>(ca_cert_path);
+ }
+
+ // A source credential used for impersonation calls the IAM Credentials API
+ // and therefore needs cloud-platform. The resulting target token uses the
+ // caller-selected scope: object read/write for storage requests, or
+ // cloud-platform when it will call IAM signBlob.
+ const char* requested_scope = token_scope == GcpTokenScope::CloudPlatform
+ ? CLOUD_PLATFORM_SCOPE
+ : GCS_READ_WRITE_SCOPE;
+ options.set<google::cloud::ScopesOption>({credential.impersonation_service_account.empty()
+ ? requested_scope
+ : CLOUD_PLATFORM_SCOPE});
+
+ std::shared_ptr<google::cloud::Credentials> credentials;
+ switch (credential.provider_type) {
+ case GcpCredentialProviderType::Default:
+ credentials = google::cloud::MakeGoogleDefaultCredentials(options);
+ break;
+ case GcpCredentialProviderType::ComputeEngine:
+ credentials = google::cloud::MakeComputeEngineCredentials(options);
+ break;
+ }
+ if (!credential.impersonation_service_account.empty()) {
+ auto impersonation_options = options;
+ impersonation_options.set<google::cloud::ScopesOption>({requested_scope});
+ credentials = google::cloud::MakeImpersonateServiceAccountCredentials(
+ std::move(credentials), credential.impersonation_service_account,
+ std::move(impersonation_options));
+ }
+ return credentials;
+}
+
+} // namespace
+
+GcpTokenProvider::GcpTokenProvider(const GcpCredentialConfig& credential,
+ const std::string& ca_cert_path, GcpTokenScope token_scope)
+ : _credentials(make_credentials(credential, ca_cert_path, token_scope)),
+ _token_generator(google::cloud::oauth2::MakeAccessTokenGenerator(*_credentials)) {}
+
+GcpTokenProvider::~GcpTokenProvider() = default;
+
+std::optional<std::string> GcpTokenProvider::get_token() const {
+ // GcpS3Client and presign callers may share one provider across request
+ // threads. Serialize access even if the underlying implementation changes
+ // its thread-safety guarantees in a future google-cloud-cpp release.
+ std::lock_guard lock(_token_generator_mutex);
+ auto token = _token_generator->GetToken();
+ if (!token) {
+ LOG_EVERY_N(WARNING, 100) << "failed to obtain GCP access token: "
+ << token.status().message();
+ return std::nullopt;
+ }
+ return token->token;
+}
+
+} // namespace doris
diff --git a/common/cpp/obj-client/auth/gcp/gcp_token_provider.h b/common/cpp/obj-client/auth/gcp/gcp_token_provider.h
new file mode 100644
index 00000000000..8c39aa2b2dc
--- /dev/null
+++ b/common/cpp/obj-client/auth/gcp/gcp_token_provider.h
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <google/cloud/credentials.h>
+#include <google/cloud/oauth2/access_token_generator.h>
+
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+
+#include "cpp/obj-client/auth/gcp/gcp_auth.h"
+
+namespace doris {
+
+enum class GcpTokenScope {
+ StorageReadWrite,
+ CloudPlatform,
+};
+
+// Owns both objects because MakeAccessTokenGenerator takes Credentials by const
+// reference. The Google auth library handles token caching and refresh internally.
+class GcpTokenProvider {
+public:
+ explicit GcpTokenProvider(const GcpCredentialConfig& credential,
+ const std::string& ca_cert_path = "",
+ GcpTokenScope token_scope = GcpTokenScope::StorageReadWrite);
+ ~GcpTokenProvider();
+
+ GcpTokenProvider(const GcpTokenProvider&) = delete;
+ GcpTokenProvider& operator=(const GcpTokenProvider&) = delete;
+
+ std::optional<std::string> get_token() const;
+
+private:
+ std::shared_ptr<google::cloud::Credentials> _credentials;
+ std::shared_ptr<google::cloud::oauth2::AccessTokenGenerator> _token_generator;
+ mutable std::mutex _token_generator_mutex;
+};
+
+} // namespace doris
diff --git a/common/cpp/obj-client/auth/gcp/gcs_signed_url.cpp b/common/cpp/obj-client/auth/gcp/gcs_signed_url.cpp
new file mode 100644
index 00000000000..9adbd09818c
--- /dev/null
+++ b/common/cpp/obj-client/auth/gcp/gcs_signed_url.cpp
@@ -0,0 +1,194 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cpp/obj-client/auth/gcp/gcs_signed_url.h"
+
+#include <fmt/format.h>
+#include <openssl/sha.h>
+
+#include <algorithm>
+#include <array>
+#include <ctime>
+#include <optional>
+#include <utility>
+
+#include "cpp/obj-client/auth/gcp/gcp_auth.h"
+
+namespace doris {
+namespace {
+
+constexpr std::string_view SIGNING_ALGORITHM = "GOOG4-RSA-SHA256";
+constexpr std::string_view SIGNED_HEADERS = "host";
+constexpr std::string_view SIGNING_REGION = "auto";
+constexpr std::string_view SIGNING_SERVICE = "storage";
+constexpr std::string_view SIGNING_TERMINATOR = "goog4_request";
+constexpr int64_t MAX_EXPIRATION_SECONDS = 7 * 24 * 60 * 60;
+
+struct Endpoint {
+ std::string scheme;
+ std::string authority;
+};
+
+bool is_unreserved(unsigned char c) {
+ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' ||
+ c == '_' || c == '.' || c == '~';
+}
+
+std::string percent_encode(std::string_view value, bool preserve_slash) {
+ constexpr char HEX[] = "0123456789ABCDEF";
+ std::string encoded;
+ encoded.reserve(value.size());
+ for (unsigned char c : value) {
+ if (is_unreserved(c) || (preserve_slash && c == '/')) {
+ encoded.push_back(static_cast<char>(c));
+ continue;
+ }
+ encoded.push_back('%');
+ encoded.push_back(HEX[c >> 4]);
+ encoded.push_back(HEX[c & 0x0F]);
+ }
+ return encoded;
+}
+
+std::string hex_encode(std::string_view value) {
+ constexpr char HEX[] = "0123456789abcdef";
+ std::string encoded;
+ encoded.reserve(value.size() * 2);
+ for (unsigned char c : value) {
+ encoded.push_back(HEX[c >> 4]);
+ encoded.push_back(HEX[c & 0x0F]);
+ }
+ return encoded;
+}
+
+std::string sha256_hex(std::string_view value) {
+ std::array<unsigned char, SHA256_DIGEST_LENGTH> digest {};
+ SHA256(reinterpret_cast<const unsigned char*>(value.data()), value.size(), digest.data());
+ return hex_encode(
+ std::string_view(reinterpret_cast<const char*>(digest.data()), digest.size()));
+}
+
+std::optional<std::string> parse_endpoint(std::string endpoint, Endpoint* parsed) {
+ if (endpoint.empty()) {
+ return "GCS endpoint must not be empty";
+ }
+ auto scheme_end = endpoint.find("://");
+ if (scheme_end == std::string::npos) {
+ parsed->scheme = "https";
+ } else {
+ parsed->scheme = endpoint.substr(0, scheme_end);
+ endpoint.erase(0, scheme_end + 3);
+ }
+ if (parsed->scheme != "http" && parsed->scheme != "https") {
+ return fmt::format("unsupported GCS endpoint scheme: {}", parsed->scheme);
+ }
+ while (endpoint.ends_with('/')) {
+ endpoint.pop_back();
+ }
+ if (endpoint.empty() || endpoint.find_first_of("/?#") != std::string::npos) {
+ return fmt::format(
+ "GCS signed URL endpoint must not contain a path, query, "
+ "or fragment: {}",
+ endpoint);
+ }
+ parsed->authority = std::move(endpoint);
+ return std::nullopt;
+}
+
+std::optional<std::string> format_signing_time(std::chrono::system_clock::time_point now,
+ std::string* timestamp, std::string* date) {
+ auto time = std::chrono::system_clock::to_time_t(now);
+ std::tm utc {};
+#if defined(_WIN32)
+ if (gmtime_s(&utc, &time) != 0) {
+ return "failed to convert GCS signing time to UTC";
+ }
+#else
+ if (gmtime_r(&time, &utc) == nullptr) {
+ return "failed to convert GCS signing time to UTC";
+ }
+#endif
+ char timestamp_buffer[17];
+ char date_buffer[9];
+ if (std::strftime(timestamp_buffer, sizeof(timestamp_buffer), "%Y%m%dT%H%M%SZ", &utc) == 0 ||
+ std::strftime(date_buffer, sizeof(date_buffer), "%Y%m%d", &utc) == 0) {
+ return "failed to format GCS signing time";
+ }
+ *timestamp = timestamp_buffer;
+ *date = date_buffer;
+ return std::nullopt;
+}
+
+} // namespace
+
+GcsV4SignedUrlResult build_gcs_v4_signed_url(const GcsV4SignedUrlOptions& options,
+ std::chrono::system_clock::time_point now,
+ const GcsSignBlobFunction& sign_blob) {
+ if (options.bucket.empty()) {
+ return {.error = "GCS bucket must not be empty"};
+ }
+ if (!is_valid_gcp_service_account_email(options.signer_email)) {
+ return {.error = "invalid GCS signing service account email"};
+ }
+ if (options.expiration_secs <= 0 || options.expiration_secs > MAX_EXPIRATION_SECONDS) {
+ return {.error = fmt::format("GCS signed URL expiration must be in [1, {}] seconds",
+ MAX_EXPIRATION_SECONDS)};
+ }
+ if (!sign_blob) {
+ return {.error = "GCS signBlob callback must not be empty"};
+ }
+
+ Endpoint endpoint;
+ if (auto error = parse_endpoint(options.endpoint, &endpoint); error.has_value()) {
+ return {.error = std::move(*error)};
+ }
+ std::string timestamp;
+ std::string date;
+ if (auto error = format_signing_time(now, ×tamp, &date); error.has_value()) {
+ return {.error = std::move(*error)};
+ }
+
+ const std::string canonical_uri =
+ "/" + percent_encode(options.bucket, false) + "/" + percent_encode(options.key, true);
+ const std::string credential_scope =
+ fmt::format("{}/{}/{}/{}", date, SIGNING_REGION, SIGNING_SERVICE, SIGNING_TERMINATOR);
+ const std::string credential = options.signer_email + "/" + credential_scope;
+ const std::string canonical_query = fmt::format(
+ "X-Goog-Algorithm={}&X-Goog-Credential={}&X-Goog-Date={}&"
+ "X-Goog-Expires={}&X-Goog-SignedHeaders={}",
+ SIGNING_ALGORITHM, percent_encode(credential, false), timestamp,
+ options.expiration_secs, SIGNED_HEADERS);
+ const std::string canonical_request =
+ fmt::format("GET\n{}\n{}\nhost:{}\n\n{}\nUNSIGNED-PAYLOAD", canonical_uri,
+ canonical_query, endpoint.authority, SIGNED_HEADERS);
+ const std::string string_to_sign = fmt::format("{}\n{}\n{}\n{}", SIGNING_ALGORITHM, timestamp,
+ credential_scope, sha256_hex(canonical_request));
+
+ auto sign_result = sign_blob(string_to_sign);
+ if (!sign_result.ok()) {
+ return {.error = std::move(sign_result.error)};
+ }
+ if (sign_result.signature.empty()) {
+ return {.error = "signBlob returned an empty signature"};
+ }
+
+ return {.signed_url = fmt::format("{}://{}{}?{}&X-Goog-Signature={}", endpoint.scheme,
+ endpoint.authority, canonical_uri, canonical_query,
+ hex_encode(sign_result.signature))};
+}
+
+} // namespace doris
diff --git a/common/cpp/obj-client/auth/gcp/gcs_signed_url.h b/common/cpp/obj-client/auth/gcp/gcs_signed_url.h
new file mode 100644
index 00000000000..8cc152925b4
--- /dev/null
+++ b/common/cpp/obj-client/auth/gcp/gcs_signed_url.h
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <chrono>
+#include <cstdint>
+#include <functional>
+#include <string>
+#include <string_view>
+
+namespace doris {
+
+struct GcsV4SignedUrlOptions {
+ std::string endpoint;
+ std::string bucket;
+ std::string key;
+ // The transport layer resolves this identity before invoking the builder.
+ std::string signer_email;
+ int64_t expiration_secs = 0;
+};
+
+struct GcsSignBlobResult {
+ std::string signature {};
+ std::string error {};
+
+ bool ok() const noexcept { return error.empty(); }
+};
+
+// Returns the raw signature bytes for the supplied GCS V4 string-to-sign. The
+// callback intentionally contains no transport-specific type so this signing
+// algorithm can be shared by BE and cloud targets.
+using GcsSignBlobFunction = std::function<GcsSignBlobResult(std::string_view string_to_sign)>;
+
+struct GcsV4SignedUrlResult {
+ std::string signed_url {};
+ std::string error {};
+
+ bool ok() const noexcept { return error.empty(); }
+};
+
+// Builds a GCS V4 signed URL from a caller-supplied signing function. Resolving
+// credentials and invoking IAM signBlob belong to the caller's transport layer.
+GcsV4SignedUrlResult build_gcs_v4_signed_url(const GcsV4SignedUrlOptions& options,
+ std::chrono::system_clock::time_point now,
+ const GcsSignBlobFunction& sign_blob);
+
+} // namespace doris
diff --git a/common/cpp/obj-client/auth/obj_credential.cpp b/common/cpp/obj-client/auth/obj_credential.cpp
new file mode 100644
index 00000000000..8317059e34e
--- /dev/null
+++ b/common/cpp/obj-client/auth/obj_credential.cpp
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cpp/obj-client/auth/obj_credential.h"
+
+#include <array>
+#include <functional>
+#include <utility>
+
+namespace doris {
+namespace {
+
+struct CredentialParseResult {
+ CredentialConfig credential {};
+ std::optional<std::string> error = std::nullopt;
+};
+
+// Adapt provider-owned parsers to the generic registry. A provider parser
+// returns no credential when none of its properties are present and an error
+// for a partially configured credential.
+CredentialParseResult parse_gcp_properties(const ObjCredentialPropertyGetter& get_property) {
+ auto result = parse_gcp_credential_properties(get_property);
+ if (result.error.has_value()) {
+ return {.error = std::move(result.error)};
+ }
+ if (!result.credential.has_value()) {
+ return {};
+ }
+ return {.credential = std::move(*result.credential)};
+}
+
+using CredentialPropertyParser =
+ CredentialParseResult (*)(const ObjCredentialPropertyGetter& get_property);
+
+// Register exactly one parser per non-monostate CredentialConfig alternative.
+// The loop below also rejects properties from multiple credential providers.
+// Add parse_oss_credential_properties/parse_azure_credential_properties here
+// when those passwordless modes are introduced.
+constexpr std::array CREDENTIAL_PROPERTY_PARSERS {
+ CredentialPropertyParser {parse_gcp_properties},
+};
+
+static_assert(CREDENTIAL_PROPERTY_PARSERS.size() + 1 == std::variant_size_v<CredentialConfig>);
+
+// These visitors are the runtime extension points. Do not add a catch-all
+// template overload: explicit overloads make a new CredentialConfig type fail
+// to compile until its hashing and validation are implemented.
+struct CredentialHashVisitor {
+ uint64_t* hash;
+
+ void operator()(const std::monostate&) const {}
+
+ void operator()(const GcpCredentialConfig& credential) const {
+ *hash ^= static_cast<size_t>(credential.provider_type) + 1;
+ *hash ^= std::hash<std::string> {}(credential.impersonation_service_account);
+ }
+};
+
+struct CredentialValidationVisitor {
+ const ObjCredentialValidationContext& context;
+
+ std::optional<std::string> operator()(const std::monostate&) const { return std::nullopt; }
+
+ std::optional<std::string> operator()(const GcpCredentialConfig& credential) const {
+ if (context.provider != ObjCredentialProvider::Gcp) {
+ return "GCP credentials require provider=GCP";
+ }
+ if (context.has_conflicting_credentials()) {
+ return "GCP credentials cannot be combined with other credentials";
+ }
+ return validate_gcp_credential(credential);
+ }
+};
+
+struct CredentialTypeNameVisitor {
+ std::string_view operator()(const std::monostate&) const { return "none"; }
+ std::string_view operator()(const GcpCredentialConfig&) const { return "gcp"; }
+};
+
+} // namespace
+
+void hash_combine_credential(uint64_t* hash, const CredentialConfig& credential) {
+ *hash ^= credential.index();
+ std::visit(CredentialHashVisitor {hash}, credential);
+}
+
+std::string_view credential_type_name(const CredentialConfig& credential) {
+ return std::visit(CredentialTypeNameVisitor {}, credential);
+}
+
+std::optional<std::string> validate_obj_credential_config(
+ const CredentialConfig& credential, const ObjCredentialValidationContext& context) {
+ return std::visit(CredentialValidationVisitor {context}, credential);
+}
+
+std::optional<std::string> parse_obj_credential_properties(
+ const ObjCredentialPropertyGetter& get_property, CredentialConfig* credential) {
+ CredentialConfig selected;
+
+ for (auto parser : CREDENTIAL_PROPERTY_PARSERS) {
+ auto result = parser(get_property);
+ if (result.error.has_value()) {
+ return std::move(result.error);
+ }
+ if (std::holds_alternative<std::monostate>(result.credential)) {
+ continue;
+ }
+ if (!std::holds_alternative<std::monostate>(selected)) {
+ return "multiple object storage credential types are configured";
+ }
+ selected = std::move(result.credential);
+ }
+
+ if (!std::holds_alternative<std::monostate>(selected)) {
+ *credential = std::move(selected);
+ }
+ return std::nullopt;
+}
+
+} // namespace doris
diff --git a/common/cpp/obj-client/auth/obj_credential.h b/common/cpp/obj-client/auth/obj_credential.h
new file mode 100644
index 00000000000..6d32385588d
--- /dev/null
+++ b/common/cpp/obj-client/auth/obj_credential.h
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <variant>
+
+#include "cpp/obj-client/auth/gcp/gcp_auth.h"
+#include "cpp/obj-client/auth/obj_credential_common.h"
+
+namespace doris {
+
+class TCredential;
+
+namespace cloud {
+class ObjectStoreInfoPB;
+}
+
+// This abstraction keeps provider-specific passwordless authentication out of
+// S3 utilities and provides one runtime credential model shared by BE and Cloud.
+// To add another provider (for example OSS or Azure):
+// 1. add <Provider>CredentialConfig in its own <provider>_auth.h;
+// 2. add that type to this variant; and
+// 3. implement the corresponding hash/validate visitors, client builder, and
+// protocol/property conversions.
+// The explicit visitors intentionally make an incomplete extension fail to compile.
+using CredentialConfig = std::variant<std::monostate, GcpCredentialConfig>;
+
+void hash_combine_credential(uint64_t* hash, const CredentialConfig& credential);
+
+std::string_view credential_type_name(const CredentialConfig& credential);
+
+std::optional<std::string> validate_obj_credential_config(
+ const CredentialConfig& credential, const ObjCredentialValidationContext& context);
+
+std::optional<std::string> parse_obj_credential_properties(
+ const ObjCredentialPropertyGetter& get_property, CredentialConfig* credential);
+
+template <typename Compare, typename Allocator>
+std::optional<std::string> parse_obj_credential_properties(
+ const std::map<std::string, std::string, Compare, Allocator>& properties,
+ CredentialConfig* credential) {
+ return parse_obj_credential_properties(make_property_getter(properties), credential);
+}
+
+void convert_obj_credential(const cloud::ObjectStoreInfoPB& source, CredentialConfig* credential);
+void convert_obj_credential(const TCredential& source, CredentialConfig* credential);
+
+} // namespace doris
diff --git a/common/cpp/obj-client/auth/obj_credential_common.h b/common/cpp/obj-client/auth/obj_credential_common.h
new file mode 100644
index 00000000000..0e855fe57f1
--- /dev/null
+++ b/common/cpp/obj-client/auth/obj_credential_common.h
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <functional>
+#include <optional>
+#include <string>
+#include <string_view>
+
+namespace doris {
+
+// Credential-layer provider identity. Storage and protocol enums are converted
+// to this type only at their respective validation boundaries.
+enum class ObjCredentialProvider {
+ Unknown,
+ Aws,
+ Azure,
+ Bos,
+ Cos,
+ Gcp,
+ Obs,
+ Oss,
+ Tos,
+};
+
+// TODO: Add AwsCredentialConfig to CredentialConfig so AWS uses the same
+// provider-neutral path. The AWS-specific role ARN, external ID, and credential-
+// provider fields remain outside this abstraction for compatibility with existing
+// configurations and protocol clients. AK/SK and session tokens are common static
+// credentials for S3-compatible providers and are not part of this migration.
+struct ObjCredentialValidationContext {
+ ObjCredentialProvider provider = ObjCredentialProvider::Unknown;
+ std::string_view ak {};
+ std::string_view sk {};
+ std::string_view token {};
+ bool has_aws_role_arn = false;
+ bool has_aws_external_id = false;
+ bool has_aws_credential_provider = false;
+ bool has_encrypted_access_keys = false;
+
+ bool has_static_credentials() const {
+ return !ak.empty() || !sk.empty() || !token.empty() || has_encrypted_access_keys;
+ }
+
+ bool has_aws_authentication() const {
+ return has_aws_role_arn || has_aws_external_id || has_aws_credential_provider;
+ }
+
+ // A provider-native credential is mutually exclusive with both common
+ // static credentials and provider-specific authentication fields.
+ bool has_conflicting_credentials() const {
+ return has_static_credentials() || has_aws_authentication();
+ }
+};
+
+// Return an owned string so provider parsers never depend on a caller's storage
+// or on the lifetime of a temporary normalized property value.
+using ObjCredentialPropertyGetter = std::function<std::optional<std::string>(std::string_view)>;
+
+template <typename PropertyMap>
+ObjCredentialPropertyGetter make_property_getter(const PropertyMap& properties) {
+ // Preserve the caller's comparator (including StringCaseMap's
+ // case-insensitive lookup) instead of normalizing property names here.
+ return [&properties](std::string_view key) -> std::optional<std::string> {
+ auto it = properties.find(std::string(key));
+ if (it == properties.end()) {
+ return std::nullopt;
+ }
+ return it->second;
+ };
+}
+
+} // namespace doris
diff --git a/common/cpp/obj-client/auth/obj_s3_client_factory.cpp b/common/cpp/obj-client/auth/obj_s3_client_factory.cpp
new file mode 100644
index 00000000000..45ffa7e3bd4
--- /dev/null
+++ b/common/cpp/obj-client/auth/obj_s3_client_factory.cpp
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cpp/obj-client/auth/obj_s3_client_factory.h"
+
+#include <aws/s3/S3Client.h>
+
+#include <utility>
+
+#include "cpp/obj-client/auth/gcp/gcp_s3_client.h"
+
+namespace doris {
+namespace {
+
+// Keep the overloads explicit so adding another CredentialConfig alternative
+// fails to compile until its client construction path is implemented here.
+struct S3ClientBuilder {
+ S3ClientBuildContext& context;
+
+ std::shared_ptr<Aws::S3::S3Client> operator()(const std::monostate&) const {
+ return std::make_shared<Aws::S3::S3Client>(
+ context.fallback_provider, std::move(context.config),
+ context.payload_signing_policy, context.use_virtual_addressing);
+ }
+
+ std::shared_ptr<Aws::S3::S3Client> operator()(const GcpCredentialConfig& credential) const {
+ return std::make_shared<GcpS3Client>(
+ credential, context.ca_cert_path, std::move(context.config),
+ context.payload_signing_policy, context.use_virtual_addressing);
+ }
+};
+
+} // namespace
+
+std::shared_ptr<Aws::S3::S3Client> make_s3_client(const CredentialConfig& credential,
+ S3ClientBuildContext context) {
+ return std::visit(S3ClientBuilder {context}, credential);
+}
+
+} // namespace doris
diff --git a/common/cpp/obj-client/auth/obj_s3_client_factory.h b/common/cpp/obj-client/auth/obj_s3_client_factory.h
new file mode 100644
index 00000000000..bd4d104de95
--- /dev/null
+++ b/common/cpp/obj-client/auth/obj_s3_client_factory.h
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <aws/core/auth/AWSCredentialsProvider.h>
+#include <aws/core/auth/signer/AWSAuthV4Signer.h>
+#include <aws/core/client/ClientConfiguration.h>
+
+#include <memory>
+#include <string>
+
+#include "cpp/obj-client/auth/obj_credential.h"
+
+namespace Aws::S3 {
+class S3Client;
+}
+
+namespace doris {
+
+// Neutral inputs shared by BE's S3ClientConf and Cloud's S3Conf. Provider-native
+// authentication is selected before construction because AWS credentials providers
+// cannot be replaced safely on an already constructed client.
+struct S3ClientBuildContext {
+ std::shared_ptr<Aws::Auth::AWSCredentialsProvider> fallback_provider;
+ Aws::Client::ClientConfiguration config;
+ Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy payload_signing_policy =
+ Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never;
+ bool use_virtual_addressing = true;
+ std::string ca_cert_path;
+};
+
+std::shared_ptr<Aws::S3::S3Client> make_s3_client(const CredentialConfig& credential,
+ S3ClientBuildContext context);
+
+} // namespace doris
diff --git a/fe/be-java-extensions/hadoop-deps/pom.xml b/fe/be-java-extensions/hadoop-deps/pom.xml
index f7074995e0b..53aab41cec9 100644
--- a/fe/be-java-extensions/hadoop-deps/pom.xml
+++ b/fe/be-java-extensions/hadoop-deps/pom.xml
@@ -51,6 +51,11 @@ under the License.
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>gcs-connector</artifactId>
+ <classifier>shaded</classifier>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index 2d6758989d5..1b3c79d796b 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -169,6 +169,11 @@ under the License.
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>gcs-connector</artifactId>
+ <classifier>shaded</classifier>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
@@ -406,6 +411,10 @@ under the License.
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.auth</groupId>
+ <artifactId>google-auth-library-oauth2-http</artifactId>
+ </dependency>
<!-- tencent cloud sts -->
<dependency>
<groupId>com.tencentcloudapi</groupId>
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java
index 1d2e231674e..b248fd5ba53 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java
@@ -126,6 +126,20 @@ public class BrokerDesc extends StorageDesc implements Writable {
return (null != storageProperties) ? storageProperties.validateAndNormalizeUri(location) : location;
}
+ /**
+ * Normalize EXPORT and OUTFILE destinations for the BE writer.
+ */
+ public String getFileLocationForExport(String location) throws UserException {
+ // BE parses HTTP(S) paths as path-style, while FE defaults to virtual-hosted parsing.
+ // Preserve these paths and the existing path handling for non-S3 backends.
+ if (StorageType.S3.equals(getStorageType())
+ && !StringUtils.startsWithIgnoreCase(location, "http://")
+ && !StringUtils.startsWithIgnoreCase(location, "https://")) {
+ return getFileLocation(location);
+ }
+ return location;
+ }
+
public static BrokerDesc createForStreamLoad() {
return new BrokerDesc("", StorageType.STREAM, null);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
index 68d625c18fa..73697843930 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
@@ -611,6 +611,11 @@ public class OutFileClause {
}
String brokerName = copiedProps.get(PROP_BROKER_NAME);
brokerDesc = new BrokerDesc(brokerName, copiedProps);
+ try {
+ filePath = brokerDesc.getFileLocationForExport(filePath);
+ } catch (RuntimeException e) {
+ throw new AnalysisException("Invalid outfile path: " + filePath + ", " + e.getMessage(), e);
+ }
/*
* Note on HDFS export behavior and URI handling:
*
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
index 948bfe470cc..dbd10e2aa6b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
@@ -24,6 +24,7 @@ import org.apache.doris.common.proc.BaseProcResult;
import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.common.util.S3Util;
import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties;
+import org.apache.doris.datasource.property.storage.GCSProperties;
import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.fs.obj.ObjStorage;
@@ -36,6 +37,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
+import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -43,7 +45,9 @@ import java.io.ByteArrayInputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
/**
@@ -73,6 +77,20 @@ import java.util.Optional;
*/
public class S3Resource extends Resource {
private static final Logger LOG = LogManager.getLogger(S3Resource.class);
+ // Only aliases already supported by GCSProperties belong here. Resource/Vault options
+ // without a GCS alias (bucket, root path, region, validity checking) keep their existing names.
+ private static final Map<String, String> GCS_PROPERTY_ALIASES = ImmutableMap.<String, String>builder()
+ .put(GCSProperties.ENDPOINT, S3Properties.ENDPOINT)
+ .put("gs.access_key", S3Properties.ACCESS_KEY)
+ .put("gs.secret_key", S3Properties.SECRET_KEY)
+ .put("gs.session_token", S3Properties.SESSION_TOKEN)
+ .put("gs.connection.maximum", S3Properties.MAX_CONNECTIONS)
+ .put("gs.connection.request.timeout", S3Properties.REQUEST_TIMEOUT_MS)
+ .put("gs.connection.timeout", S3Properties.CONNECTION_TIMEOUT_MS)
+ .put("gs.use_path_style", S3Properties.USE_PATH_STYLE)
+ .put("gs.force_parsing_by_standard_uri", "force_parsing_by_standard_uri")
+ .build();
+
@SerializedName(value = "properties")
private Map<String, String> properties;
@@ -93,6 +111,7 @@ public class S3Resource extends Resource {
protected void setProperties(ImmutableMap<String, String> newProperties) throws DdlException {
Preconditions.checkState(newProperties != null);
this.properties = Maps.newHashMap(newProperties);
+ normalizeProperties(this.properties, this.properties.get(StorageProperties.FS_PROVIDER_KEY));
// check properties
S3Properties.requiredS3PingProperties(properties);
@@ -174,70 +193,109 @@ public class S3Resource extends Resource {
LOG.info("success to ping s3");
}
+ private static void normalizeProperties(Map<String, String> properties, String provider) {
+ // Validate the raw aliases before a preferred value can hide conflicting credentials.
+ GCSProperties.validateNativeCredentialCompatibility(properties);
+ if (provider == null) {
+ return;
+ }
+ switch (provider.toUpperCase(Locale.ROOT)) {
+ case "GCP":
+ // Normalize before validation, policy checks and persistence, so FE connector
+ // binding and Resource/Vault protocol builders consume the same values.
+ GCS_PROPERTY_ALIASES.forEach((alias, key) -> {
+ String value = properties.remove(alias);
+ // Match connector binding: nonblank gs.* values take precedence.
+ if (StringUtils.isNotBlank(value)) {
+ properties.put(key, value);
+ }
+ });
+ break;
+ default:
+ break;
+ }
+ }
+
@Override
- public void modifyProperties(Map<String, String> properties) throws DdlException {
+ public synchronized void modifyProperties(Map<String, String> newProperties) throws DdlException {
+ // Serialize the snapshot, validation and publication. A lock only around publication
+ // would allow a concurrent ALTER to replace a successful update with an older snapshot.
+ Map<String, String> properties = new HashMap<>(newProperties);
+ String provider = StringUtils.defaultIfEmpty(properties.get(StorageProperties.FS_PROVIDER_KEY),
+ this.properties.get(StorageProperties.FS_PROVIDER_KEY));
+ // Preserve AWS_* ALTER compatibility before merging with stored canonical properties.
+ S3Properties.convertToStdProperties(properties);
+ Map<String, String> effectiveProperties = new HashMap<>(this.properties);
+ for (Map.Entry<String, String> update : properties.entrySet()) {
+ // Match persistence: empty updates are ignored, except when clearing a session token.
+ replaceIfEffectiveValue(effectiveProperties, update.getKey(), update.getValue());
+ if (S3Properties.SESSION_TOKEN.equals(update.getKey()) || S3Properties.Env.TOKEN.equals(update.getKey())) {
+ effectiveProperties.put(update.getKey(), update.getValue());
+ }
+ }
+ // Resolve aliases only after merging: nonblank gs.* values, including stored aliases,
+ // take precedence over s3.* values in the final configuration.
+ normalizeProperties(effectiveProperties, provider);
+ S3Properties.convertToStdProperties(effectiveProperties);
+
+ // Normalize the update keys for modification restrictions and credential-mode changes.
+ // Their values must not overwrite the configuration resolved above.
+ normalizeProperties(properties, provider);
if (references.containsValue(ReferenceType.POLICY)) {
// can't change, because remote fs use it info to find data.
List<String> cantChangeProperties = Arrays.asList(S3Properties.ENDPOINT, S3Properties.REGION,
S3Properties.ROOT_PATH, S3Properties.BUCKET, S3Properties.Env.ENDPOINT, S3Properties.Env.REGION,
S3Properties.Env.ROOT_PATH, S3Properties.Env.BUCKET);
- Optional<String> any = cantChangeProperties.stream().filter(properties::containsKey).findAny();
+ Optional<String> any = cantChangeProperties.stream()
+ .filter(key -> properties.containsKey(key)
+ || !Objects.equals(this.properties.get(key), effectiveProperties.get(key)))
+ .findAny();
if (any.isPresent()) {
throw new DdlException("current not support modify property : " + any.get());
}
}
- // compatible with old version, Need convert if modified properties map uses old properties.
- S3Properties.convertToStdProperties(properties);
- if (!Strings.isNullOrEmpty(properties.get(S3Properties.ENDPOINT))) {
- properties.put(S3Properties.Env.ENDPOINT, properties.get(S3Properties.ENDPOINT));
+ if (!Strings.isNullOrEmpty(effectiveProperties.get(S3Properties.ENDPOINT))) {
+ effectiveProperties.put(S3Properties.Env.ENDPOINT, effectiveProperties.get(S3Properties.ENDPOINT));
}
- boolean needCheck = isNeedCheck(properties);
+ for (Map.Entry<String, String> kv : properties.entrySet()) {
+ if (kv.getKey().equalsIgnoreCase(S3Properties.ROLE_ARN)
+ && !Strings.isNullOrEmpty(kv.getValue())) {
+ effectiveProperties.remove(S3Properties.ACCESS_KEY);
+ effectiveProperties.remove(S3Properties.Env.ACCESS_KEY);
+ effectiveProperties.remove(S3Properties.SECRET_KEY);
+ effectiveProperties.remove(S3Properties.Env.SECRET_KEY);
+ }
+ if (kv.getKey().equalsIgnoreCase(S3Properties.ACCESS_KEY)
+ && !Strings.isNullOrEmpty(kv.getValue())) {
+ effectiveProperties.remove(S3Properties.ROLE_ARN);
+ effectiveProperties.remove(S3Properties.Env.ROLE_ARN);
+ effectiveProperties.remove(S3Properties.EXTERNAL_ID);
+ effectiveProperties.remove(S3Properties.Env.EXTERNAL_ID);
+ }
+ }
+ GCSProperties.validateNativeCredentialCompatibility(effectiveProperties);
+ boolean needCheck = isNeedCheck(effectiveProperties);
if (LOG.isDebugEnabled()) {
LOG.debug("s3 info need check validity : {}", needCheck);
}
if (needCheck) {
- S3Properties.requiredS3PingProperties(this.properties);
- Map<String, String> changedProperties = new HashMap<>(this.properties);
- changedProperties.putAll(properties);
+ S3Properties.requiredS3PingProperties(effectiveProperties);
+ Map<String, String> changedProperties = new HashMap<>(effectiveProperties);
String endpoint = S3Util.buildEndpointUrl(changedProperties.get(S3Properties.ENDPOINT));
changedProperties.put(S3Properties.ENDPOINT, endpoint);
changedProperties.put(S3Properties.Env.ENDPOINT, endpoint);
- String bucketName = properties.getOrDefault(S3Properties.BUCKET, this.properties.get(S3Properties.BUCKET));
- String rootPath = properties.getOrDefault(S3Properties.ROOT_PATH,
- this.properties.get(S3Properties.ROOT_PATH));
-
- pingS3(bucketName, rootPath, changedProperties);
+ pingS3(effectiveProperties.get(S3Properties.BUCKET), effectiveProperties.get(S3Properties.ROOT_PATH),
+ changedProperties);
}
- // modify properties
writeLock();
-
- for (Map.Entry<String, String> kv : properties.entrySet()) {
- replaceIfEffectiveValue(this.properties, kv.getKey(), kv.getValue());
- if (kv.getKey().equals(S3Properties.Env.TOKEN)
- || kv.getKey().equals(S3Properties.SESSION_TOKEN)) {
- this.properties.put(kv.getKey(), kv.getValue());
- }
-
- if (kv.getKey().equalsIgnoreCase(S3Properties.ROLE_ARN)
- && !Strings.isNullOrEmpty(kv.getValue())) {
- this.properties.remove(S3Properties.ACCESS_KEY);
- this.properties.remove(S3Properties.Env.ACCESS_KEY);
- this.properties.remove(S3Properties.SECRET_KEY);
- this.properties.remove(S3Properties.Env.SECRET_KEY);
- }
-
- if (kv.getKey().equalsIgnoreCase(S3Properties.ACCESS_KEY)
- && !Strings.isNullOrEmpty(kv.getValue())) {
- this.properties.remove(S3Properties.ROLE_ARN);
- this.properties.remove(S3Properties.Env.ROLE_ARN);
- this.properties.remove(S3Properties.EXTERNAL_ID);
- this.properties.remove(S3Properties.Env.EXTERNAL_ID);
- }
+ try {
+ this.properties = effectiveProperties;
+ ++version;
+ } finally {
+ writeUnlock();
}
- ++version;
- writeUnlock();
- super.modifyProperties(properties);
+ super.modifyProperties(effectiveProperties);
}
private CloudCredentialWithEndpoint getS3PingCredentials(Map<String, String> properties) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
index d312cc0e45d..77b3f4e455a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
@@ -19,6 +19,7 @@ package org.apache.doris.catalog;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.GCSProperties;
import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.nereids.trees.plans.commands.CreateResourceCommand;
@@ -73,6 +74,8 @@ public class S3StorageVault extends StorageVault {
public static final String ROLE_ARN = S3Properties.ROLE_ARN;
public static final String EXTERNAL_ID = S3Properties.EXTERNAL_ID;
public static final String CREDENTIALS_PROVIDER_TYPE = S3Properties.CREDENTIALS_PROVIDER_TYPE;
+ public static final String GCP_CREDENTIAL_PROVIDER_TYPE = GCSProperties.CREDENTIAL_PROVIDER_TYPE;
+ public static final String GCP_IMPERSONATION_SERVICE_ACCOUNT = GCSProperties.IMPERSONATION_SERVICE_ACCOUNT;
}
public static final HashSet<String> ALLOW_ALTER_PROPERTIES = new HashSet<>(Arrays.asList(
@@ -83,7 +86,9 @@ public class S3StorageVault extends StorageVault {
PropertyKey.USE_PATH_STYLE,
PropertyKey.ROLE_ARN,
PropertyKey.EXTERNAL_ID,
- PropertyKey.CREDENTIALS_PROVIDER_TYPE
+ PropertyKey.CREDENTIALS_PROVIDER_TYPE,
+ PropertyKey.GCP_CREDENTIAL_PROVIDER_TYPE,
+ PropertyKey.GCP_IMPERSONATION_SERVICE_ACCOUNT
));
@SerializedName(value = "properties")
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
index 1b3158b561a..dd2a61f71c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
@@ -25,6 +25,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock;
+import org.apache.doris.datasource.property.storage.GCSProperties;
import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.nereids.trees.plans.commands.CreateStorageVaultCommand;
@@ -146,16 +147,57 @@ public class StorageVaultMgr {
}
}
- private Cloud.StorageVaultPB.Builder buildAlterS3VaultRequest(Map<String, String> properties, String name)
+ private Cloud.StorageVaultPB.Builder buildS3VaultRequest(Map<String, String> properties, String name)
throws Exception {
- Cloud.ObjectStoreInfoPB.Builder objBuilder = S3Properties.getObjStoreInfoPB(properties);
- Cloud.StorageVaultPB.Builder alterObjVaultBuilder = Cloud.StorageVaultPB.newBuilder();
- alterObjVaultBuilder.setName(name);
- alterObjVaultBuilder.setObjInfo(objBuilder.build());
+ Cloud.StorageVaultPB.Builder builder = Cloud.StorageVaultPB.newBuilder()
+ .setName(name).setObjInfo(S3Properties.getObjStoreInfoPB(properties));
if (properties.containsKey(StorageVault.PropertyKey.VAULT_NAME)) {
- alterObjVaultBuilder.setAlterName(properties.get(StorageVault.PropertyKey.VAULT_NAME));
+ builder.setAlterName(properties.get(StorageVault.PropertyKey.VAULT_NAME));
}
- return alterObjVaultBuilder;
+ return builder;
+ }
+
+ private Cloud.StorageVaultPB.Builder buildAlterS3VaultRequest(Map<String, String> properties, String name)
+ throws Exception {
+ Cloud.StorageVaultPB.Builder builder = buildS3VaultRequest(properties, name);
+ Cloud.ObjectStoreInfoPB.Builder objBuilder = builder.getObjInfoBuilder();
+ // CREATE may infer GCP from native credentials. ALTER must not send an inferred
+ // provider because MS rejects changes to this immutable storage field.
+ if (!properties.containsKey(StorageProperties.FS_PROVIDER_KEY)) {
+ objBuilder.clearProvider();
+ }
+ // getObjStoreInfoPB supplies INSTANCE_PROFILE as the create-time default
+ // when role_arn is present. ALTER is a patch operation, so only send the
+ // credential provider when the user explicitly changed it.
+ if (!properties.containsKey(S3Properties.CREDENTIALS_PROVIDER_TYPE)
+ && !properties.containsKey(S3Properties.Env.CREDENTIALS_PROVIDER_TYPE)) {
+ objBuilder.clearCredProviderType();
+ }
+ // Empty strings are meaningful for ALTER: they explicitly clear the
+ // stored role or external ID. The generic create builder omits them.
+ if (properties.containsKey(S3Properties.ROLE_ARN)) {
+ objBuilder.setRoleArn(properties.get(S3Properties.ROLE_ARN));
+ }
+ if (properties.containsKey(S3Properties.EXTERNAL_ID)) {
+ objBuilder.setExternalId(properties.get(S3Properties.EXTERNAL_ID));
+ }
+ // GCS CREATE defaults to ADC, but a storage-only ALTER must not change authentication.
+ if (!GCSProperties.hasNativeCredentialProperties(properties)
+ && objBuilder.hasCredential() && objBuilder.getCredential().hasGcpCredential()) {
+ objBuilder.clearCredential();
+ }
+ if (objBuilder.hasCredential() && objBuilder.getCredential().hasGcpCredential()) {
+ Cloud.GcpCredentialPB.Builder credential = objBuilder.getCredentialBuilder().getGcpCredentialBuilder();
+ if (!properties.containsKey(GCSProperties.CREDENTIAL_PROVIDER_TYPE)) {
+ credential.clearCredentialProviderType();
+ }
+ // Preserve explicit empty strings so MS can distinguish clearing
+ // impersonation from leaving the stored target unchanged.
+ if (properties.containsKey(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT)) {
+ credential.setImpersonationServiceAccount(properties.get(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT));
+ }
+ }
+ return builder;
}
private Cloud.StorageVaultPB.Builder buildAlterHdfsVaultRequest(Map<String, String> properties, String name)
@@ -183,9 +225,15 @@ public class StorageVaultMgr {
return builder;
}
- private Cloud.StorageVaultPB.Builder buildAlterStorageVaultRequest(StorageVault vault) throws Exception {
- Cloud.StorageVaultPB.Builder builder = buildAlterStorageVaultRequest(vault.getType(),
- vault.getCopiedProperties(), vault.getName());
+ private Cloud.StorageVaultPB.Builder buildCreateStorageVaultRequest(StorageVault vault) throws Exception {
+ Cloud.StorageVaultPB.Builder builder;
+ if (vault.getType() == StorageVaultType.S3) {
+ builder = buildS3VaultRequest(vault.getCopiedProperties(), vault.getName());
+ } else if (vault.getType() == StorageVaultType.HDFS) {
+ builder = buildAlterHdfsVaultRequest(vault.getCopiedProperties(), vault.getName());
+ } else {
+ throw new DdlException("Unknown storage vault type");
+ }
Cloud.StorageVaultPB.PathFormat.Builder pathBuilder = Cloud.StorageVaultPB.PathFormat.newBuilder();
pathBuilder.setShardNum(vault.getNumShard());
pathBuilder.setPathVersion(vault.getPathVersion());
@@ -320,7 +368,7 @@ public class StorageVaultMgr {
@VisibleForTesting
public void createHdfsVault(StorageVault vault) throws Exception {
- Cloud.StorageVaultPB.Builder alterHdfsInfoBuilder = buildAlterStorageVaultRequest(vault);
+ Cloud.StorageVaultPB.Builder alterHdfsInfoBuilder = buildCreateStorageVaultRequest(vault);
Cloud.AlterObjStoreInfoRequest.Builder requestBuilder
= Cloud.AlterObjStoreInfoRequest.newBuilder().setRequestIp(FrontendOptions.getLocalHostAddressCached());
requestBuilder.setOp(Cloud.AlterObjStoreInfoRequest.Operation.ADD_HDFS_INFO);
@@ -370,7 +418,7 @@ public class StorageVaultMgr {
}
public void createS3Vault(StorageVault vault) throws Exception {
- Cloud.StorageVaultPB.Builder s3StorageVaultBuilder = buildAlterStorageVaultRequest(vault);
+ Cloud.StorageVaultPB.Builder s3StorageVaultBuilder = buildCreateStorageVaultRequest(vault);
Cloud.AlterObjStoreInfoRequest.Builder requestBuilder
= Cloud.AlterObjStoreInfoRequest.newBuilder().setRequestIp(FrontendOptions.getLocalHostAddressCached());
requestBuilder.setOp(Cloud.AlterObjStoreInfoRequest.Operation.ADD_S3_VAULT);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/RemoteBase.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/RemoteBase.java
index 7a19c8a22a4..7c309824b0e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/RemoteBase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/RemoteBase.java
@@ -57,17 +57,20 @@ public abstract class RemoteBase {
// only used for aws
private final String externalId;
private final String token;
+ private final boolean hasGcpCredential;
// Used to get sts token
public ObjectInfo(Cloud.ObjectStoreInfoPB.Provider provider, String ak, String sk,
String endpoint, String region, String roleName, String arn, String externalId) {
- this(provider, null, ak, sk, null, endpoint, region, null, roleName, arn, externalId, null);
+ this(provider, null, ak, sk, null, endpoint, region, null, roleName, arn, externalId, null,
+ false);
}
// Used by UT
public ObjectInfo(Cloud.ObjectStoreInfoPB.Provider provider,
String ak, String sk, String bucket, String endpoint, String region, String prefix) {
- this(provider, null, ak, sk, bucket, endpoint, region, prefix, null, null, null, null);
+ this(provider, null, ak, sk, bucket, endpoint, region, prefix, null, null, null, null,
+ false);
}
// Used by upload for internal stage
@@ -76,7 +79,7 @@ public abstract class RemoteBase {
objectStoreInfoPB.getAk(), objectStoreInfoPB.getSk(), objectStoreInfoPB.getBucket(),
objectStoreInfoPB.getEndpoint(), objectStoreInfoPB.getRegion(),
objectStoreInfoPB.getPrefix(), null, objectStoreInfoPB.getRoleArn(),
- objectStoreInfoPB.getExternalId(), null);
+ objectStoreInfoPB.getExternalId(), null, hasGcpCredential(objectStoreInfoPB));
}
private ObjectInfo(Cloud.ObjectStoreInfoPB objectStoreInfoPB, String roleName, String arn,
@@ -84,7 +87,8 @@ public abstract class RemoteBase {
this(objectStoreInfoPB.getProvider(), getCredProviderType(objectStoreInfoPB),
objectStoreInfoPB.getAk(), objectStoreInfoPB.getSk(), objectStoreInfoPB.getBucket(),
objectStoreInfoPB.getEndpoint(), objectStoreInfoPB.getRegion(),
- objectStoreInfoPB.getPrefix(), roleName, arn, externalId, token);
+ objectStoreInfoPB.getPrefix(), roleName, arn, externalId, token,
+ hasGcpCredential(objectStoreInfoPB));
}
private static Cloud.CredProviderTypePB getCredProviderType(ObjectStoreInfoPB objectStoreInfoPB) {
@@ -93,9 +97,15 @@ public abstract class RemoteBase {
: Cloud.CredProviderTypePB.INSTANCE_PROFILE;
}
+ private static boolean hasGcpCredential(ObjectStoreInfoPB objectStoreInfoPB) {
+ return objectStoreInfoPB.hasCredential()
+ && objectStoreInfoPB.getCredential().hasGcpCredential();
+ }
+
private ObjectInfo(Cloud.ObjectStoreInfoPB.Provider provider, Cloud.CredProviderTypePB credProviderType,
String ak, String sk, String bucket, String endpoint, String region, String prefix,
- String roleName, String arn, String externalId, String token) {
+ String roleName, String arn, String externalId, String token,
+ boolean hasGcpCredential) {
this.provider = provider;
this.credProviderType = credProviderType;
this.ak = ak;
@@ -108,6 +118,11 @@ public abstract class RemoteBase {
this.arn = arn;
this.externalId = externalId;
this.token = token;
+ this.hasGcpCredential = hasGcpCredential;
+ }
+
+ public boolean supportsPresignedUrl() {
+ return !hasGcpCredential;
}
@Override
@@ -125,6 +140,7 @@ public abstract class RemoteBase {
+ ", arn='" + arn + '\''
+ ", externalId='" + externalId + '\''
+ ", token='" + token + '\''
+ + ", hasGcpCredential=" + hasGcpCredential
+ '}';
}
}
@@ -245,7 +261,7 @@ public abstract class RemoteBase {
ObjectInfo objInfo = new ObjectInfo(infoPB.getProvider(), null, stsToken.getLeft(),
stsToken.getMiddle(), infoPB.getBucket(), infoPB.getEndpoint(), infoPB.getRegion(),
infoPB.getPrefix(), stagePB.getRoleName(), stagePB.getArn(), encodedExternalId,
- stsToken.getRight());
+ stsToken.getRight(), ObjectInfo.hasGcpCredential(infoPB));
LOG.info("Parse object storage info, before={}, after={}", new ObjectInfo(infoPB), objInfo);
return objInfo;
} catch (Throwable e) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/S3Remote.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/S3Remote.java
index c283e7900aa..793b3fcb2a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/S3Remote.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/S3Remote.java
@@ -45,6 +45,10 @@ public class S3Remote extends DefaultRemote {
@Override
public String getPresignedUrl(String fileName) {
+ if (!obj.supportsPresignedUrl()) {
+ throw new UnsupportedOperationException("Presigned URLs are not supported for native GCP OAuth "
+ + "credentials; GCS V4 signing with IAM signBlob is required");
+ }
try {
PutObjectRequest objectRequest = PutObjectRequest.builder()
.bucket(obj.getBucket())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
index be75e961fde..a89912294d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
@@ -26,6 +26,7 @@ import org.apache.doris.common.credentials.CloudCredential;
import com.google.common.base.Strings;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@@ -42,10 +43,13 @@ import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsPr
import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
import software.amazon.awssdk.core.retry.RetryPolicy;
import software.amazon.awssdk.core.retry.backoff.EqualJitterBackoffStrategy;
+import software.amazon.awssdk.core.signer.NoOpSigner;
import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.sts.StsClient;
@@ -255,6 +259,53 @@ public class S3Util {
.build();
}
+ public static S3Client buildBearerTokenS3Client(String endpoint, String region, boolean isUsePathStyle,
+ ExecutionInterceptor bearerTokenInterceptor) {
+ ClientOverrideConfiguration clientConf = buildBearerTokenClientConfiguration(bearerTokenInterceptor);
+ return S3Client.builder()
+ .httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30))
+ .connectionTimeout(Duration.ofSeconds(30)).build())
+ .endpointOverride(URI.create(buildEndpointUrl(endpoint)))
+ .credentialsProvider(AnonymousCredentialsProvider.create())
+ .region(Region.of(region))
+ .overrideConfiguration(clientConf)
+ .serviceConfiguration(bearerTokenS3ServiceConfiguration(isUsePathStyle))
+ .build();
+ }
+
+ public static S3AsyncClient buildBearerTokenS3AsyncClient(String endpoint, String region,
+ boolean isUsePathStyle,
+ ExecutionInterceptor bearerTokenInterceptor) {
+ return S3AsyncClient.builder()
+ .endpointOverride(URI.create(buildEndpointUrl(endpoint)))
+ .credentialsProvider(AnonymousCredentialsProvider.create())
+ .region(Region.of(region))
+ .overrideConfiguration(buildBearerTokenClientConfiguration(bearerTokenInterceptor))
+ .serviceConfiguration(bearerTokenS3ServiceConfiguration(isUsePathStyle))
+ .build();
+ }
+
+ private static ClientOverrideConfiguration buildBearerTokenClientConfiguration(
+ ExecutionInterceptor bearerTokenInterceptor) {
+ RetryPolicy retryPolicy = RetryPolicy.builder()
+ .numRetries(3)
+ .backoffStrategy(EqualJitterBackoffStrategy.builder()
+ .baseDelay(Duration.ofSeconds(1)).maxBackoffTime(Duration.ofMinutes(1)).build())
+ .build();
+ return ClientOverrideConfiguration.builder()
+ .retryPolicy(retryPolicy)
+ .putAdvancedOption(SdkAdvancedClientOption.SIGNER, new NoOpSigner())
+ .addExecutionInterceptor(bearerTokenInterceptor)
+ .build();
+ }
+
+ private static S3Configuration bearerTokenS3ServiceConfiguration(boolean isUsePathStyle) {
+ return S3Configuration.builder()
+ .chunkedEncodingEnabled(false)
+ .pathStyleAccessEnabled(isUsePathStyle)
+ .build();
+ }
+
public static S3Client buildS3Client(String endpoint, String region, boolean isUsePathStyle, String accessKey,
String secretKey, String sessionToken, String roleArn, String externalId) {
EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/fileio/GcpS3FileIOAwsClientFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/fileio/GcpS3FileIOAwsClientFactory.java
new file mode 100644
index 00000000000..974dadb8020
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/fileio/GcpS3FileIOAwsClientFactory.java
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg.fileio;
+
+import org.apache.doris.datasource.property.storage.GCSProperties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.apache.iceberg.aws.s3.S3FileIOAwsClientFactory;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/** Supplies Iceberg S3FileIO clients configured with native GCP bearer authentication. */
+public class GcpS3FileIOAwsClientFactory implements S3FileIOAwsClientFactory {
+ private static final long serialVersionUID = 1L;
+
+ private Map<String, String> properties;
+
+ public GcpS3FileIOAwsClientFactory() {
+ }
+
+ @Override
+ public void initialize(Map<String, String> properties) {
+ this.properties = new HashMap<>(properties);
+ }
+
+ @Override
+ public S3Client s3() {
+ return gcsProperties().createS3Client();
+ }
+
+ @Override
+ public S3AsyncClient s3Async() {
+ return gcsProperties().createS3AsyncClient();
+ }
+
+ private GCSProperties gcsProperties() {
+ if (properties == null) {
+ throw new IllegalStateException("GCP Iceberg client factory has not been initialized");
+ }
+ StorageProperties storageProperties = StorageProperties.createPrimary(properties);
+ if (!(storageProperties instanceof GCSProperties)) {
+ throw new IllegalArgumentException("Native GCP Iceberg authentication requires GCS properties");
+ }
+ return (GCSProperties) storageProperties;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
index 2a267161060..662b7f3fff7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
@@ -21,12 +21,15 @@ import org.apache.doris.common.security.authentication.ExecutionAuthenticator;
import org.apache.doris.datasource.SessionContext;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
import org.apache.doris.datasource.iceberg.IcebergUtils;
+import org.apache.doris.datasource.iceberg.fileio.GcpS3FileIOAwsClientFactory;
import org.apache.doris.datasource.metacache.CacheSpec;
import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager;
import org.apache.doris.datasource.property.common.IcebergAwsAssumeRoleProperties;
import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties;
+import org.apache.doris.datasource.property.storage.GCSProperties;
import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.doris.datasource.property.storage.auth.GcpCredential;
import org.apache.doris.foundation.property.ConnectorProperty;
import com.google.common.base.Preconditions;
@@ -37,8 +40,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.s3.S3FileIO;
import org.apache.iceberg.aws.s3.S3FileIOProperties;
import org.apache.iceberg.catalog.Catalog;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import java.util.HashMap;
import java.util.List;
@@ -244,6 +249,7 @@ public abstract class AbstractIcebergProperties extends MetastoreProperties {
}
if (s3Target != null) {
toS3FileIOProperties(s3Target, fileIOProperties);
+ configureProviderSpecificFileIO(s3Target, fileIOProperties);
} else {
String region = AbstractS3CompatibleProperties.getRegionFromProperties(fileIOProperties);
if (!Strings.isNullOrEmpty(region)) {
@@ -285,6 +291,41 @@ public abstract class AbstractIcebergProperties extends MetastoreProperties {
}
}
+ /**
+ * Selects a provider-aware FileIO when the generic Iceberg S3FileIO cannot represent the
+ * provider's native authentication model.
+ *
+ * <p>GCP OAuth is implemented by Doris' object-storage client. Iceberg S3FileIO supplies the
+ * complete streaming implementation (including seek and multipart writes), while the custom
+ * client factory injects the same refreshable bearer authentication used by FE object storage.
+ * When OSS or Azure passwordless credentials are added to {@code ObjCredential}, add their
+ * provider-specific client factory in this method.</p>
+ */
+ private void configureProviderSpecificFileIO(AbstractS3CompatibleProperties storageProperties,
+ Map<String, String> options) {
+ if (storageProperties instanceof GCSProperties && ((GCSProperties) storageProperties).isAnonymous()) {
+ // S3FileIO builds its own clients and does not consume the Hadoop credential provider.
+ options.put(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER,
+ AnonymousCredentialsProvider.class.getName());
+ }
+ if (storageProperties instanceof GCSProperties
+ && ((GCSProperties) storageProperties).hasGcpCredential()) {
+ // The client factory reconstructs GCSProperties from this map. Forward
+ // only the provider-native credential fields; copying the entire catalog
+ // property map can expose unrelated secrets to FileIO implementations.
+ GcpCredential credential = ((GCSProperties) storageProperties).getGcpCredential();
+ options.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE,
+ credential.getCredentialProviderType().name());
+ if (StringUtils.isNotBlank(credential.getImpersonationServiceAccount())) {
+ options.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT,
+ credential.getImpersonationServiceAccount());
+ }
+ options.put(StorageProperties.FS_PROVIDER_KEY, "GCP");
+ options.put(CatalogProperties.FILE_IO_IMPL, S3FileIO.class.getName());
+ options.put(S3FileIOProperties.CLIENT_FACTORY, GcpS3FileIOAwsClientFactory.class.getName());
+ }
+ }
+
protected Catalog buildIcebergCatalog(String catalogName, Map<String, String> options, Configuration conf) {
// For Iceberg SDK, "type" means catalog type, such as hive, jdbc, rest.
// But in Doris, "type" is "iceberg".
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
index 53a2fcbc98b..af299e43feb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
@@ -18,6 +18,7 @@
package org.apache.doris.datasource.property.storage;
import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.S3Util;
import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode;
import org.apache.doris.foundation.property.ConnectorPropertiesUtils;
import org.apache.doris.foundation.property.ConnectorProperty;
@@ -32,6 +33,7 @@ import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.services.s3.S3Client;
import java.lang.reflect.Field;
import java.util.Arrays;
@@ -145,6 +147,17 @@ public abstract class AbstractS3CompatibleProperties extends StorageProperties i
return null;
}
+ /**
+ * Creates the S3-compatible client used by {@code S3ObjStorage}.
+ *
+ * <p>Providers with native passwordless authentication override this method
+ * and keep their authentication details in the provider-specific properties
+ * class. This prevents {@code S3ObjStorage} from accumulating provider checks.
+ */
+ public S3Client createS3Client(String endpoint, boolean usePathStyle) {
+ return S3Util.buildS3Client(endpoint, getRegion(), usePathStyle, getAwsCredentialsProvider());
+ }
+
@Override
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java
index a414b6c2d95..d84648cb68d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java
@@ -17,14 +17,24 @@
package org.apache.doris.datasource.property.storage;
+import org.apache.doris.common.util.S3Util;
+import org.apache.doris.datasource.property.storage.auth.GcpAuth;
+import org.apache.doris.datasource.property.storage.auth.GcpCredential;
+import org.apache.doris.datasource.property.storage.auth.GcpCredentialProviderType;
+import org.apache.doris.datasource.property.storage.auth.GcsAuth;
+import org.apache.doris.datasource.property.storage.auth.GcsAuthResolver;
import org.apache.doris.foundation.property.ConnectorProperty;
+import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem;
import com.google.common.collect.ImmutableSet;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
import java.util.HashSet;
import java.util.Map;
@@ -52,21 +62,24 @@ import java.util.regex.Pattern;
* the user must explicitly declare that this is GCS storage and configure the mapping.</li>
* </ul>
* </li>
- * <li>Additional authentication methods (e.g., OAuth2, Service Account) may be supported in the future.</li>
+ * <li>Without HMAC keys, authentication defaults to ADC, including the VM's attached service account.
+ * Set {@code gs.credential_provider_type=ANONYMOUS} explicitly for public objects.</li>
+ * <li>Native OAuth2 credentials are supported through ADC, Compute Engine metadata credentials,
+ * and Compute Engine credentials with service account impersonation.</li>
* </ul>
* </p>
*/
public class GCSProperties extends AbstractS3CompatibleProperties {
- private static final Set<String> GS_ENDPOINT_ALIAS = ImmutableSet.of(
- "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT");
-
- private static final String GCS_ENDPOINT_KEY_NAME = "gs.endpoint";
+ public static final String ENDPOINT = "gs.endpoint";
+ public static final String CREDENTIAL_PROVIDER_TYPE = GcpCredential.CREDENTIAL_PROVIDER_TYPE;
+ public static final String IMPERSONATION_SERVICE_ACCOUNT = GcpCredential.IMPERSONATION_SERVICE_ACCOUNT;
+ private GcsAuth auth;
@Setter
@Getter
- @ConnectorProperty(names = {"gs.endpoint", "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT"},
+ @ConnectorProperty(names = {ENDPOINT, "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT"},
required = false,
description = "The endpoint of GCS.")
protected String endpoint = "https://storage.googleapis.com";
@@ -89,8 +102,9 @@ public class GCSProperties extends AbstractS3CompatibleProperties {
protected String secretKey = "";
@Getter
- @ConnectorProperty(names = {"gs.session_token", "s3.session_token", "session_token"},
+ @ConnectorProperty(names = {"gs.session_token", "s3.session_token", "session_token", S3Properties.Env.TOKEN},
required = false,
+ sensitive = true,
description = "The session token of GCS.")
protected String sessionToken = "";
@@ -135,6 +149,18 @@ public class GCSProperties extends AbstractS3CompatibleProperties {
@Getter
protected String forceParsingByStandardUrl = "false";
+ /** Resolved GCP credential provider; empty when authentication uses HMAC keys. */
+ @Getter
+ @ConnectorProperty(names = {CREDENTIAL_PROVIDER_TYPE}, required = false,
+ description = "GCP credential source: DEFAULT (ADC), COMPUTE_ENGINE or ANONYMOUS. "
+ + "Defaults to ADC without HMAC keys.")
+ protected String credentialProviderType = "";
+
+ @Getter
+ @ConnectorProperty(names = {IMPERSONATION_SERVICE_ACCOUNT}, required = false,
+ description = "Target service account email used for GCP service account impersonation.")
+ protected String impersonationServiceAccount = "";
+
/**
* Constructor to initialize the object storage properties with the provided type and original properties map.
*
@@ -145,20 +171,7 @@ public class GCSProperties extends AbstractS3CompatibleProperties {
}
public static boolean guessIsMe(Map<String, String> props) {
- // check has gcs specific keys,ignore case
- if (props.containsKey(GCS_ENDPOINT_KEY_NAME) && StringUtils.isNotBlank(props.get(GCS_ENDPOINT_KEY_NAME))) {
- return true;
- }
- String endpoint;
- for (String key : props.keySet()) {
- if (GS_ENDPOINT_ALIAS.contains(key.toLowerCase())) {
- endpoint = props.get(key);
- if (StringUtils.isNotBlank(endpoint) && endpoint.toLowerCase().endsWith("storage.googleapis.com")) {
- return true;
- }
- }
- }
- return false;
+ return GcsAuthResolver.guessIsGcs(props);
}
@Override
@@ -174,31 +187,138 @@ public class GCSProperties extends AbstractS3CompatibleProperties {
@Override
public void initializeHadoopStorageConfig() {
- super.initializeHadoopStorageConfig();
- hadoopStorageConfig.set("fs.gs.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
+ if (hasGcpCredential()) {
+ // HadoopCatalog namespace operations bypass Iceberg FileIO. Native GCP credentials
+ // must use the GCS connector here, rather than the S3A filesystem's AWS credential chain.
+ GcpCredential credential = getGcpCredential();
+ hadoopStorageConfig = new Configuration();
+ hadoopStorageConfig.set("fs.gs.impl", GoogleHadoopFileSystem.class.getName());
+ hadoopStorageConfig.set("fs.gs.auth.type",
+ credential.getCredentialProviderType() == GcpCredentialProviderType.DEFAULT
+ ? "APPLICATION_DEFAULT" : "COMPUTE_ENGINE");
+ hadoopStorageConfig.set("fs.gs.auth.impersonation.service.account",
+ credential.getImpersonationServiceAccount());
+ } else {
+ super.initializeHadoopStorageConfig();
+ hadoopStorageConfig.set("fs.gs.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
+ if (auth.isAnonymous()) {
+ hadoopStorageConfig.set("fs.s3a.aws.credentials.provider",
+ "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider");
+ }
+ }
}
protected Map<String, String> doGetBackendConfigProperties() {
Map<String, String> backendProperties = generateBackendS3Configuration();
backendProperties.put("provider", "GCP");
+ if (hasGcpCredential()) {
+ GcpCredential credential = getGcpCredential();
+ backendProperties.put(CREDENTIAL_PROVIDER_TYPE, credential.getCredentialProviderType().name());
+ if (StringUtils.isNotBlank(credential.getImpersonationServiceAccount())) {
+ backendProperties.put(IMPERSONATION_SERVICE_ACCOUNT,
+ credential.getImpersonationServiceAccount());
+ }
+ }
return backendProperties;
}
+ @Override
+ public void initNormalizeAndCheckProps() {
+ // Resolve raw aliases before binding, then share the immutable choice across all consumers.
+ auth = GcsAuthResolver.resolve(origProps)
+ .orElseThrow(() -> new IllegalArgumentException("GCS storage properties are required."));
+ super.initNormalizeAndCheckProps();
+ if (hasGcpCredential()) {
+ GcpCredential credential = getGcpCredential();
+ credentialProviderType = credential.getCredentialProviderType().name();
+ impersonationServiceAccount = credential.getImpersonationServiceAccount();
+ } else if (auth.isAnonymous()) {
+ credentialProviderType = GcpCredentialProviderType.ANONYMOUS.name();
+ impersonationServiceAccount = "";
+ } else {
+ credentialProviderType = "";
+ impersonationServiceAccount = "";
+ }
+ if (auth.getMode() != GcsAuth.Mode.HMAC) {
+ // Cache identity follows resolved authentication, including implicit ADC and aliases.
+ matchedProperties.put(CREDENTIAL_PROVIDER_TYPE, credentialProviderType);
+ matchedProperties.put(IMPERSONATION_SERVICE_ACCOUNT, impersonationServiceAccount);
+ }
+ }
+
+ public static boolean hasNativeCredentialProperties(Map<String, String> properties) {
+ return GcsAuthResolver.hasNativeCredentialProperties(properties);
+ }
+
+ public static void validateNativeCredentialCompatibility(Map<String, String> properties) {
+ GcsAuthResolver.resolve(properties);
+ }
+
+ public boolean hasGcpCredential() {
+ return auth.getNativeCredential().isPresent();
+ }
+
+ public boolean isAnonymous() {
+ return auth.isAnonymous();
+ }
+
+ public GcpCredentialProviderType getGcpCredentialProviderType() {
+ return getGcpCredential().getCredentialProviderType();
+ }
+
+ public GcpCredential getGcpCredential() {
+ return auth.getNativeCredential()
+ .orElseThrow(() -> new IllegalArgumentException(CREDENTIAL_PROVIDER_TYPE
+ + " does not select native GCP authentication."));
+ }
+
+ public S3Client createS3Client() {
+ return createS3Client(endpoint, Boolean.parseBoolean(usePathStyle));
+ }
+
+ @Override
+ public S3Client createS3Client(String endpoint, boolean usePathStyle) {
+ if (!hasGcpCredential()) {
+ return super.createS3Client(endpoint, usePathStyle);
+ }
+ try {
+ return S3Util.buildBearerTokenS3Client(endpoint, getRegion(), usePathStyle,
+ GcpAuth.bearerTokenInterceptor(GcpAuth.createCredentials(getGcpCredential())));
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to initialize native GCP authentication", e);
+ }
+ }
+
+ public S3AsyncClient createS3AsyncClient() {
+ if (!hasGcpCredential()) {
+ throw new IllegalStateException("Native GCP authentication is required for the Iceberg client");
+ }
+ try {
+ return S3Util.buildBearerTokenS3AsyncClient(endpoint, region, Boolean.parseBoolean(usePathStyle),
+ GcpAuth.bearerTokenInterceptor(GcpAuth.createCredentials(getGcpCredential())));
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to initialize native GCP authentication", e);
+ }
+ }
+
@Override
protected Set<String> schemas() {
return ImmutableSet.of("gs");
}
+ @Override
+ protected String getAwsCredentialsProviderTypeForBackend() {
+ return auth.isAnonymous() ? GcpCredentialProviderType.ANONYMOUS.name() : null;
+ }
+
@Override
public AwsCredentialsProvider getAwsCredentialsProvider() {
- AwsCredentialsProvider credentialsProvider = super.getAwsCredentialsProvider();
- if (credentialsProvider != null) {
- return credentialsProvider;
- }
- if (StringUtils.isBlank(accessKey) && StringUtils.isBlank(secretKey)) {
- // For anonymous access (no credentials required)
+ if (auth.isAnonymous()) {
return AnonymousCredentialsProvider.create();
}
- return null;
+ if (auth.getMode() == GcsAuth.Mode.HMAC) {
+ return super.getAwsCredentialsProvider();
+ }
+ throw new IllegalStateException("Native GCP authentication requires the bearer-token client");
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index c63e3acce55..88f6def0a9b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -24,9 +24,13 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.property.common.AwsCredentialsProviderFactory;
import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode;
+import org.apache.doris.datasource.property.storage.auth.GcsAuth;
+import org.apache.doris.datasource.property.storage.auth.GcsAuthResolver;
+import org.apache.doris.datasource.property.storage.auth.ObjCredentialFactory;
import org.apache.doris.foundation.property.ConnectorPropertiesUtils;
import org.apache.doris.foundation.property.ConnectorProperty;
import org.apache.doris.thrift.TCredProviderType;
+import org.apache.doris.thrift.TObjStorageType;
import org.apache.doris.thrift.TS3StorageParam;
import com.google.common.base.Preconditions;
@@ -46,6 +50,7 @@ import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider
import java.util.Arrays;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
@@ -230,6 +235,10 @@ public class S3Properties extends AbstractS3CompatibleProperties {
@Override
public void initNormalizeAndCheckProps() {
+ if (GCSProperties.hasNativeCredentialProperties(origProps)) {
+ GCSProperties.validateNativeCredentialCompatibility(origProps);
+ throw new IllegalArgumentException("Native GCP authentication cannot be used with S3 properties.");
+ }
super.initNormalizeAndCheckProps();
if (StringUtils.isNotBlank(s3ExternalId) && StringUtils.isBlank(s3IAMRole)) {
throw new IllegalArgumentException("s3.external_id must be used with s3.role_arn");
@@ -245,6 +254,11 @@ public class S3Properties extends AbstractS3CompatibleProperties {
* @return
*/
protected static boolean guessIsMe(Map<String, String> origProps) {
+ // GCS is transported through the S3-compatible protocol. Check all GCS-specific signals
+ // before generic fields such as region so native OAuth is never initialized as AWS S3.
+ if (GCSProperties.guessIsMe(origProps)) {
+ return false;
+ }
String endpoint = Stream.of(ENDPOINT_NAMES_FOR_GUESSING)
.map(origProps::get)
.filter(Objects::nonNull)
@@ -519,6 +533,7 @@ public class S3Properties extends AbstractS3CompatibleProperties {
}
public static void convertToStdProperties(Map<String, String> properties) {
+ GCSProperties.validateNativeCredentialCompatibility(properties);
if (properties.containsKey(Env.ENDPOINT)) {
properties.putIfAbsent(ENDPOINT, properties.get(Env.ENDPOINT));
}
@@ -662,7 +677,20 @@ public class S3Properties extends AbstractS3CompatibleProperties {
|| properties.containsKey(Env.CREDENTIALS_PROVIDER_TYPE);
}
+ private static boolean hasAnonymousCredentialsProvider(Map<String, String> properties) {
+ return properties.entrySet().stream().anyMatch(entry ->
+ (CREDENTIALS_PROVIDER_TYPE.equalsIgnoreCase(entry.getKey())
+ || Env.CREDENTIALS_PROVIDER_TYPE.equalsIgnoreCase(entry.getKey()))
+ && "ANONYMOUS".equalsIgnoreCase(StringUtils.trim(entry.getValue())));
+ }
+
public static Cloud.ObjectStoreInfoPB.Builder getObjStoreInfoPB(Map<String, String> properties) {
+ Optional<GcsAuth> gcsAuth = GcsAuthResolver.resolve(properties);
+ if (gcsAuth.filter(GcsAuth::isAnonymous).isPresent()) {
+ // Keep this restriction in the Vault protobuf path: FE clients and BE Thrift
+ // still support anonymous public-object access, including legacy S3 aliases.
+ throw new IllegalArgumentException("Anonymous GCS authentication is not supported for storage vaults.");
+ }
Cloud.ObjectStoreInfoPB.Builder builder = Cloud.ObjectStoreInfoPB.newBuilder();
if (properties.containsKey(S3Properties.ENDPOINT)) {
builder.setEndpoint(properties.get(S3Properties.ENDPOINT));
@@ -705,6 +733,9 @@ public class S3Properties extends AbstractS3CompatibleProperties {
builder.setCredProviderType(getCredProviderTypePB(properties));
}
+ ObjCredentialFactory.fromProperties(properties, gcsAuth)
+ .ifPresent(credential -> credential.applyTo(builder));
+
if (properties.containsKey(S3Properties.ROLE_ARN)) {
String roleArn = properties.get(S3Properties.ROLE_ARN);
if (!Strings.isNullOrEmpty(roleArn)) {
@@ -722,8 +753,14 @@ public class S3Properties extends AbstractS3CompatibleProperties {
}
public static TS3StorageParam getS3TStorageParam(Map<String, String> properties) {
+ Optional<GcsAuth> gcsAuth = GcsAuthResolver.resolve(properties);
TS3StorageParam s3Info = new TS3StorageParam();
+ if (properties.containsKey(StorageProperties.FS_PROVIDER_KEY)
+ && StringUtils.isNotBlank(properties.get(StorageProperties.FS_PROVIDER_KEY))) {
+ s3Info.setProvider(toThriftProvider(properties.get(StorageProperties.FS_PROVIDER_KEY)));
+ }
+
if (properties.containsKey(S3Properties.ROLE_ARN)) {
s3Info.setRoleArn(properties.get(S3Properties.ROLE_ARN));
if (properties.containsKey(S3Properties.EXTERNAL_ID)) {
@@ -751,7 +788,19 @@ public class S3Properties extends AbstractS3CompatibleProperties {
? S3Properties.Env.DEFAULT_CONNECTION_TIMEOUT_MS : connTimeoutMs));
String usePathStyle = properties.getOrDefault(S3Properties.USE_PATH_STYLE, "false");
s3Info.setUsePathStyle(Boolean.parseBoolean(usePathStyle));
+ ObjCredentialFactory.fromProperties(properties, gcsAuth)
+ .ifPresent(credential -> credential.applyTo(s3Info));
+ if (gcsAuth.filter(GcsAuth::isAnonymous).isPresent()
+ || (!gcsAuth.isPresent() && hasAnonymousCredentialsProvider(properties))) {
+ s3Info.setCredProviderType(TCredProviderType.ANONYMOUS);
+ }
return s3Info;
}
+ private static TObjStorageType toThriftProvider(String provider) {
+ String normalized = provider.trim().toUpperCase(Locale.ROOT);
+ // The public/protobuf provider is named S3, while the legacy Thrift enum uses AWS.
+ return "S3".equals(normalized) ? TObjStorageType.AWS : TObjStorageType.valueOf(normalized);
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcpAuth.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcpAuth.java
new file mode 100644
index 00000000000..58eff282300
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcpAuth.java
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage.auth;
+
+import com.google.auth.oauth2.ComputeEngineCredentials;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.auth.oauth2.ImpersonatedCredentials;
+import software.amazon.awssdk.core.interceptor.Context;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.http.SdkHttpRequest;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/** Creates GCP OAuth2 credentials and adapts them to the S3-compatible client. */
+public final class GcpAuth {
+ public static final String DEVSTORAGE_SCOPE =
+ "https://www.googleapis.com/auth/devstorage.read_write";
+ private static final String CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform";
+ private static final int IMPERSONATION_LIFETIME_SECONDS = 3600;
+
+ private GcpAuth() {
+ }
+
+ public static GoogleCredentials createCredentials(GcpCredential credential) throws IOException {
+ GoogleCredentials source;
+ switch (credential.getCredentialProviderType()) {
+ case DEFAULT:
+ source = GoogleCredentials.getApplicationDefault();
+ break;
+ case COMPUTE_ENGINE:
+ source = ComputeEngineCredentials.create();
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported GCP credential provider: "
+ + credential.getCredentialProviderType());
+ }
+ if (credential.getImpersonationServiceAccount().isEmpty()) {
+ return source.createScoped(Collections.singletonList(DEVSTORAGE_SCOPE));
+ }
+ // The source token calls the IAM Credentials API; the impersonated
+ // target token itself only needs object read/write access.
+ source = source.createScoped(Collections.singletonList(CLOUD_PLATFORM_SCOPE));
+ return ImpersonatedCredentials.create(source, credential.getImpersonationServiceAccount(), null,
+ Collections.singletonList(DEVSTORAGE_SCOPE), IMPERSONATION_LIFETIME_SECONDS);
+ }
+
+ public static ExecutionInterceptor bearerTokenInterceptor(GoogleCredentials credentials) {
+ return new ExecutionInterceptor() {
+ @Override
+ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ Map<String, List<String>> metadata = credentials.getRequestMetadata();
+ List<String> authorization = metadata.get("Authorization");
+ if (authorization == null || authorization.isEmpty()
+ || authorization.get(0) == null || authorization.get(0).isEmpty()) {
+ throw new IOException("GCP credentials did not provide an Authorization header");
+ }
+ return context.httpRequest().toBuilder()
+ .putHeader("Authorization", authorization.get(0))
+ .build();
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to obtain a GCP access token", e);
+ }
+ }
+ };
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcpCredential.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcpCredential.java
new file mode 100644
index 00000000000..d8661bd99cc
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcpCredential.java
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage.auth;
+
+import org.apache.doris.cloud.proto.Cloud.GcpCredentialPB;
+import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB;
+import org.apache.doris.thrift.TCredential;
+import org.apache.doris.thrift.TGcpCredential;
+import org.apache.doris.thrift.TGcpCredentialProviderType;
+import org.apache.doris.thrift.TObjStorageType;
+import org.apache.doris.thrift.TS3StorageParam;
+
+import org.apache.commons.lang3.StringUtils;
+
+/** GCP native credential shared by the Cloud protobuf and BE Thrift paths. */
+public final class GcpCredential implements ObjCredential {
+ public static final String CREDENTIAL_PROVIDER_TYPE = "gs.credential_provider_type";
+ public static final String IMPERSONATION_SERVICE_ACCOUNT = "gs.impersonation_service_account";
+
+ private static final String SERVICE_ACCOUNT_PATTERN =
+ "^[^@\\s]+@(?:[^@\\s]+\\.iam\\.gserviceaccount\\.com|"
+ + "developer\\.gserviceaccount\\.com|appspot\\.gserviceaccount\\.com)$";
+
+ private final GcpCredentialProviderType providerType;
+ private final String impersonationServiceAccount;
+
+ public GcpCredential(GcpCredentialProviderType providerType, String impersonationServiceAccount) {
+ this.providerType = providerType;
+ this.impersonationServiceAccount = impersonationServiceAccount == null ? "" : impersonationServiceAccount;
+ validate();
+ }
+
+ public GcpCredentialProviderType getCredentialProviderType() {
+ return providerType;
+ }
+
+ public String getImpersonationServiceAccount() {
+ return impersonationServiceAccount;
+ }
+
+ private void validate() {
+ if (providerType == GcpCredentialProviderType.ANONYMOUS) {
+ throw new IllegalArgumentException("Anonymous access does not use a native GCP credential");
+ }
+ if (StringUtils.isEmpty(impersonationServiceAccount)) {
+ return;
+ }
+ if (!impersonationServiceAccount.matches(SERVICE_ACCOUNT_PATTERN)) {
+ throw new IllegalArgumentException("Invalid GCP service account email: "
+ + impersonationServiceAccount);
+ }
+ }
+
+ @Override
+ public void applyTo(ObjectStoreInfoPB.Builder builder) {
+ if (!builder.hasProvider()) {
+ builder.setProvider(ObjectStoreInfoPB.Provider.GCP);
+ }
+ GcpCredentialPB.Builder credential = GcpCredentialPB.newBuilder()
+ .setCredentialProviderType(providerType == GcpCredentialProviderType.DEFAULT
+ ? GcpCredentialPB.CredentialProviderType.DEFAULT
+ : GcpCredentialPB.CredentialProviderType.COMPUTE_ENGINE);
+ if (StringUtils.isNotBlank(impersonationServiceAccount)) {
+ credential.setImpersonationServiceAccount(impersonationServiceAccount);
+ }
+ builder.getCredentialBuilder().setGcpCredential(credential);
+ }
+
+ @Override
+ public void applyTo(TS3StorageParam param) {
+ if (!param.isSetProvider()) {
+ param.setProvider(TObjStorageType.GCP);
+ }
+ TGcpCredential credential = new TGcpCredential()
+ .setCredentialProviderType(providerType == GcpCredentialProviderType.DEFAULT
+ ? TGcpCredentialProviderType.DEFAULT
+ : TGcpCredentialProviderType.COMPUTE_ENGINE);
+ if (StringUtils.isNotBlank(impersonationServiceAccount)) {
+ credential.setImpersonationServiceAccount(impersonationServiceAccount);
+ }
+ param.setCredential(new TCredential().setGcpCredential(credential));
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcpCredentialProviderType.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcpCredentialProviderType.java
new file mode 100644
index 00000000000..b40fd342ae1
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcpCredentialProviderType.java
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage.auth;
+
+public enum GcpCredentialProviderType {
+ DEFAULT,
+ COMPUTE_ENGINE,
+ ANONYMOUS;
+
+ public static GcpCredentialProviderType fromString(String propertyName, String value) {
+ try {
+ return valueOf(value.trim().toUpperCase());
+ } catch (RuntimeException e) {
+ throw new IllegalArgumentException("Invalid " + propertyName + " value: " + value
+ + ". Only DEFAULT, COMPUTE_ENGINE and ANONYMOUS are supported.", e);
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcsAuth.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcsAuth.java
new file mode 100644
index 00000000000..a3a0923c3bd
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcsAuth.java
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage.auth;
+
+import java.util.Optional;
+
+/** Immutable authentication choice shared by GCS clients, cache identity and protocol builders. */
+public final class GcsAuth {
+ public enum Mode {
+ HMAC, ADC, COMPUTE_ENGINE, ANONYMOUS
+ }
+
+ private final Mode mode;
+ private final GcpCredential nativeCredential;
+
+ GcsAuth(Mode mode, String impersonationServiceAccount) {
+ this.mode = mode;
+ this.nativeCredential = mode == Mode.ADC || mode == Mode.COMPUTE_ENGINE
+ ? new GcpCredential(mode == Mode.ADC ? GcpCredentialProviderType.DEFAULT
+ : GcpCredentialProviderType.COMPUTE_ENGINE, impersonationServiceAccount)
+ : null;
+ }
+
+ public Mode getMode() {
+ return mode;
+ }
+
+ public boolean isAnonymous() {
+ return mode == Mode.ANONYMOUS;
+ }
+
+ public Optional<GcpCredential> getNativeCredential() {
+ return Optional.ofNullable(nativeCredential);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcsAuthResolver.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcsAuthResolver.java
new file mode 100644
index 00000000000..2dac711981a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/GcsAuthResolver.java
@@ -0,0 +1,161 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage.auth;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Resolves GCS authentication without creating clients or loading credentials.
+ * Vault ALTER still preserves field presence when constructing its patch from this result.
+ */
+public final class GcsAuthResolver {
+ private static final String CREDENTIAL_PROVIDER_TYPE = GcpCredential.CREDENTIAL_PROVIDER_TYPE;
+ private static final String IMPERSONATION_SERVICE_ACCOUNT = GcpCredential.IMPERSONATION_SERVICE_ACCOUNT;
+ private static final String[] ACCESS_KEYS = {"gs.access_key", "s3.access_key", "AWS_ACCESS_KEY", "access_key"};
+ private static final String[] SECRET_KEYS = {"gs.secret_key", "s3.secret_key", "AWS_SECRET_KEY", "secret_key"};
+ private static final String[] TOKENS = {"gs.session_token", "s3.session_token", "AWS_TOKEN", "session_token"};
+ private static final String[] AWS_PROVIDERS = {"s3.credentials_provider_type", "AWS_CREDENTIALS_PROVIDER_TYPE"};
+ private static final String[] AWS_ROLES = {"s3.role_arn", "AWS_ROLE_ARN", "s3.external_id", "AWS_EXTERNAL_ID"};
+
+ private GcsAuthResolver() {
+ }
+
+ public static Optional<GcsAuth> resolve(Map<String, String> properties) {
+ boolean hasNativeProperties = hasNativeCredentialProperties(properties);
+ boolean hasGcsSelector = guessIsGcs(properties)
+ || "true".equalsIgnoreCase(getPropertyIgnoreCase(properties, "fs.gcs.support"));
+ boolean hasAccessKey = hasNonBlankProperty(properties, ACCESS_KEYS);
+ boolean hasSecretKey = hasNonBlankProperty(properties, SECRET_KEYS);
+ boolean hasLegacyAnonymous = hasLegacyAnonymousProvider(properties);
+ String provider = getPropertyIgnoreCase(properties, "provider");
+ boolean useLegacyAnonymousDefault = "S3".equalsIgnoreCase(provider)
+ && !hasNonBlankProperty(properties, AWS_PROVIDERS);
+ if (StringUtils.isNotBlank(provider) && !"GCP".equalsIgnoreCase(provider)) {
+ if (hasNativeProperties) {
+ throw new IllegalArgumentException("Native GCP authentication requires provider=GCP, but found: "
+ + provider);
+ }
+ // Legacy GCS configurations may explicitly select the S3-compatible protocol.
+ // Preserve their HMAC/anonymous authentication without enabling native ADC for S3.
+ if (!"S3".equalsIgnoreCase(provider) || !hasGcsSelector
+ || !(hasAccessKey || hasSecretKey || hasLegacyAnonymous || useLegacyAnonymousDefault)) {
+ return Optional.empty();
+ }
+ }
+ if (!hasNativeProperties && !hasGcsSelector) {
+ return Optional.empty();
+ }
+
+ String providerType = getPropertyIgnoreCase(properties, CREDENTIAL_PROVIDER_TYPE);
+ String impersonation = getPropertyIgnoreCase(properties, IMPERSONATION_SERVICE_ACCOUNT);
+ GcsAuth.Mode mode;
+ if (hasNativeProperties) {
+ GcpCredentialProviderType type = providerType == null ? GcpCredentialProviderType.DEFAULT
+ : GcpCredentialProviderType.fromString(CREDENTIAL_PROVIDER_TYPE, providerType);
+ mode = type == GcpCredentialProviderType.ANONYMOUS ? GcsAuth.Mode.ANONYMOUS
+ : type == GcpCredentialProviderType.COMPUTE_ENGINE ? GcsAuth.Mode.COMPUTE_ENGINE : GcsAuth.Mode.ADC;
+ } else if (hasLegacyAnonymous) {
+ mode = GcsAuth.Mode.ANONYMOUS;
+ } else if (hasAccessKey || hasSecretKey) {
+ mode = GcsAuth.Mode.HMAC;
+ } else if (useLegacyAnonymousDefault) {
+ mode = GcsAuth.Mode.ANONYMOUS;
+ } else {
+ mode = GcsAuth.Mode.ADC;
+ }
+
+ // The storage properties validate paired HMAC keys. Raw Vault patches may update only one key.
+ if (mode != GcsAuth.Mode.HMAC) {
+ if (hasAccessKey || hasSecretKey || hasNonBlankProperty(properties, TOKENS)) {
+ throw new IllegalArgumentException(CREDENTIAL_PROVIDER_TYPE
+ + " cannot be used together with access key, secret key, or session token.");
+ }
+ if (hasNonBlankProperty(properties, AWS_ROLES)
+ || ((hasNativeProperties || mode != GcsAuth.Mode.ANONYMOUS)
+ && hasNonBlankProperty(properties, AWS_PROVIDERS))) {
+ throw new IllegalArgumentException("Native GCP authentication cannot be used together with "
+ + "AWS role, external ID, or credentials provider properties.");
+ }
+ }
+ if (mode == GcsAuth.Mode.ANONYMOUS && StringUtils.isNotEmpty(impersonation)) {
+ throw new IllegalArgumentException("ANONYMOUS cannot be used with " + IMPERSONATION_SERVICE_ACCOUNT);
+ }
+ return Optional.of(new GcsAuth(mode, impersonation));
+ }
+
+ public static boolean hasNativeCredentialProperties(Map<String, String> properties) {
+ return properties.keySet().stream().anyMatch(key -> CREDENTIAL_PROVIDER_TYPE.equalsIgnoreCase(key)
+ || IMPERSONATION_SERVICE_ACCOUNT.equalsIgnoreCase(key));
+ }
+
+ // Storage selection also uses these signals, before authentication is resolved.
+ public static boolean guessIsGcs(Map<String, String> properties) {
+ for (Map.Entry<String, String> entry : properties.entrySet()) {
+ String key = entry.getKey().toLowerCase(Locale.ROOT);
+ String value = entry.getValue();
+ if ("provider".equals(key) && "GCP".equalsIgnoreCase(value)) {
+ return true;
+ }
+ if ((CREDENTIAL_PROVIDER_TYPE.equals(key) || IMPERSONATION_SERVICE_ACCOUNT.equals(key))
+ && StringUtils.isNotBlank(value)) {
+ return true;
+ }
+ if (("uri".equals(key) || "warehouse".equals(key)) && StringUtils.startsWithIgnoreCase(value, "gs://")) {
+ return true;
+ }
+ if ("gs.endpoint".equals(key) && StringUtils.isNotBlank(value)) {
+ return true;
+ }
+ if (("s3.endpoint".equals(key) || "aws_endpoint".equals(key) || "endpoint".equals(key))
+ && StringUtils.endsWithIgnoreCase(value, "storage.googleapis.com")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean hasLegacyAnonymousProvider(Map<String, String> properties) {
+ for (String name : AWS_PROVIDERS) {
+ if ("ANONYMOUS".equalsIgnoreCase(StringUtils.trim(getPropertyIgnoreCase(properties, name)))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean hasNonBlankProperty(Map<String, String> properties, String... names) {
+ for (String name : names) {
+ if (StringUtils.isNotBlank(getPropertyIgnoreCase(properties, name))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static String getPropertyIgnoreCase(Map<String, String> properties, String name) {
+ return properties.entrySet().stream()
+ .filter(entry -> name.equalsIgnoreCase(entry.getKey()))
+ .map(Map.Entry::getValue)
+ .findFirst()
+ .orElse(null);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/ObjCredential.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/ObjCredential.java
new file mode 100644
index 00000000000..2f6c9794f92
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/ObjCredential.java
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage.auth;
+
+import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB;
+import org.apache.doris.thrift.TS3StorageParam;
+
+/**
+ * Provider-native credential that can be serialized for Cloud or BE.
+ *
+ * <p>This transport abstraction is deliberately independent of {@code S3ObjStorage}.
+ * Runtime S3 client creation is extended through
+ * {@code AbstractS3CompatibleProperties.createS3Client} instead.
+ */
+public interface ObjCredential {
+ void applyTo(ObjectStoreInfoPB.Builder builder);
+
+ void applyTo(TS3StorageParam param);
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/ObjCredentialFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/ObjCredentialFactory.java
new file mode 100644
index 00000000000..e16bbf57115
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/auth/ObjCredentialFactory.java
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage.auth;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/** Selects exactly one provider-native credential from object-storage properties. */
+public final class ObjCredentialFactory {
+ private interface Parser {
+ Optional<ObjCredential> parse(Map<String, String> properties, Optional<GcsAuth> gcsAuth);
+ }
+
+ // Add one parser here when introducing OSS, Azure, or another native credential.
+ // Provider-specific resolvers select credentials; credential classes handle transport conversion.
+ private static final List<Parser> PARSERS = Arrays.asList((properties, gcsAuth) ->
+ gcsAuth.flatMap(GcsAuth::getNativeCredential).map(credential -> credential));
+
+ // TODO: Add AWS role-based authentication after its existing property and
+ // transport fields can be migrated without breaking compatibility. Static
+ // AK/SK credentials remain outside this provider-native credential model.
+
+ private ObjCredentialFactory() {
+ }
+
+ public static Optional<ObjCredential> fromProperties(Map<String, String> properties) {
+ return fromProperties(properties, GcsAuthResolver.resolve(properties));
+ }
+
+ /** Reuse the authentication choice already resolved by a protocol builder. */
+ public static Optional<ObjCredential> fromProperties(Map<String, String> properties, Optional<GcsAuth> gcsAuth) {
+ ObjCredential selected = null;
+ for (Parser parser : PARSERS) {
+ Optional<ObjCredential> parsed = parser.parse(properties, gcsAuth);
+ if (!parsed.isPresent()) {
+ continue;
+ }
+ if (selected != null) {
+ throw new IllegalArgumentException(
+ "Only one provider-native object storage credential may be configured.");
+ }
+ selected = parsed.get();
+ }
+ return Optional.ofNullable(selected);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java
index de342349022..c2d7fa1c8e7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java
@@ -115,8 +115,7 @@ public class S3ObjStorage implements ObjStorage<S3Client> {
@Override
public S3Client getClient() {
if (client == null) {
- client = S3Util.buildS3Client(s3Properties.getEndpoint(), s3Properties.getRegion(),
- isUsePathStyle, s3Properties.getAwsCredentialsProvider());
+ client = s3Properties.createS3Client(s3Properties.getEndpoint(), isUsePathStyle);
}
return client;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
index ea42631b2dd..3a12e97fef7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
@@ -274,7 +274,11 @@ public class ExportCommand extends Command implements NeedAuditEncryption, Forwa
exportJob.setWhereExpression(this.expr);
exportJob.setWhereStr(this.expr.isPresent() ? this.expr.get().toSql() : "");
// set path
- exportJob.setExportPath(this.path);
+ try {
+ exportJob.setExportPath(this.brokerDesc.get().getFileLocationForExport(this.path));
+ } catch (RuntimeException e) {
+ throw new AnalysisException("Invalid export path: " + this.path + ", " + e.getMessage(), e);
+ }
// set column separator
String columnSeparator = Separator.convertSeparator(fileProperties.getOrDefault(
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/StorageDescPersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/StorageDescPersistTest.java
index c0e7071fbae..e3c031836b4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/StorageDescPersistTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/StorageDescPersistTest.java
@@ -18,10 +18,12 @@
package org.apache.doris.analysis;
import org.apache.doris.datasource.property.storage.BrokerProperties;
+import org.apache.doris.datasource.property.storage.GCSProperties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.load.EtlJobType;
import org.apache.doris.load.loadv2.BrokerLoadJob;
import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.thrift.TFileType;
import com.google.common.collect.Maps;
import org.junit.Assert;
@@ -72,6 +74,25 @@ public class StorageDescPersistTest {
Assert.assertEquals("test-bucket", restoredStorageProperties.getOrigProps().get("s3.bucket"));
}
+ @Test
+ public void testGcpNativeCredentialForExportBroker() throws Exception {
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put("provider", "GCP");
+ properties.put("s3.region", "us-east1");
+ properties.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+
+ // EXPORT forwards this BrokerDesc map to BE Writer. Verify the shared FE conversion
+ // preserves the provider-specific credential while keeping the S3-compatible file type.
+ BrokerDesc brokerDesc = new BrokerDesc("S3", StorageBackend.StorageType.S3, properties);
+ Assert.assertTrue(brokerDesc.getStorageProperties() instanceof GCSProperties);
+ Assert.assertEquals(TFileType.FILE_S3, brokerDesc.getFileType());
+ Assert.assertEquals("GCP", brokerDesc.getBackendConfigProperties().get("provider"));
+ Assert.assertEquals("DEFAULT", brokerDesc.getBackendConfigProperties()
+ .get(GCSProperties.CREDENTIAL_PROVIDER_TYPE));
+ Assert.assertEquals("s3://export-bucket/path",
+ brokerDesc.getFileLocation("gs://export-bucket/path"));
+ }
+
private static void setField(Class<?> clazz, Object target, String fieldName, Object value)
throws ReflectiveOperationException {
Field field = clazz.getDeclaredField(fieldName);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
index e725b967315..59953396d29 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
@@ -21,13 +21,16 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.GCSProperties;
import org.apache.doris.datasource.property.storage.S3Properties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.meta.MetaContext;
import org.apache.doris.mysql.privilege.AccessControllerManager;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.trees.plans.commands.CreateResourceCommand;
import org.apache.doris.nereids.trees.plans.commands.info.CreateResourceInfo;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.thrift.TS3StorageParam;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
@@ -43,11 +46,16 @@ import org.junit.Test;
import java.io.DataInputStream;
import java.io.DataOutputStream;
+import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
public class S3ResourceTest {
private static final Logger LOG = LogManager.getLogger(S3ResourceTest.class);
@@ -211,6 +219,69 @@ public class S3ResourceTest {
Files.deleteIfExists(path);
}
+ @Test
+ public void testConcurrentAltersPreserveBothUpdates() throws Exception {
+ CountDownLatch firstReady = new CountDownLatch(1);
+ CountDownLatch releaseFirst = new CountDownLatch(1);
+ CountDownLatch secondStarted = new CountDownLatch(1);
+ AtomicBoolean first = new AtomicBoolean(true);
+ AtomicReference<Throwable> failure = new AtomicReference<>();
+ S3Resource resource = new S3Resource("concurrent_alter") {
+ @Override
+ public void writeLock() {
+ if (first.compareAndSet(true, false)) {
+ firstReady.countDown();
+ try {
+ Assert.assertTrue(releaseFirst.await(10, TimeUnit.SECONDS));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(e);
+ }
+ }
+ super.writeLock();
+ }
+ };
+ resource.setProperties(ImmutableMap.copyOf(s3Properties));
+ Thread updateConnections = new Thread(() -> {
+ try {
+ resource.modifyProperties(ImmutableMap.of(S3Properties.MAX_CONNECTIONS, "77"));
+ } catch (Throwable e) {
+ failure.compareAndSet(null, e);
+ }
+ });
+ Thread updateTimeout = new Thread(() -> {
+ secondStarted.countDown();
+ try {
+ resource.modifyProperties(ImmutableMap.of(S3Properties.REQUEST_TIMEOUT_MS, "8888"));
+ } catch (Throwable e) {
+ failure.compareAndSet(null, e);
+ }
+ });
+ try {
+ updateConnections.start();
+ Assert.assertTrue(firstReady.await(10, TimeUnit.SECONDS));
+ updateTimeout.start();
+ Assert.assertTrue(secondStarted.await(10, TimeUnit.SECONDS));
+ // The second ALTER either waits for the first one's monitor, or (before the fix)
+ // finishes and is subsequently overwritten by the first one's stale snapshot.
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (updateTimeout.isAlive() && updateTimeout.getState() != Thread.State.BLOCKED
+ && System.nanoTime() < deadline) {
+ Thread.yield();
+ }
+ Assert.assertTrue(!updateTimeout.isAlive() || updateTimeout.getState() == Thread.State.BLOCKED);
+ } finally {
+ releaseFirst.countDown();
+ updateConnections.join(10000);
+ updateTimeout.join(10000);
+ }
+ Assert.assertFalse(updateConnections.isAlive());
+ Assert.assertFalse(updateTimeout.isAlive());
+ Assert.assertNull(failure.get());
+ Assert.assertEquals("77", resource.getProperty(S3Properties.MAX_CONNECTIONS));
+ Assert.assertEquals("8888", resource.getProperty(S3Properties.REQUEST_TIMEOUT_MS));
+ }
+
@Test
public void testModifyProperties() throws Exception {
ImmutableMap<String, String> properties = ImmutableMap.of(
@@ -319,4 +390,372 @@ public class S3ResourceTest {
Assert.assertTrue(e.getMessage(), false);
}
}
+
+ @Test
+ public void testGsEndpointAndAlterAliases() throws Exception {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("provider", "gcp");
+ properties.put("s3.endpoint", "https://old.example.com");
+ properties.put("gs.endpoint", "https://storage.googleapis.com");
+ properties.put("s3.region", "us-central1");
+ properties.put("s3.bucket", "bucket");
+ properties.put("s3.root.path", "prefix");
+ properties.put("s3.access_key", "access");
+ properties.put("s3.secret_key", "secret");
+ properties.put("s3_validity_check", "false");
+ properties.put("type", "s3");
+ S3Resource resource = new S3Resource("gcp_resource");
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ Assert.assertEquals("us-central1", resource.getProperty(S3Properties.REGION));
+ Assert.assertEquals("bucket", resource.getProperty(S3Properties.BUCKET));
+ Assert.assertEquals("prefix", resource.getProperty(S3Properties.ROOT_PATH));
+ Assert.assertEquals("https://storage.googleapis.com", resource.getProperty(S3Properties.ENDPOINT));
+ Assert.assertEquals("https://storage.googleapis.com",
+ S3Properties.getS3TStorageParam(resource.getCopiedProperties()).getEndpoint());
+ Assert.assertFalse(resource.getCopiedProperties().containsKey("gs.endpoint"));
+ resource.modifyProperties(ImmutableMap.of("gs.endpoint", "https://custom.example.com",
+ "s3_validity_check", "false"));
+ Assert.assertEquals("https://custom.example.com", resource.getProperty(S3Properties.ENDPOINT));
+ resource.modifyProperties(ImmutableMap.of("s3.endpoint", "https://storage.googleapis.com",
+ "s3_validity_check", "false"));
+ Assert.assertEquals("https://storage.googleapis.com", resource.getProperty(S3Properties.ENDPOINT));
+ Assert.assertEquals("https://storage.googleapis.com", properties.get("gs.endpoint"));
+ Assert.assertEquals("https://old.example.com", properties.get("s3.endpoint"));
+ }
+
+ @Test
+ public void testGsEndpointForStorageVault(@Mocked Env env) throws Exception {
+ for (String provider : new String[] {"GCP", "gcp", "S3", "OSS"}) {
+ Map<String, String> properties = new HashMap<>(s3Properties);
+ properties.put("provider", provider);
+ properties.put("gs.endpoint", "https://storage.googleapis.com");
+ properties.put("gs.access_key", "gcp-access");
+ properties.put("gs.secret_key", "gcp-secret");
+ properties.put("gs.use_path_style", "true");
+ properties.put("s3.root.path", "prefix");
+ CreateResourceCommand command = new CreateResourceCommand(
+ new CreateResourceInfo(false, false, "gcp_vault", ImmutableMap.copyOf(properties)));
+ command.getInfo().analyzeResourceType();
+ S3StorageVault vault = new S3StorageVault("gcp_vault", false, false, command);
+ vault.checkCreationProperties(ImmutableMap.copyOf(properties));
+ Assert.assertEquals("prefix", vault.getCopiedProperties().get(S3Properties.ROOT_PATH));
+ String expectedEndpoint = "GCP".equalsIgnoreCase(provider)
+ ? "https://storage.googleapis.com" : s3Endpoint;
+ Assert.assertEquals(expectedEndpoint,
+ S3Properties.getObjStoreInfoPB(vault.getCopiedProperties()).getEndpoint());
+ Assert.assertEquals("GCP".equalsIgnoreCase(provider) ? "gcp-access" : s3AccessKey,
+ S3Properties.getObjStoreInfoPB(vault.getCopiedProperties()).getAk());
+ Assert.assertEquals("GCP".equalsIgnoreCase(provider) ? "gcp-secret" : s3SecretKey,
+ S3Properties.getObjStoreInfoPB(vault.getCopiedProperties()).getSk());
+ Assert.assertEquals("GCP".equalsIgnoreCase(provider),
+ S3Properties.getObjStoreInfoPB(vault.getCopiedProperties()).getUsePathStyle());
+ }
+ }
+
+ @Test
+ public void testStorageVaultRejectsNonAllowlistedAliases() throws Exception {
+ StorageVaultMgr manager = new StorageVaultMgr(null);
+ for (String key : new String[] {"gs.endpoint", S3Properties.ENDPOINT, S3Properties.Env.ENDPOINT,
+ "gs.access_key", "gs.secret_key", "gs.session_token", "gs.connection.maximum",
+ "gs.connection.request.timeout", "gs.connection.timeout", "gs.use_path_style",
+ "gs.force_parsing_by_standard_uri"}) {
+ try {
+ manager.alterStorageVault(StorageVault.StorageVaultType.S3,
+ ImmutableMap.of(key, "https://custom.example.com"), "gcp_vault");
+ Assert.fail("Storage Vault must reject properties outside its ALTER whitelist: " + key);
+ } catch (IllegalArgumentException exception) {
+ Assert.assertEquals("Alter property " + key + " is not allowed.", exception.getMessage());
+ }
+ }
+ }
+
+ @Test
+ public void testBlankGsEndpointPreservesResourceAliases() throws Exception {
+ for (String blank : new String[] {"", " \t"}) {
+ for (String endpointKey : new String[] {S3Properties.ENDPOINT, S3Properties.Env.ENDPOINT}) {
+ Map<String, String> properties = new HashMap<>(s3Properties);
+ properties.put("provider", "GCP");
+ properties.remove(S3Properties.Env.ENDPOINT);
+ properties.put(endpointKey, s3Endpoint);
+ properties.put("gs.endpoint", blank);
+ properties.put("gs.connection.timeout", blank);
+ S3Resource resource = new S3Resource("gcp_resource");
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ Assert.assertEquals(s3Endpoint, resource.getProperty(S3Properties.ENDPOINT));
+ Assert.assertFalse(resource.getCopiedProperties().containsKey("gs.endpoint"));
+ Assert.assertEquals(Integer.parseInt(s3ConnTimeoutMs),
+ S3Properties.getS3TStorageParam(resource.getCopiedProperties()).getConnTimeoutMs());
+ Map<String, String> updated = new HashMap<>();
+ updated.put("gs.endpoint", blank);
+ updated.put("s3_validity_check", "false");
+ resource.modifyProperties(updated);
+ Assert.assertEquals(s3Endpoint, resource.getProperty(S3Properties.ENDPOINT));
+ Assert.assertEquals(blank, updated.get("gs.endpoint"));
+ Assert.assertFalse(resource.getCopiedProperties().containsKey("gs.connection.timeout"));
+ }
+ }
+ }
+
+ @Test
+ public void testGsEndpointCannotAlterPolicyResource() throws Exception {
+ S3Resource resource = new S3Resource("gcp_resource");
+ Map<String, String> properties = new HashMap<>(s3Properties);
+ properties.put("provider", "GCP");
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ resource.references.put("policy", Resource.ReferenceType.POLICY);
+ try {
+ resource.modifyProperties(ImmutableMap.of("gs.endpoint", "https://custom.example.com"));
+ Assert.fail("A gs.endpoint alias must not bypass the policy endpoint restriction");
+ } catch (DdlException exception) {
+ Assert.assertTrue(exception.getMessage().contains(S3Properties.ENDPOINT));
+ }
+ Assert.assertEquals(s3Endpoint, resource.getProperty(S3Properties.ENDPOINT));
+ }
+
+ @Test
+ public void testGsEndpointDoesNotOverrideOtherProviders() throws Exception {
+ for (String provider : new String[] {"S3", "OSS", "AZURE", ""}) {
+ Map<String, String> properties = new HashMap<>(s3Properties);
+ if (!provider.isEmpty()) {
+ properties.put("provider", provider);
+ }
+ properties.put("gs.endpoint", "https://storage.googleapis.com");
+ S3Resource resource = new S3Resource("other_resource");
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ Assert.assertEquals(s3Endpoint, resource.getProperty(S3Properties.ENDPOINT));
+ resource.modifyProperties(ImmutableMap.of("gs.endpoint", "https://custom.example.com"));
+ Assert.assertEquals(s3Endpoint, resource.getProperty(S3Properties.ENDPOINT));
+ Assert.assertEquals(s3Endpoint, resource.getProperty(S3Properties.Env.ENDPOINT));
+ }
+ }
+
+ @Test
+ public void testGsEndpointUsesEffectiveProviderOnAlter() throws Exception {
+ S3Resource resource = new S3Resource("gcp_resource");
+ Map<String, String> properties = new HashMap<>(s3Properties);
+ properties.put("provider", "GCP");
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ resource.modifyProperties(ImmutableMap.of("provider", "S3", "gs.endpoint", "https://ignored.example.com"));
+ Assert.assertEquals("S3", resource.getProperty("provider"));
+ Assert.assertEquals(s3Endpoint, resource.getProperty(S3Properties.ENDPOINT));
+ resource.modifyProperties(ImmutableMap.of("provider", "gcp", "gs.endpoint", "https://storage.googleapis.com"));
+ Assert.assertEquals("gcp", resource.getProperty("provider"));
+ Assert.assertEquals("https://storage.googleapis.com", resource.getProperty(S3Properties.ENDPOINT));
+ resource.modifyProperties(ImmutableMap.of("provider", "", "gs.endpoint", "https://custom.example.com"));
+ Assert.assertEquals("gcp", resource.getProperty("provider"));
+ Assert.assertEquals("https://custom.example.com", resource.getProperty(S3Properties.ENDPOINT));
+ }
+
+ @Test
+ public void testGsAliasesMatchConnectorAndResourceProtocols() throws Exception {
+ for (String provider : new String[] {"GCP", "gcp"}) {
+ Map<String, String> properties = new HashMap<>(s3Properties);
+ properties.put("provider", provider);
+ properties.put("gs.endpoint", "https://storage.googleapis.com");
+ properties.put("gs.access_key", "gcp-access");
+ properties.put("gs.secret_key", "gcp-secret");
+ properties.put("gs.session_token", "gcp-token");
+ properties.put("gs.connection.maximum", "123");
+ properties.put("gs.connection.request.timeout", "456");
+ properties.put("gs.connection.timeout", "789");
+ properties.put("gs.use_path_style", "true");
+ properties.put("gs.force_parsing_by_standard_uri", "true");
+ properties.put(S3Properties.SECRET_KEY, "other-secret");
+ properties.put(S3Properties.CONNECTION_TIMEOUT_MS, "999");
+ Map<String, String> original = new HashMap<>(properties);
+ GCSProperties connector = (GCSProperties) StorageProperties.createPrimary(properties);
+
+ S3Resource resource = new S3Resource("gcp_resource");
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ Map<String, String> normalized = resource.getCopiedProperties();
+ Assert.assertFalse(normalized.keySet().stream().anyMatch(key -> key.startsWith("gs.")));
+ Assert.assertEquals("true", normalized.get("force_parsing_by_standard_uri"));
+ TS3StorageParam thrift = S3Properties.getS3TStorageParam(normalized);
+ Assert.assertEquals(connector.getEndpoint(), thrift.getEndpoint());
+ Assert.assertEquals(connector.getAccessKey(), thrift.getAk());
+ Assert.assertEquals(connector.getSecretKey(), thrift.getSk());
+ Assert.assertEquals(connector.getSessionToken(), thrift.getToken());
+ Assert.assertEquals(Integer.parseInt(connector.getMaxConnections()), thrift.getMaxConn());
+ Assert.assertEquals(Integer.parseInt(connector.getRequestTimeoutS()), thrift.getRequestTimeoutMs());
+ Assert.assertEquals(Integer.parseInt(connector.getConnectionTimeoutS()), thrift.getConnTimeoutMs());
+ Assert.assertEquals(Boolean.parseBoolean(connector.getUsePathStyle()), thrift.isUsePathStyle());
+ Assert.assertEquals(connector.getAccessKey(), S3Properties.getObjStoreInfoPB(normalized).getAk());
+ Assert.assertEquals(connector.getSecretKey(), S3Properties.getObjStoreInfoPB(normalized).getSk());
+ Assert.assertEquals(thrift.isUsePathStyle(), S3Properties.getObjStoreInfoPB(normalized).getUsePathStyle());
+ Assert.assertEquals(original, properties);
+
+ resource.modifyProperties(ImmutableMap.of("gs.secret_key", "new-secret",
+ "gs.connection.timeout", "321", "gs.use_path_style", "false"));
+ thrift = S3Properties.getS3TStorageParam(resource.getCopiedProperties());
+ Assert.assertEquals("new-secret", thrift.getSk());
+ Assert.assertEquals(321, thrift.getConnTimeoutMs());
+ Assert.assertFalse(thrift.isUsePathStyle());
+ resource.modifyProperties(ImmutableMap.of(S3Properties.SECRET_KEY, "canonical-secret",
+ S3Properties.SESSION_TOKEN, ""));
+ connector = (GCSProperties) StorageProperties.createPrimary(resource.getCopiedProperties());
+ Assert.assertEquals("canonical-secret", connector.getSecretKey());
+ Assert.assertEquals("", connector.getSessionToken());
+ }
+ }
+
+ @Test
+ public void testBlankGsCredentialAndConnectionAliases() throws Exception {
+ for (String blank : new String[] {"", " \t"}) {
+ Map<String, String> properties = new HashMap<>(s3Properties);
+ properties.put("provider", "GCP");
+ properties.put("gs.access_key", blank);
+ properties.put("gs.secret_key", blank);
+ properties.put("gs.session_token", blank);
+ properties.put(S3Properties.SESSION_TOKEN, "canonical-token");
+ properties.put("gs.connection.maximum", blank);
+ properties.put(S3Properties.MAX_CONNECTIONS, "123");
+ properties.put("gs.use_path_style", blank);
+ properties.put(S3Properties.USE_PATH_STYLE, "true");
+ S3Resource resource = new S3Resource("gcp_resource");
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ resource.modifyProperties(ImmutableMap.of("gs.secret_key", blank, "gs.connection.maximum", blank));
+ TS3StorageParam thrift = S3Properties.getS3TStorageParam(resource.getCopiedProperties());
+ Assert.assertEquals(s3AccessKey, thrift.getAk());
+ Assert.assertEquals(s3SecretKey, thrift.getSk());
+ Assert.assertEquals("canonical-token", thrift.getToken());
+ Assert.assertEquals(123, thrift.getMaxConn());
+ Assert.assertTrue(thrift.isUsePathStyle());
+ Assert.assertFalse(resource.getCopiedProperties().keySet().stream().anyMatch(key -> key.startsWith("gs.")));
+ }
+ }
+
+ @Test
+ public void testGsAliasesDoNotOverrideOtherProviders() throws Exception {
+ for (String provider : new String[] {"S3", "OSS", "AZURE", ""}) {
+ Map<String, String> properties = new HashMap<>(s3Properties);
+ properties.put("provider", provider);
+ properties.put("gs.access_key", "ignored-access");
+ properties.put("gs.secret_key", "ignored-secret");
+ properties.put("gs.connection.timeout", "789");
+ properties.put("gs.use_path_style", "true");
+ S3Resource resource = new S3Resource("other_resource");
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ resource.modifyProperties(ImmutableMap.of("gs.secret_key", "also-ignored",
+ "gs.connection.timeout", "321"));
+ TS3StorageParam thrift = S3Properties.getS3TStorageParam(resource.getCopiedProperties());
+ Assert.assertEquals(s3AccessKey, thrift.getAk());
+ Assert.assertEquals(s3SecretKey, thrift.getSk());
+ Assert.assertEquals(Integer.parseInt(s3ConnTimeoutMs), thrift.getConnTimeoutMs());
+ Assert.assertFalse(thrift.isUsePathStyle());
+ }
+ }
+
+ @Test
+ public void testGsCredentialAliasesCannotBypassNativeAuthValidation() throws Exception {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("provider", "GCP");
+ properties.put("gs.endpoint", "https://storage.googleapis.com");
+ properties.put(S3Properties.BUCKET, "bucket");
+ properties.put(S3Properties.VALIDITY_CHECK, "false");
+ properties.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ for (String key : new String[] {"gs.access_key", "gs.secret_key", "gs.session_token",
+ S3Properties.ACCESS_KEY, S3Properties.SECRET_KEY, S3Properties.SESSION_TOKEN}) {
+ S3Resource resource = new S3Resource("gcp_resource");
+ Map<String, String> conflicting = new HashMap<>(properties);
+ conflicting.put(key, "conflicting-credential");
+ Assert.assertThrows(IllegalArgumentException.class,
+ () -> resource.setProperties(ImmutableMap.copyOf(conflicting)));
+
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ Map<String, String> before = resource.getCopiedProperties();
+ Assert.assertThrows(IllegalArgumentException.class,
+ () -> resource.modifyProperties(ImmutableMap.of(key, "conflicting-credential")));
+ Assert.assertEquals(before, resource.getCopiedProperties());
+ Assert.assertEquals("DEFAULT", resource.getProperty(GCSProperties.CREDENTIAL_PROVIDER_TYPE));
+ }
+ S3Resource resource = new S3Resource("gcp_resource");
+ properties.remove(GCSProperties.CREDENTIAL_PROVIDER_TYPE);
+ properties.put("gs.access_key", "access");
+ properties.put("gs.secret_key", "secret");
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ Assert.assertThrows(IllegalArgumentException.class, () -> resource.modifyProperties(
+ ImmutableMap.of(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT")));
+ // Empty AK/SK updates are ignored by Resource ALTER and cannot clear stored credentials.
+ Assert.assertThrows(IllegalArgumentException.class, () -> resource.modifyProperties(
+ ImmutableMap.of(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT",
+ S3Properties.ACCESS_KEY, "", S3Properties.SECRET_KEY, "")));
+ }
+
+ @Test
+ public void testGsAliasesTakePrecedenceAfterMergingAlter() throws Exception {
+ Map<String, String> properties = new HashMap<>(s3Properties);
+ properties.put("provider", "GCP");
+ S3Resource resource = new S3Resource("gcp_resource");
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ // Simulate a resource persisted before gs.* normalization was introduced.
+ Field field = S3Resource.class.getDeclaredField("properties");
+ field.setAccessible(true);
+ Map<String, String> legacy = resource.getCopiedProperties();
+ legacy.put("gs.secret_key", "old-secret");
+ legacy.put("gs.session_token", "old-token");
+ legacy.put("gs.connection.timeout", "789");
+ field.set(resource, legacy);
+ resource.modifyProperties(ImmutableMap.of(S3Properties.SECRET_KEY, "new-secret",
+ S3Properties.SESSION_TOKEN, ""));
+ Map<String, String> normalized = resource.getCopiedProperties();
+ GCSProperties connector = (GCSProperties) StorageProperties.createPrimary(normalized);
+ Assert.assertEquals("old-secret", connector.getSecretKey());
+ Assert.assertEquals("old-token", connector.getSessionToken());
+ Assert.assertEquals(connector.getSecretKey(), S3Properties.getS3TStorageParam(normalized).getSk());
+ Assert.assertEquals(connector.getSessionToken(), S3Properties.getS3TStorageParam(normalized).getToken());
+ Assert.assertEquals(connector.getSecretKey(), S3Properties.getObjStoreInfoPB(normalized).getSk());
+ Assert.assertEquals(789, S3Properties.getS3TStorageParam(normalized).getConnTimeoutMs());
+ Assert.assertFalse(normalized.containsKey("gs.secret_key"));
+ Assert.assertFalse(normalized.containsKey("gs.session_token"));
+ Assert.assertFalse(normalized.containsKey("gs.connection.timeout"));
+
+ // Within the same ALTER input, gs.* also takes precedence over s3.*.
+ resource.modifyProperties(ImmutableMap.of("gs.secret_key", "updated-gs-secret",
+ S3Properties.SECRET_KEY, "ignored-s3-secret"));
+ Assert.assertEquals("updated-gs-secret", resource.getProperty(S3Properties.SECRET_KEY));
+ }
+
+ @Test
+ public void testBlankStoredGsAliasesFallBackToAlterValues() throws Exception {
+ for (String blank : new String[] {"", " \t"}) {
+ Map<String, String> properties = new HashMap<>(s3Properties);
+ properties.put("provider", "GCP");
+ S3Resource resource = new S3Resource("gcp_resource");
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ Field field = S3Resource.class.getDeclaredField("properties");
+ field.setAccessible(true);
+ Map<String, String> legacy = resource.getCopiedProperties();
+ legacy.put("gs.secret_key", blank);
+ legacy.put("gs.session_token", blank);
+ field.set(resource, legacy);
+
+ resource.modifyProperties(ImmutableMap.of(S3Properties.SECRET_KEY, "new-secret",
+ S3Properties.SESSION_TOKEN, ""));
+ Map<String, String> normalized = resource.getCopiedProperties();
+ GCSProperties connector = (GCSProperties) StorageProperties.createPrimary(normalized);
+ Assert.assertEquals("new-secret", connector.getSecretKey());
+ Assert.assertEquals("", connector.getSessionToken());
+ Assert.assertEquals(connector.getSecretKey(), S3Properties.getS3TStorageParam(normalized).getSk());
+ Assert.assertFalse(normalized.containsKey("gs.secret_key"));
+ Assert.assertFalse(normalized.containsKey("gs.session_token"));
+ }
+ }
+
+ @Test
+ public void testStoredGsEndpointCannotChangePolicyResourceOnAlter() throws Exception {
+ Map<String, String> properties = new HashMap<>(s3Properties);
+ properties.put("provider", "GCP");
+ S3Resource resource = new S3Resource("gcp_resource");
+ resource.setProperties(ImmutableMap.copyOf(properties));
+ resource.references.put("policy", Resource.ReferenceType.POLICY);
+ Field field = S3Resource.class.getDeclaredField("properties");
+ field.setAccessible(true);
+ Map<String, String> legacy = resource.getCopiedProperties();
+ legacy.put("gs.endpoint", "https://different.example.com");
+ field.set(resource, legacy);
+
+ Assert.assertThrows(DdlException.class, () -> resource.modifyProperties(
+ ImmutableMap.of(S3Properties.SECRET_KEY, "new-secret")));
+ Assert.assertEquals(legacy, resource.getCopiedProperties());
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/S3StorageVaultTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/S3StorageVaultTest.java
new file mode 100644
index 00000000000..02b9d545941
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/S3StorageVaultTest.java
@@ -0,0 +1,253 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.cloud.catalog;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.StorageVault;
+import org.apache.doris.catalog.StorageVault.StorageVaultType;
+import org.apache.doris.catalog.StorageVaultMgr;
+import org.apache.doris.cloud.proto.Cloud;
+import org.apache.doris.cloud.rpc.MetaServiceProxy;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.property.storage.GCSProperties;
+import org.apache.doris.datasource.property.storage.S3Properties;
+import org.apache.doris.nereids.trees.plans.commands.CreateStorageVaultCommand;
+import org.apache.doris.system.SystemInfoService;
+
+import mockit.Mock;
+import mockit.MockUp;
+import mockit.Mocked;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class S3StorageVaultTest {
+ private static final String ROLE = "arn:aws:iam::123456789012:role/test-role";
+ private static final String ACCOUNT = "target@my-project.iam.gserviceaccount.com";
+
+ @Mocked
+ private Env env;
+
+ // Capture the actual CREATE/ALTER RPC, then stop before cache updates and backend synchronization.
+ private Cloud.ObjectStoreInfoPB captureRequest(Map<String, String> auth, boolean isAlter) throws Exception {
+ AtomicReference<Cloud.AlterObjStoreInfoRequest> captured = new AtomicReference<>();
+ new MockUp<MetaServiceProxy>() {
+ @Mock
+ public Cloud.AlterObjStoreInfoResponse alterStorageVault(Cloud.AlterObjStoreInfoRequest request) {
+ captured.set(request);
+ return Cloud.AlterObjStoreInfoResponse.newBuilder().setStatus(
+ Cloud.MetaServiceResponseStatus.newBuilder().setCode(Cloud.MetaServiceCode.INVALID_ARGUMENT)
+ .setMsg("captured vault request")).build();
+ }
+ };
+ String oldEndpoint = Config.meta_service_endpoint;
+ String oldCloudId = Config.cloud_unique_id;
+ Config.meta_service_endpoint = "127.0.0.1:20121";
+ Config.cloud_unique_id = "vault-default-test";
+ try {
+ StorageVaultMgr mgr = new StorageVaultMgr(new SystemInfoService());
+ if (isAlter) {
+ Assertions.assertThrows(DdlException.class,
+ () -> mgr.alterStorageVault(StorageVaultType.S3, new HashMap<>(auth), "iam_vault"));
+ } else {
+ Map<String, String> properties = new HashMap<>(auth);
+ boolean isGcp = "GCP".equals(auth.get("provider"))
+ || auth.keySet().stream().anyMatch(key -> key.toLowerCase().startsWith("gs."));
+ properties.put("type", "S3");
+ properties.put("provider", isGcp ? "GCP" : "S3");
+ properties.put(S3Properties.ENDPOINT, isGcp ? "storage.googleapis.com" : "s3.us-east-1.amazonaws.com");
+ properties.put(S3Properties.REGION, "us-east-1");
+ properties.put(S3Properties.BUCKET, "test-bucket");
+ properties.put(S3Properties.ROOT_PATH, "test-root");
+ properties.put(S3Properties.VALIDITY_CHECK, "false");
+ CreateStorageVaultCommand command = new CreateStorageVaultCommand(false, "iam_vault", properties);
+ command.setStorageVaultType(StorageVaultType.S3);
+ StorageVault vault = StorageVault.fromCommand(command);
+ Assertions.assertThrows(DdlException.class, () -> mgr.createS3Vault(vault));
+ }
+ Assertions.assertNotNull(captured.get(), "The request must reach the MS RPC boundary");
+ Assertions.assertEquals(isAlter ? Cloud.AlterObjStoreInfoRequest.Operation.ALTER_S3_VAULT
+ : Cloud.AlterObjStoreInfoRequest.Operation.ADD_S3_VAULT, captured.get().getOp());
+ if (isAlter) {
+ Assertions.assertFalse(captured.get().getVault().getObjInfo().hasProvider(),
+ "ALTER must not send an inferred immutable provider");
+ }
+ return captured.get().getVault().getObjInfo();
+ } finally {
+ Config.meta_service_endpoint = oldEndpoint;
+ Config.cloud_unique_id = oldCloudId;
+ }
+ }
+
+ @Test
+ public void testAwsCreatePreservesDefaultAndExplicitProvider() throws Exception {
+ Map<String, String> properties = new HashMap<>();
+ properties.put(S3Properties.ROLE_ARN, ROLE);
+ Cloud.ObjectStoreInfoPB obj = captureRequest(properties, false);
+ Assertions.assertTrue(obj.hasCredProviderType());
+ Assertions.assertEquals(Cloud.CredProviderTypePB.INSTANCE_PROFILE, obj.getCredProviderType());
+ for (String key : new String[] {S3Properties.CREDENTIALS_PROVIDER_TYPE,
+ S3Properties.Env.CREDENTIALS_PROVIDER_TYPE}) {
+ Map<String, String> explicit = new HashMap<>(properties);
+ explicit.put(key, "CONTAINER");
+ obj = captureRequest(explicit, false);
+ Assertions.assertTrue(obj.hasCredProviderType());
+ Assertions.assertEquals(Cloud.CredProviderTypePB.CONTAINER, obj.getCredProviderType());
+ }
+ }
+
+ @Test
+ public void testAwsAlterPreservesOmissionAndExplicitClears() throws Exception {
+ Map<String, String> properties = new HashMap<>();
+ properties.put(S3Properties.ROLE_ARN, ROLE);
+ Assertions.assertFalse(captureRequest(properties, true).hasCredProviderType());
+ properties.put(S3Properties.CREDENTIALS_PROVIDER_TYPE, "DEFAULT");
+ Cloud.ObjectStoreInfoPB obj = captureRequest(properties, true);
+ Assertions.assertTrue(obj.hasCredProviderType());
+ Assertions.assertEquals(Cloud.CredProviderTypePB.DEFAULT, obj.getCredProviderType());
+ properties.clear();
+ properties.put(S3Properties.ROLE_ARN, "");
+ properties.put(S3Properties.EXTERNAL_ID, "");
+ obj = captureRequest(properties, true);
+ Assertions.assertTrue(obj.hasRoleArn());
+ Assertions.assertEquals("", obj.getRoleArn());
+ Assertions.assertTrue(obj.hasExternalId());
+ Assertions.assertEquals("", obj.getExternalId());
+ Assertions.assertFalse(obj.hasCredProviderType());
+ }
+
+ @Test
+ public void testGcpCreateDefaultsAndExplicitProvider() throws Exception {
+ Map<String, String> properties = new HashMap<>();
+ properties.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT, ACCOUNT);
+ Cloud.GcpCredentialPB credential = captureRequest(properties, false).getCredential().getGcpCredential();
+ Assertions.assertTrue(credential.hasCredentialProviderType());
+ Assertions.assertEquals(Cloud.GcpCredentialPB.CredentialProviderType.DEFAULT,
+ credential.getCredentialProviderType());
+ Assertions.assertEquals(ACCOUNT, credential.getImpersonationServiceAccount());
+ properties.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "COMPUTE_ENGINE");
+ credential = captureRequest(properties, false).getCredential().getGcpCredential();
+ Assertions.assertTrue(credential.hasCredentialProviderType());
+ Assertions.assertEquals(Cloud.GcpCredentialPB.CredentialProviderType.COMPUTE_ENGINE,
+ credential.getCredentialProviderType());
+ }
+
+ @Test
+ public void testGcpEmptyTargetDefaultsToDefault() throws Exception {
+ Map<String, String> properties = new HashMap<>();
+ properties.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT, "");
+ Cloud.GcpCredentialPB credential = captureRequest(properties, false).getCredential().getGcpCredential();
+ Assertions.assertTrue(credential.hasCredentialProviderType());
+ Assertions.assertEquals(Cloud.GcpCredentialPB.CredentialProviderType.DEFAULT,
+ credential.getCredentialProviderType());
+ Assertions.assertFalse(credential.hasImpersonationServiceAccount());
+
+ credential = captureRequest(properties, true).getCredential().getGcpCredential();
+ Assertions.assertFalse(credential.hasCredentialProviderType());
+ Assertions.assertTrue(credential.hasImpersonationServiceAccount());
+ Assertions.assertEquals("", credential.getImpersonationServiceAccount());
+ }
+
+ @Test
+ public void testGcpAlterPreservesOmittedFields() throws Exception {
+ Map<String, String> properties = new HashMap<>();
+ properties.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT, ACCOUNT);
+ Cloud.GcpCredentialPB credential = captureRequest(properties, true).getCredential().getGcpCredential();
+ Assertions.assertFalse(credential.hasCredentialProviderType());
+ Assertions.assertTrue(credential.hasImpersonationServiceAccount());
+ Assertions.assertEquals(ACCOUNT, credential.getImpersonationServiceAccount());
+ properties.clear();
+ properties.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "COMPUTE_ENGINE");
+ credential = captureRequest(properties, true).getCredential().getGcpCredential();
+ Assertions.assertTrue(credential.hasCredentialProviderType());
+ Assertions.assertEquals(Cloud.GcpCredentialPB.CredentialProviderType.COMPUTE_ENGINE,
+ credential.getCredentialProviderType());
+ Assertions.assertFalse(credential.hasImpersonationServiceAccount());
+ properties.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT, ACCOUNT);
+ credential = captureRequest(properties, true).getCredential().getGcpCredential();
+ Assertions.assertTrue(credential.hasCredentialProviderType());
+ Assertions.assertEquals(Cloud.GcpCredentialPB.CredentialProviderType.COMPUTE_ENGINE,
+ credential.getCredentialProviderType());
+ Assertions.assertTrue(credential.hasImpersonationServiceAccount());
+ Assertions.assertEquals(ACCOUNT, credential.getImpersonationServiceAccount());
+ properties.clear();
+ properties.put(S3Properties.USE_PATH_STYLE, "true");
+ Cloud.ObjectStoreInfoPB obj = captureRequest(properties, true);
+ Assertions.assertFalse(obj.hasCredential());
+ Assertions.assertFalse(obj.hasCredProviderType());
+ }
+
+ @Test
+ public void testGcpCreateWithoutAuthenticationDefaultsToAdc() throws Exception {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("provider", "GCP");
+ Cloud.ObjectStoreInfoPB obj = captureRequest(properties, false);
+ Assertions.assertTrue(obj.hasCredential());
+ Assertions.assertEquals(Cloud.GcpCredentialPB.CredentialProviderType.DEFAULT,
+ obj.getCredential().getGcpCredential().getCredentialProviderType());
+ Assertions.assertFalse(obj.getCredential().getGcpCredential().hasImpersonationServiceAccount());
+ }
+
+ @Test
+ public void testGcpAnonymousAliasesAreRejectedBeforeCreateRpc() throws Exception {
+ new MockUp<MetaServiceProxy>() {
+ @Mock
+ public Cloud.AlterObjStoreInfoResponse alterStorageVault(Cloud.AlterObjStoreInfoRequest request) {
+ throw new AssertionError("Anonymous GCS vaults must be rejected before the MS RPC");
+ }
+ };
+ StorageVaultMgr mgr = new StorageVaultMgr(new SystemInfoService());
+ for (String key : new String[] {GCSProperties.CREDENTIAL_PROVIDER_TYPE,
+ S3Properties.CREDENTIALS_PROVIDER_TYPE, S3Properties.Env.CREDENTIALS_PROVIDER_TYPE}) {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("type", "S3");
+ properties.put("provider", "GCP");
+ properties.put(S3Properties.ENDPOINT, "storage.googleapis.com");
+ properties.put(S3Properties.REGION, "us-east1");
+ properties.put(S3Properties.BUCKET, "test-bucket");
+ properties.put(S3Properties.ROOT_PATH, "test-root");
+ properties.put(S3Properties.VALIDITY_CHECK, "false");
+ properties.put(key, " anonymous ");
+ CreateStorageVaultCommand command = new CreateStorageVaultCommand(false, "anonymous_gcp", properties);
+ command.setStorageVaultType(StorageVaultType.S3);
+ StorageVault vault = StorageVault.fromCommand(command);
+ IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class,
+ () -> mgr.createS3Vault(vault), key);
+ Assertions.assertTrue(error.getMessage().contains("storage vaults"), key);
+ }
+ }
+
+ @Test
+ public void testGcpStorageOnlyAlterDoesNotInjectAdc() throws Exception {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("s3.endpoint", "storage.googleapis.com");
+ // Inspect the builder directly: ALTER's public API may reject immutable storage fields.
+ java.lang.reflect.Method method = StorageVaultMgr.class.getDeclaredMethod(
+ "buildAlterS3VaultRequest", Map.class, String.class);
+ method.setAccessible(true);
+ StorageVaultMgr mgr = new StorageVaultMgr(new SystemInfoService());
+ Cloud.StorageVaultPB.Builder request = (Cloud.StorageVaultPB.Builder) method.invoke(mgr, properties, "gcp");
+ Assertions.assertFalse(request.getObjInfo().hasCredential());
+ Assertions.assertFalse(request.getObjInfo().hasProvider());
+ }
+
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/storage/DefaultRemoteTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/storage/DefaultRemoteTest.java
index 14f708f639b..877fb0f316c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/cloud/storage/DefaultRemoteTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/storage/DefaultRemoteTest.java
@@ -93,6 +93,24 @@ public class DefaultRemoteTest {
Assertions.assertEquals("static-sk", credentials.secretAccessKey());
}
+ @Test
+ public void testNativeGcpCredentialRejectsAwsPresigning() {
+ Cloud.ObjectStoreInfoPB objInfo = Cloud.ObjectStoreInfoPB.newBuilder()
+ .setProvider(Cloud.ObjectStoreInfoPB.Provider.GCP)
+ .setBucket("gcs-bucket")
+ .setEndpoint("storage.googleapis.com")
+ .setRegion("us-east1")
+ .setCredential(Cloud.ObjectStoreCredentialPB.newBuilder()
+ .setGcpCredential(Cloud.GcpCredentialPB.newBuilder()
+ .setCredentialProviderType(Cloud.GcpCredentialPB.CredentialProviderType.DEFAULT)))
+ .build();
+
+ TestableS3Remote remote = new TestableS3Remote(new RemoteBase.ObjectInfo(objInfo), null);
+ UnsupportedOperationException exception = Assertions.assertThrows(
+ UnsupportedOperationException.class, () -> remote.getPresignedUrl("file"));
+ Assertions.assertTrue(exception.getMessage().contains("GCS V4 signing"));
+ }
+
private static class TestableS3Remote extends S3Remote {
private final StsClient stsClient;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
index 764196ae8b8..01891add94c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
@@ -25,6 +25,8 @@ import org.junit.Before;
import org.junit.Test;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.S3Client;
import java.net.URI;
@@ -84,6 +86,24 @@ public class S3UtilTest {
}
+ @Test
+ public void testBearerTokenClientsApplyConfiguredEndpointScheme() {
+ ExecutionInterceptor interceptor = new ExecutionInterceptor() {};
+ for (String scheme : Arrays.asList("https", "http")) {
+ Config.s3_client_http_scheme = scheme;
+ for (String endpoint : Arrays.asList("127.0.0.1:9000", "http://127.0.0.1:9000",
+ "https://127.0.0.1:9000")) {
+ URI expected = URI.create(endpoint.contains("://") ? endpoint : scheme + "://" + endpoint);
+ try (S3Client client = S3Util.buildBearerTokenS3Client(endpoint, "us-east1", true, interceptor);
+ S3AsyncClient asyncClient = S3Util.buildBearerTokenS3AsyncClient(
+ endpoint, "us-east1", true, interceptor)) {
+ Assert.assertEquals(expected, client.serviceClientConfiguration().endpointOverride().get());
+ Assert.assertEquals(expected, asyncClient.serviceClientConfiguration().endpointOverride().get());
+ }
+ }
+ }
+ }
+
@Test
public void testExtendGlobNumberRange_simpleRange() {
// Test simple range expansion {1..3}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStorePropertiesTest.java
index 6aa33c2c47d..46c91a0e609 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStorePropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStorePropertiesTest.java
@@ -17,11 +17,27 @@
package org.apache.doris.datasource.property.metastore;
+import org.apache.doris.datasource.iceberg.fileio.GcpS3FileIOAwsClientFactory;
+import org.apache.doris.datasource.property.storage.GCSProperties;
import org.apache.doris.datasource.property.storage.HdfsProperties;
+import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.s3.S3FileIO;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
+import org.apache.iceberg.io.FileIO;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
import java.util.Collections;
import java.util.HashMap;
@@ -30,6 +46,107 @@ import java.util.Map;
public class IcebergFileSystemMetaStorePropertiesTest {
+ @Test
+ public void testGcpNativeCredentialUsesS3FileIOClientFactory() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "hadoop");
+ props.put("warehouse", "gs://catalog-bucket/warehouse");
+ props.put("provider", "GCP");
+ props.put("gs.endpoint", "https://storage.googleapis.com");
+ props.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "COMPUTE_ENGINE");
+ props.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT,
+ "target@my-project.iam.gserviceaccount.com");
+ props.put("unrelated.catalog.secret", "must-not-reach-file-io");
+
+ IcebergFileSystemMetaStoreProperties icebergProperties =
+ (IcebergFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ List<StorageProperties> storagePropertiesList = StorageProperties.createAll(props);
+ Assertions.assertTrue(storagePropertiesList.stream().anyMatch(GCSProperties.class::isInstance));
+
+ Map<String, String> fileIOProperties = new HashMap<>();
+ icebergProperties.toFileIOProperties(storagePropertiesList, fileIOProperties, new Configuration());
+
+ // S3FileIO provides Iceberg's seekable streams and create/overwrite implementation. The
+ // custom factory only replaces client authentication with the native GCP bearer flow.
+ Assertions.assertEquals(S3FileIO.class.getName(),
+ fileIOProperties.get(CatalogProperties.FILE_IO_IMPL));
+ Assertions.assertEquals(GcpS3FileIOAwsClientFactory.class.getName(),
+ fileIOProperties.get(S3FileIOProperties.CLIENT_FACTORY));
+ Assertions.assertEquals("GCP", fileIOProperties.get("provider"));
+ Assertions.assertEquals("https://storage.googleapis.com", fileIOProperties.get(S3FileIOProperties.ENDPOINT));
+ Assertions.assertEquals("us-east1", fileIOProperties.get(AwsClientProperties.CLIENT_REGION));
+ Assertions.assertEquals("COMPUTE_ENGINE",
+ fileIOProperties.get(GCSProperties.CREDENTIAL_PROVIDER_TYPE));
+ Assertions.assertEquals("target@my-project.iam.gserviceaccount.com",
+ fileIOProperties.get(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT));
+ Assertions.assertFalse(fileIOProperties.containsKey("unrelated.catalog.secret"));
+
+ GcpS3FileIOAwsClientFactory clientFactory = new GcpS3FileIOAwsClientFactory();
+ clientFactory.initialize(fileIOProperties);
+ try (S3Client client = clientFactory.s3(); S3AsyncClient asyncClient = clientFactory.s3Async()) {
+ Assertions.assertNotNull(client);
+ Assertions.assertNotNull(asyncClient);
+ }
+
+ FileIO fileIO = new S3FileIO();
+ fileIO.initialize(fileIOProperties);
+ try {
+ Assertions.assertEquals("gs://catalog-bucket/warehouse/metadata/v1.metadata.json",
+ fileIO.newInputFile("gs://catalog-bucket/warehouse/metadata/v1.metadata.json").location());
+ Assertions.assertEquals("gs://catalog-bucket/warehouse/metadata/v2.metadata.json",
+ fileIO.newOutputFile("gs://catalog-bucket/warehouse/metadata/v2.metadata.json").location());
+ } finally {
+ fileIO.close();
+ }
+ }
+
+ @Test
+ public void testGcsAnonymousCredentialsReachIcebergS3Client() throws Exception {
+ for (String key : new String[] {GCSProperties.CREDENTIAL_PROVIDER_TYPE,
+ S3Properties.CREDENTIALS_PROVIDER_TYPE, S3Properties.Env.CREDENTIALS_PROVIDER_TYPE}) {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "hadoop");
+ props.put("warehouse", "gs://public-bucket/warehouse");
+ props.put(key, " anonymous ");
+ props.put(CatalogProperties.FILE_IO_IMPL, S3FileIO.class.getName());
+ IcebergFileSystemMetaStoreProperties icebergProperties =
+ (IcebergFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ Map<String, String> options = new HashMap<>(props);
+ icebergProperties.toFileIOProperties(StorageProperties.createAll(props), options, new Configuration());
+
+ // Exercise Iceberg's actual credential binding, not just the emitted property string.
+ S3ClientBuilder builder = Mockito.mock(S3ClientBuilder.class);
+ new S3FileIOProperties(options).applyCredentialConfigurations(new AwsClientProperties(options), builder);
+ ArgumentCaptor<AwsCredentialsProvider> credentials = ArgumentCaptor.forClass(AwsCredentialsProvider.class);
+ Mockito.verify(builder).credentialsProvider(credentials.capture());
+ Assertions.assertInstanceOf(AnonymousCredentialsProvider.class, credentials.getValue(), key);
+ Assertions.assertEquals(S3FileIO.class.getName(), options.get(CatalogProperties.FILE_IO_IMPL));
+ Assertions.assertFalse(options.containsKey(S3FileIOProperties.CLIENT_FACTORY));
+ }
+ }
+
+ @Test
+ public void testGcsHmacDoesNotOverrideFileIO() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "hadoop");
+ props.put("warehouse", "gs://catalog-bucket/warehouse");
+ props.put("provider", "GCP");
+ props.put("gs.access_key", "access-key");
+ props.put("gs.secret_key", "secret-key");
+
+ IcebergFileSystemMetaStoreProperties icebergProperties =
+ (IcebergFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ Map<String, String> fileIOProperties = new HashMap<>();
+ icebergProperties.toFileIOProperties(StorageProperties.createAll(props), fileIOProperties,
+ new Configuration());
+
+ Assertions.assertFalse(fileIOProperties.containsKey(CatalogProperties.FILE_IO_IMPL));
+ Assertions.assertFalse(fileIOProperties.containsKey(S3FileIOProperties.CLIENT_FACTORY));
+ }
+
@Test
public void testKerberosCatalog() throws Exception {
Map<String, String> props = new HashMap<>();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/GCSPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/GCSPropertiesTest.java
index 46a9424fe79..82f542e80e7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/GCSPropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/GCSPropertiesTest.java
@@ -17,15 +17,36 @@
package org.apache.doris.datasource.property.storage;
+import org.apache.doris.cloud.proto.Cloud.CredProviderTypePB;
+import org.apache.doris.cloud.proto.Cloud.GcpCredentialPB;
+import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB;
+import org.apache.doris.common.Config;
+import org.apache.doris.datasource.property.storage.auth.GcpCredential;
+import org.apache.doris.datasource.property.storage.auth.GcpCredentialProviderType;
+import org.apache.doris.fs.obj.S3ObjStorage;
+import org.apache.doris.thrift.TCredProviderType;
+import org.apache.doris.thrift.TGcpCredentialProviderType;
+import org.apache.doris.thrift.TObjStorageType;
+import org.apache.doris.thrift.TS3StorageParam;
+
+import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem;
import com.google.common.collect.Maps;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.fs.s3a.Constants;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.services.s3.S3Client;
+import java.net.URI;
import java.util.HashMap;
+import java.util.Locale;
import java.util.Map;
+import java.util.UUID;
public class GCSPropertiesTest {
@@ -50,6 +71,33 @@ public class GCSPropertiesTest {
Assertions.assertTrue(GCSProperties.guessIsMe(origProps));
}
+ @Test
+ public void testGuessIsMeByNativeAuthenticationAndLocation() {
+ origProps.put(StorageProperties.FS_PROVIDER_KEY, "gcp");
+ origProps.put(S3Properties.REGION, "us-east1");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "COMPUTE_ENGINE");
+ Assertions.assertTrue(GCSProperties.guessIsMe(origProps));
+ Assertions.assertInstanceOf(GCSProperties.class, StorageProperties.createPrimary(origProps));
+
+ origProps.clear();
+ origProps.put(S3Properties.REGION, "us-east1");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ Assertions.assertFalse(S3Properties.guessIsMe(origProps));
+ Assertions.assertInstanceOf(GCSProperties.class, StorageProperties.createPrimary(origProps));
+
+ origProps.clear();
+ origProps.put("uri", "gs://load-and-tvf-bucket/path/file.parquet");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ Assertions.assertTrue(GCSProperties.guessIsMe(origProps));
+ Assertions.assertInstanceOf(GCSProperties.class, StorageProperties.createPrimary(origProps));
+
+ origProps.clear();
+ origProps.put("warehouse", "GS://external-catalog-bucket/warehouse");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ Assertions.assertTrue(GCSProperties.guessIsMe(origProps));
+ Assertions.assertInstanceOf(GCSProperties.class, StorageProperties.createPrimary(origProps));
+ }
+
@Test
public void testDefaultValues() {
origProps.put("gs.endpoint", "https://storage.googleapis.com");
@@ -104,8 +152,10 @@ public class GCSPropertiesTest {
public void testGCSAwsCredentialsProvider() throws Exception {
Map<String, String> gcsProps = new HashMap<>();
gcsProps.put("fs.gcs.support", "true");
+ gcsProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "ANONYMOUS");
GCSProperties gcsStorageProperties = (GCSProperties) StorageProperties.createPrimary(gcsProps);
Assertions.assertEquals(AnonymousCredentialsProvider.class, gcsStorageProperties.getAwsCredentialsProvider().getClass());
+ gcsProps.remove(GCSProperties.CREDENTIAL_PROVIDER_TYPE);
gcsProps.put("gs.access_key", "myAccessKey");
gcsProps.put("gs.secret_key", "mySecretKey");
gcsStorageProperties = (GCSProperties) StorageProperties.createPrimary(gcsProps);
@@ -129,4 +179,706 @@ public class GCSPropertiesTest {
s3Properties = (GCSProperties) StorageProperties.createPrimary(props);
Assertions.assertFalse(s3Properties.hadoopStorageConfig.getBoolean("fs.gs.impl.disable.cache", false));
}
+
+ @Test
+ public void testNativeHadoopAuthenticationConfiguration() throws Exception {
+ for (String provider : new String[] {"default", "compute_engine"}) {
+ for (String account : new String[] {"", "target@test-project.iam.gserviceaccount.com"}) {
+ Map<String, String> props = Maps.newHashMap();
+ props.put("provider", "GCP");
+ props.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, provider);
+ props.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT, account);
+ GCSProperties storage = (GCSProperties) StorageProperties.createPrimary(props);
+ Configuration conf = storage.hadoopStorageConfig;
+ Assertions.assertEquals(GoogleHadoopFileSystem.class.getName(), conf.get("fs.gs.impl"));
+ Assertions.assertEquals("default".equals(provider) ? "APPLICATION_DEFAULT" : "COMPUTE_ENGINE",
+ conf.get("fs.gs.auth.type"));
+ Assertions.assertEquals(account, conf.get("fs.gs.auth.impersonation.service.account"));
+ // Resolve and initialize the real connector without requesting live GCP credentials.
+ conf.setBoolean("fs.gs.lazy.init.enable", true);
+ try (FileSystem fs = FileSystem.newInstance(URI.create("gs://test-bucket/warehouse"), conf)) {
+ Assertions.assertInstanceOf(GoogleHadoopFileSystem.class, fs);
+ Assertions.assertEquals("gs", fs.getScheme());
+ }
+ Assertions.assertEquals(storage.getFsCacheFingerprint(), conf.get(StorageProperties.FS_CACHE_KEY_PROPERTY));
+ }
+ }
+ }
+
+ @Test
+ public void testHmacHadoopAuthenticationIsUnchanged() {
+ origProps.put(StorageProperties.FS_GCS_SUPPORT, "true");
+ origProps.put("gs.access_key", "test-ak");
+ origProps.put("gs.secret_key", "test-sk");
+ GCSProperties storage = (GCSProperties) StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("org.apache.hadoop.fs.s3a.S3AFileSystem",
+ storage.hadoopStorageConfig.get("fs.gs.impl"));
+ Assertions.assertEquals("org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider",
+ storage.hadoopStorageConfig.get(Constants.AWS_CREDENTIALS_PROVIDER));
+ }
+
+ @Test
+ public void testNativeGcpDefaultCredentials() {
+ origProps.put("fs.gcs.support", "true");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "default");
+
+ GCSProperties properties = (GCSProperties) StorageProperties.createPrimary(origProps);
+ Assertions.assertTrue(properties.hasGcpCredential());
+ Assertions.assertEquals(GcpCredentialProviderType.DEFAULT,
+ properties.getGcpCredentialProviderType());
+ Assertions.assertEquals("DEFAULT", properties.getBackendConfigProperties()
+ .get(GCSProperties.CREDENTIAL_PROVIDER_TYPE));
+ Assertions.assertFalse(properties.getBackendConfigProperties()
+ .containsKey("AWS_CREDENTIALS_PROVIDER_TYPE"));
+ }
+
+ @Test
+ public void testNativeGcpCredentialProviderTypeNormalization() {
+ Map<String, GcpCredentialProviderType> cases = new HashMap<>();
+ cases.put("DEFAULT", GcpCredentialProviderType.DEFAULT);
+ cases.put("default", GcpCredentialProviderType.DEFAULT);
+ cases.put("DeFaUlT", GcpCredentialProviderType.DEFAULT);
+ cases.put(" COMPUTE_ENGINE ", GcpCredentialProviderType.COMPUTE_ENGINE);
+
+ cases.forEach((value, expected) -> {
+ origProps.clear();
+ origProps.put(StorageProperties.FS_PROVIDER_KEY, "GCP");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, value);
+
+ GCSProperties properties = (GCSProperties) StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals(expected, properties.getGcpCredentialProviderType(), value);
+ Assertions.assertEquals(expected.name(), properties.getBackendConfigProperties()
+ .get(GCSProperties.CREDENTIAL_PROVIDER_TYPE), value);
+ });
+ }
+
+ @Test
+ public void testNativeGcpRejectsInvalidCredentialProviderTypes() {
+ for (String value : new String[] {"ADC", "", "unknown"}) {
+ origProps.clear();
+ origProps.put(StorageProperties.FS_PROVIDER_KEY, "GCP");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, value);
+
+ IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class,
+ () -> StorageProperties.createPrimary(origProps), value);
+ Assertions.assertTrue(exception.getMessage().contains(GCSProperties.CREDENTIAL_PROVIDER_TYPE), value);
+ Assertions.assertTrue(exception.getMessage().contains("DEFAULT"), value);
+ Assertions.assertTrue(exception.getMessage().contains("COMPUTE_ENGINE"), value);
+ }
+ }
+
+ @Test
+ public void testNativeGcpDefaultsImpersonationToDefault() {
+ origProps.put(StorageProperties.FS_PROVIDER_KEY, "GCP");
+ origProps.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT,
+ "target@my-project.iam.gserviceaccount.com");
+
+ GCSProperties properties = (GCSProperties) StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals(GcpCredentialProviderType.DEFAULT,
+ properties.getGcpCredentialProviderType());
+ ObjectStoreInfoPB obj = S3Properties.getObjStoreInfoPB(origProps).build();
+ Assertions.assertTrue(obj.getCredential().getGcpCredential().hasCredentialProviderType());
+ Assertions.assertEquals(GcpCredentialPB.CredentialProviderType.DEFAULT,
+ obj.getCredential().getGcpCredential().getCredentialProviderType());
+ Assertions.assertEquals(TGcpCredentialProviderType.DEFAULT,
+ S3Properties.getS3TStorageParam(origProps).getCredential()
+ .getGcpCredential().getCredentialProviderType());
+ }
+
+ @Test
+ public void testNativeGcpEmptyTargetDefaultsToDefault() {
+ origProps.put(StorageProperties.FS_PROVIDER_KEY, "GCP");
+ Assertions.assertTrue(S3Properties.getObjStoreInfoPB(origProps).hasCredential());
+ origProps.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT, "");
+ GCSProperties properties = (GCSProperties) StorageProperties.createPrimary(origProps);
+ Assertions.assertTrue(properties.hasGcpCredential());
+ Assertions.assertEquals(GcpCredentialProviderType.DEFAULT, properties.getGcpCredentialProviderType());
+ Assertions.assertEquals("", properties.getGcpCredential().getImpersonationServiceAccount());
+ Assertions.assertEquals(TGcpCredentialProviderType.DEFAULT,
+ S3Properties.getS3TStorageParam(origProps).getCredential()
+ .getGcpCredential().getCredentialProviderType());
+ origProps.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT, " ");
+ Assertions.assertThrows(IllegalArgumentException.class, () -> StorageProperties.createPrimary(origProps));
+ }
+
+ @Test
+ public void testNativeGcpComputeEngineAndImpersonation() {
+ origProps.put("fs.gcs.support", "true");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "COMPUTE_ENGINE");
+ origProps.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT,
+ "target@my-project.iam.gserviceaccount.com");
+
+ GCSProperties properties = (GCSProperties) StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals(GcpCredentialProviderType.COMPUTE_ENGINE,
+ properties.getGcpCredentialProviderType());
+ Assertions.assertEquals("target@my-project.iam.gserviceaccount.com",
+ properties.getBackendConfigProperties()
+ .get(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT));
+ }
+
+ @Test
+ public void testNativeGcpDefaultAndImpersonation() {
+ origProps.put("fs.gcs.support", "true");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ origProps.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT,
+ "target@my-project.iam.gserviceaccount.com");
+
+ GCSProperties properties = (GCSProperties) StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals(GcpCredentialProviderType.DEFAULT,
+ properties.getGcpCredentialProviderType());
+ Assertions.assertEquals("target@my-project.iam.gserviceaccount.com",
+ properties.getBackendConfigProperties()
+ .get(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT));
+ }
+
+ @Test
+ public void testNativeGcpDefaultServiceAccountEmails() {
+ origProps.put("fs.gcs.support", "true");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "COMPUTE_ENGINE");
+
+ origProps.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT,
+ "123456789-compute@developer.gserviceaccount.com");
+ Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps));
+
+ origProps.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT,
+ "my-project@appspot.gserviceaccount.com");
+ Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps));
+ }
+
+ @Test
+ public void testNativeGcpComputeEngineCredentials() {
+ origProps.put("fs.gcs.support", "true");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "COMPUTE_ENGINE");
+
+ GCSProperties properties = (GCSProperties) StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals(GcpCredentialProviderType.COMPUTE_ENGINE,
+ properties.getGcpCredentialProviderType());
+ Assertions.assertEquals("COMPUTE_ENGINE", properties.getBackendConfigProperties()
+ .get(GCSProperties.CREDENTIAL_PROVIDER_TYPE));
+ Assertions.assertFalse(properties.getBackendConfigProperties()
+ .containsKey(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT));
+ }
+
+ @Test
+ public void testNativeGcpCloudAndThriftMapping() {
+ origProps.put(S3Properties.ENDPOINT, "https://storage.googleapis.com");
+ origProps.put(S3Properties.REGION, "us-east1");
+ origProps.put(S3Properties.BUCKET, "bucket");
+ origProps.put(S3Properties.ROOT_PATH, "root");
+ origProps.put(S3Properties.ACCESS_KEY, "");
+ origProps.put(S3Properties.SECRET_KEY, "");
+ origProps.put(StorageProperties.FS_PROVIDER_KEY, "GCP");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "COMPUTE_ENGINE");
+ origProps.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT,
+ "target@my-project.iam.gserviceaccount.com");
+
+ ObjectStoreInfoPB objectStoreInfo = S3Properties.getObjStoreInfoPB(origProps).build();
+ Assertions.assertTrue(objectStoreInfo.hasCredential());
+ Assertions.assertTrue(objectStoreInfo.getCredential().hasGcpCredential());
+ Assertions.assertEquals(GcpCredentialPB.CredentialProviderType.COMPUTE_ENGINE,
+ objectStoreInfo.getCredential().getGcpCredential().getCredentialProviderType());
+ Assertions.assertEquals("target@my-project.iam.gserviceaccount.com",
+ objectStoreInfo.getCredential().getGcpCredential().getImpersonationServiceAccount());
+ TS3StorageParam thrift = S3Properties.getS3TStorageParam(origProps);
+ Assertions.assertEquals(TObjStorageType.GCP, thrift.getProvider());
+ Assertions.assertTrue(thrift.getCredential().isSetGcpCredential());
+ Assertions.assertEquals(TGcpCredentialProviderType.COMPUTE_ENGINE,
+ thrift.getCredential().getGcpCredential().getCredentialProviderType());
+ Assertions.assertEquals("target@my-project.iam.gserviceaccount.com",
+ thrift.getCredential().getGcpCredential().getImpersonationServiceAccount());
+
+ origProps.remove(GCSProperties.CREDENTIAL_PROVIDER_TYPE);
+ origProps.remove(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT);
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE.toUpperCase(), "default");
+ origProps.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT.toUpperCase(),
+ "uppercase@my-project.iam.gserviceaccount.com");
+ objectStoreInfo = S3Properties.getObjStoreInfoPB(origProps).build();
+ Assertions.assertEquals(GcpCredentialPB.CredentialProviderType.DEFAULT,
+ objectStoreInfo.getCredential().getGcpCredential().getCredentialProviderType());
+ Assertions.assertEquals("uppercase@my-project.iam.gserviceaccount.com",
+ objectStoreInfo.getCredential().getGcpCredential().getImpersonationServiceAccount());
+ }
+
+ @Test
+ public void testNativeGcpValidation() {
+ origProps.put("fs.gcs.support", "true");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ origProps.put("gs.access_key", "ak");
+ origProps.put("gs.secret_key", "sk");
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> StorageProperties.createPrimary(origProps));
+
+ origProps.clear();
+ origProps.put("fs.gcs.support", "true");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ origProps.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT,
+ "target@my-project.iam.gserviceaccount.com");
+ Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps));
+ }
+
+ @Test
+ public void testNativeGcpRejectsAwsAuthenticationAndProviderConflicts() {
+ String[] conflictingKeys = {
+ S3Properties.ACCESS_KEY,
+ S3Properties.SECRET_KEY,
+ S3Properties.SESSION_TOKEN,
+ S3Properties.Env.ACCESS_KEY,
+ S3Properties.Env.SECRET_KEY,
+ S3Properties.Env.TOKEN,
+ S3Properties.ROLE_ARN,
+ S3Properties.EXTERNAL_ID,
+ S3Properties.CREDENTIALS_PROVIDER_TYPE,
+ S3Properties.Env.ROLE_ARN,
+ S3Properties.Env.EXTERNAL_ID,
+ S3Properties.Env.CREDENTIALS_PROVIDER_TYPE
+ };
+ for (String key : conflictingKeys) {
+ origProps.clear();
+ origProps.put("fs.gcs.support", "true");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ origProps.put(key, "value");
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> StorageProperties.createPrimary(origProps), key);
+ }
+
+ for (String provider : new String[] {"S3", "AWS"}) {
+ origProps.clear();
+ origProps.put(S3Properties.REGION, "us-east1");
+ origProps.put(StorageProperties.FS_PROVIDER_KEY, provider);
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> StorageProperties.createPrimary(origProps), provider);
+ }
+
+ origProps.clear();
+ origProps.put(StorageProperties.FS_S3_SUPPORT, "true");
+ origProps.put(S3Properties.REGION, "us-east-1");
+ origProps.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> StorageProperties.createPrimary(origProps));
+ }
+
+ @Test
+ public void testAwsCredentialProviderTypeDoesNotSelectGcp() {
+ origProps.put(S3Properties.ENDPOINT, "s3.us-west-2.amazonaws.com");
+ origProps.put(S3Properties.REGION, "us-west-2");
+ origProps.put(S3Properties.CREDENTIALS_PROVIDER_TYPE, "INSTANCE_PROFILE");
+
+ Assertions.assertFalse(GCSProperties.guessIsMe(origProps));
+ Assertions.assertInstanceOf(S3Properties.class, StorageProperties.createPrimary(origProps));
+ }
+
+ @Test
+ public void testGsPrefixedConnectionProperties() {
+ origProps.put("provider", "gcp");
+ origProps.put("gs.endpoint", "https://custom.example.com");
+ origProps.put("gs.access_key", "access");
+ origProps.put("gs.secret_key", "secret");
+ origProps.put("gs.session_token", "token");
+ origProps.put("gs.connection.maximum", "123");
+ origProps.put("gs.connection.request.timeout", "456");
+ origProps.put("gs.connection.timeout", "789");
+ origProps.put("gs.use_path_style", "true");
+ origProps.put("gs.force_parsing_by_standard_uri", "true");
+ GCSProperties properties = (GCSProperties) StorageProperties.createPrimary(origProps);
+ Map<String, String> backend = properties.getBackendConfigProperties();
+ Assertions.assertEquals("https://custom.example.com", backend.get("AWS_ENDPOINT"));
+ Assertions.assertEquals("access", backend.get("AWS_ACCESS_KEY"));
+ Assertions.assertEquals("secret", backend.get("AWS_SECRET_KEY"));
+ Assertions.assertEquals("token", backend.get("AWS_TOKEN"));
+ Assertions.assertEquals("123", backend.get("AWS_MAX_CONNECTIONS"));
+ Assertions.assertEquals("456", backend.get("AWS_REQUEST_TIMEOUT_MS"));
+ Assertions.assertEquals("789", backend.get("AWS_CONNECTION_TIMEOUT_MS"));
+ Assertions.assertEquals("true", backend.get("use_path_style"));
+ Assertions.assertEquals("true", properties.getForceParsingByStandardUrl());
+
+ origProps.remove("provider");
+ Assertions.assertInstanceOf(GCSProperties.class, StorageProperties.createPrimary(origProps));
+ }
+
+ @Test
+ public void testGsPrefixPrecedenceAndLegacyAliases() {
+ origProps.put("s3.endpoint", "https://storage.googleapis.com");
+ origProps.put("gs.credential_provider_type", "DEFAULT");
+ GCSProperties legacy = (GCSProperties) StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals(GcpCredentialProviderType.DEFAULT, legacy.getGcpCredentialProviderType());
+
+ origProps.put("gs.endpoint", "https://custom.example.com");
+ origProps.put("gs.credential_provider_type", "COMPUTE_ENGINE");
+ origProps.put("gs.impersonation_service_account", "target@project.iam.gserviceaccount.com");
+ GCSProperties properties = (GCSProperties) StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("https://custom.example.com", properties.getEndpoint());
+ Assertions.assertEquals(GcpCredentialProviderType.COMPUTE_ENGINE, properties.getGcpCredentialProviderType());
+ Map<String, String> backend = properties.getBackendConfigProperties();
+ Assertions.assertEquals("COMPUTE_ENGINE", backend.get("gs.credential_provider_type"));
+ Assertions.assertEquals("target@project.iam.gserviceaccount.com",
+ backend.get("gs.impersonation_service_account"));
+
+ origProps.put("gs.credential_provider_type", "");
+ Assertions.assertThrows(IllegalArgumentException.class, () -> StorageProperties.createPrimary(origProps));
+ }
+
+ @Test
+ public void testGcpCredentialConflictsBeforeAliasConversion() {
+ for (String key : new String[] {"gs.access_key", "gs.secret_key", "gs.session_token",
+ "s3.access_key", "s3.secret_key", "s3.session_token"}) {
+ origProps.clear();
+ origProps.put("provider", "GCP");
+ origProps.put("gs.credential_provider_type", "DEFAULT");
+ origProps.put(key, "conflicting-credential");
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> StorageProperties.createPrimary(origProps), key);
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> S3Properties.getObjStoreInfoPB(origProps), key);
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> S3Properties.getS3TStorageParam(origProps), key);
+ }
+ }
+
+ @Test
+ public void testOmittedAuthenticationUsesAdcAcrossStorageSelectors() {
+ Map<String, String> selectors = new HashMap<>();
+ selectors.put("provider", "GCP");
+ selectors.put("fs.gcs.support", "true");
+ selectors.put("gs.endpoint", "https://storage.googleapis.com");
+ selectors.put("s3.endpoint", "https://storage.googleapis.com");
+ selectors.put("uri", "gs://test-bucket/data.parquet");
+ selectors.put("warehouse", "gs://test-bucket/warehouse");
+ selectors.forEach((key, value) -> {
+ Map<String, String> props = new HashMap<>();
+ props.put(key, value);
+ GCSProperties storage = (GCSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertTrue(storage.hasGcpCredential(), key);
+ Assertions.assertEquals(GcpCredentialProviderType.DEFAULT, storage.getGcpCredentialProviderType(), key);
+ Assertions.assertEquals("DEFAULT", storage.getCredentialProviderType(), key);
+ Assertions.assertEquals("", storage.getGcpCredential().getImpersonationServiceAccount(), key);
+ Assertions.assertEquals("APPLICATION_DEFAULT", storage.hadoopStorageConfig.get("fs.gs.auth.type"), key);
+ Assertions.assertEquals(GoogleHadoopFileSystem.class.getName(),
+ storage.hadoopStorageConfig.get("fs.gs.impl"), key);
+ Map<String, String> backend = storage.getBackendConfigProperties();
+ Assertions.assertEquals("DEFAULT", backend.get(GCSProperties.CREDENTIAL_PROVIDER_TYPE), key);
+ Assertions.assertFalse(backend.containsKey("AWS_CREDENTIALS_PROVIDER_TYPE"), key);
+ ObjectStoreInfoPB objectStoreInfo = S3Properties.getObjStoreInfoPB(props).build();
+ Assertions.assertEquals(ObjectStoreInfoPB.Provider.GCP, objectStoreInfo.getProvider(), key);
+ Assertions.assertEquals(GcpCredentialPB.CredentialProviderType.DEFAULT,
+ objectStoreInfo.getCredential().getGcpCredential().getCredentialProviderType(), key);
+ TS3StorageParam thrift = S3Properties.getS3TStorageParam(props);
+ Assertions.assertEquals(TObjStorageType.GCP, thrift.getProvider(), key);
+ Assertions.assertTrue(thrift.isSetCredential(), key);
+ Assertions.assertEquals(TGcpCredentialProviderType.DEFAULT,
+ thrift.getCredential().getGcpCredential().getCredentialProviderType(), key);
+ });
+ }
+
+ @Test
+ public void testExplicitAnonymousUsesUnsignedFeHadoopAndBackendClients() {
+ for (String key : new String[] {GCSProperties.CREDENTIAL_PROVIDER_TYPE,
+ GCSProperties.CREDENTIAL_PROVIDER_TYPE.toUpperCase(),
+ S3Properties.CREDENTIALS_PROVIDER_TYPE, S3Properties.Env.CREDENTIALS_PROVIDER_TYPE}) {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "GCP");
+ props.put(key, " anonymous ");
+ GCSProperties storage = (GCSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertFalse(storage.hasGcpCredential(), key);
+ Assertions.assertEquals("ANONYMOUS", storage.getCredentialProviderType(), key);
+ Assertions.assertInstanceOf(AnonymousCredentialsProvider.class, storage.getAwsCredentialsProvider(), key);
+ Assertions.assertEquals("org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider",
+ storage.hadoopStorageConfig.get(Constants.AWS_CREDENTIALS_PROVIDER), key);
+ Map<String, String> backend = storage.getBackendConfigProperties();
+ Assertions.assertEquals("ANONYMOUS", backend.get("AWS_CREDENTIALS_PROVIDER_TYPE"), key);
+ Assertions.assertFalse(backend.containsKey(GCSProperties.CREDENTIAL_PROVIDER_TYPE), key);
+ TS3StorageParam thrift = S3Properties.getS3TStorageParam(props);
+ Assertions.assertFalse(thrift.isSetCredential(), key);
+ Assertions.assertEquals(TCredProviderType.ANONYMOUS, thrift.getCredProviderType(), key);
+ // Building the real client must not try to discover ADC or contact the metadata server.
+ try (software.amazon.awssdk.services.s3.S3Client client = storage.createS3Client()) {
+ Assertions.assertNotNull(client);
+ }
+ }
+ }
+
+ @Test
+ public void testGcsClientsApplyConfiguredEndpointScheme() throws Exception {
+ String originalScheme = Config.s3_client_http_scheme;
+ try {
+ Config.s3_client_http_scheme = "https";
+ for (String providerType : new String[] {"COMPUTE_ENGINE", "ANONYMOUS"}) {
+ for (String endpoint : new String[] {"127.0.0.1:9000", "http://127.0.0.1:9000"}) {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "GCP");
+ props.put(GCSProperties.ENDPOINT, endpoint);
+ props.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, providerType);
+ GCSProperties storage = (GCSProperties) StorageProperties.createPrimary(props);
+ URI expected = URI.create(endpoint.startsWith("http://") ? endpoint : "https://" + endpoint);
+ try (S3Client client = storage.createS3Client();
+ S3ObjStorage objStorage = new S3ObjStorage(storage)) {
+ Assertions.assertEquals(expected, client.serviceClientConfiguration().endpointOverride().get());
+ Assertions.assertEquals(expected,
+ objStorage.getClient().serviceClientConfiguration().endpointOverride().get());
+ }
+ }
+ }
+ } finally {
+ Config.s3_client_http_scheme = originalScheme;
+ }
+ }
+
+ @Test
+ public void testAnonymousAliasesDoNotReuseAdcFileSystem() throws Exception {
+ Map<String, String> base = new HashMap<>();
+ base.put("provider", "GCP");
+ GCSProperties adc = (GCSProperties) StorageProperties.createPrimary(base);
+ String anonymousFingerprint = null;
+ for (String key : new String[] {GCSProperties.CREDENTIAL_PROVIDER_TYPE,
+ GCSProperties.CREDENTIAL_PROVIDER_TYPE.toUpperCase(Locale.ROOT),
+ S3Properties.CREDENTIALS_PROVIDER_TYPE, S3Properties.Env.CREDENTIALS_PROVIDER_TYPE}) {
+ Map<String, String> props = new HashMap<>(base);
+ props.put(key, " anonymous ");
+ GCSProperties anonymous = (GCSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertNotEquals(adc.getFsCacheFingerprint(), anonymous.getFsCacheFingerprint(), key);
+ if (anonymousFingerprint == null) {
+ anonymousFingerprint = anonymous.getFsCacheFingerprint();
+ }
+ Assertions.assertEquals(anonymousFingerprint, anonymous.getFsCacheFingerprint(), key);
+ Assertions.assertEquals(anonymousFingerprint,
+ anonymous.getBackendConfigProperties().get(StorageProperties.FS_CACHE_KEY_PROPERTY), key);
+
+ for (boolean anonymousFirst : new boolean[] {false, true}) {
+ Configuration firstConf = cacheOnlyConfiguration(anonymousFirst ? anonymous : adc);
+ Configuration secondConf = cacheOnlyConfiguration(anonymousFirst ? adc : anonymous);
+ URI uri = URI.create("gs://cache-test-" + UUID.randomUUID() + "/warehouse");
+ try (FileSystem first = FileSystem.get(uri, firstConf);
+ FileSystem second = FileSystem.get(uri, secondConf)) {
+ Assertions.assertNotSame(first, second, key);
+ Assertions.assertSame(first, FileSystem.get(uri, firstConf), key);
+ Assertions.assertSame(second, FileSystem.get(uri, secondConf), key);
+ }
+ }
+ }
+ }
+
+ private Configuration cacheOnlyConfiguration(GCSProperties storage) {
+ // Exercise the real Hadoop cache with the emitted credential key, without GCS I/O or ADC discovery.
+ Configuration conf = new Configuration(false);
+ conf.set("fs.gs.impl", RawLocalFileSystem.class.getName());
+ conf.set(StorageProperties.FS_CACHE_KEY_PROPERTY,
+ storage.hadoopStorageConfig.get(StorageProperties.FS_CACHE_KEY_PROPERTY));
+ return conf;
+ }
+
+ @Test
+ public void testNativeCredentialCacheIdentityUsesResolvedProperties() {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "GCP");
+ String adcFingerprint = StorageProperties.createPrimary(props).getFsCacheFingerprint();
+ for (String key : new String[] {GCSProperties.CREDENTIAL_PROVIDER_TYPE,
+ GCSProperties.CREDENTIAL_PROVIDER_TYPE.toUpperCase(Locale.ROOT)}) {
+ props.put(key, " default ");
+ Assertions.assertEquals(adcFingerprint, StorageProperties.createPrimary(props).getFsCacheFingerprint());
+ props.put(key, " compute_engine ");
+ Assertions.assertNotEquals(adcFingerprint, StorageProperties.createPrimary(props).getFsCacheFingerprint());
+ props.remove(key);
+ }
+
+ String targetFingerprint = null;
+ for (String key : new String[] {GCSProperties.IMPERSONATION_SERVICE_ACCOUNT,
+ GCSProperties.IMPERSONATION_SERVICE_ACCOUNT.toUpperCase(Locale.ROOT)}) {
+ props.put(key, "target@test-project.iam.gserviceaccount.com");
+ String fingerprint = StorageProperties.createPrimary(props).getFsCacheFingerprint();
+ Assertions.assertNotEquals(adcFingerprint, fingerprint);
+ if (targetFingerprint == null) {
+ targetFingerprint = fingerprint;
+ }
+ Assertions.assertEquals(targetFingerprint, fingerprint);
+ props.put(key, "other@test-project.iam.gserviceaccount.com");
+ Assertions.assertNotEquals(targetFingerprint, StorageProperties.createPrimary(props).getFsCacheFingerprint());
+ props.remove(key);
+ }
+ }
+
+ @Test
+ public void testAnonymousRejectsConflictingCredentials() {
+ for (String key : new String[] {"gs.access_key", "gs.secret_key", "gs.session_token",
+ GCSProperties.IMPERSONATION_SERVICE_ACCOUNT, S3Properties.ROLE_ARN}) {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "GCP");
+ props.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "ANONYMOUS");
+ props.put(key, "target@test.iam.gserviceaccount.com");
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> StorageProperties.createPrimary(props), key);
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> S3Properties.getS3TStorageParam(props), key);
+ }
+ }
+
+ @Test
+ public void testAnonymousVaultRemainsUnsupported() {
+ Map<String, String> selectors = new HashMap<>();
+ selectors.put("provider", "GCP");
+ selectors.put("fs.gcs.support", "true");
+ selectors.put("gs.endpoint", "https://storage.googleapis.com");
+ selectors.put("s3.endpoint", "https://storage.googleapis.com");
+ selectors.put("uri", "gs://test-bucket/data.parquet");
+ selectors.put("warehouse", "gs://test-bucket/warehouse");
+ selectors.forEach((selector, value) -> {
+ for (String key : new String[] {GCSProperties.CREDENTIAL_PROVIDER_TYPE,
+ GCSProperties.CREDENTIAL_PROVIDER_TYPE.toUpperCase(Locale.ROOT),
+ S3Properties.CREDENTIALS_PROVIDER_TYPE, S3Properties.Env.CREDENTIALS_PROVIDER_TYPE}) {
+ Map<String, String> props = new HashMap<>();
+ props.put(selector, value);
+ props.put(key, " anonymous ");
+ IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class,
+ () -> S3Properties.getObjStoreInfoPB(props), selector + ": " + key);
+ Assertions.assertTrue(error.getMessage().contains("storage vaults"));
+ // Vault validation must not reject the same anonymous configuration for FE/BE reads.
+ GCSProperties storage = (GCSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertFalse(storage.hasGcpCredential());
+ Assertions.assertEquals(TCredProviderType.ANONYMOUS,
+ S3Properties.getS3TStorageParam(props).getCredProviderType());
+ }
+ });
+ }
+
+ @Test
+ public void testGcpVaultAnonymousValidationDoesNotChangeAwsMapping() {
+ for (String key : new String[] {S3Properties.CREDENTIALS_PROVIDER_TYPE,
+ S3Properties.Env.CREDENTIALS_PROVIDER_TYPE}) {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "S3");
+ props.put(key, "ANONYMOUS");
+ Assertions.assertEquals(CredProviderTypePB.ANONYMOUS,
+ S3Properties.getObjStoreInfoPB(props).getCredProviderType());
+ Assertions.assertEquals(TCredProviderType.ANONYMOUS,
+ S3Properties.getS3TStorageParam(props).getCredProviderType());
+ }
+ }
+
+ @Test
+ public void testDefaultGcpCredentialsDoNotOverrideHmacAliasesOrOtherProviders() {
+ for (String[] keys : new String[][] {{"gs.access_key", "gs.secret_key"},
+ {"s3.access_key", "s3.secret_key"}, {"AWS_ACCESS_KEY", "AWS_SECRET_KEY"},
+ {"access_key", "secret_key"}, {"ACCESS_KEY", "SECRET_KEY"}}) {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "GCP");
+ props.put(keys[0], "hmac-ak");
+ props.put(keys[1], "hmac-sk");
+ GCSProperties storage = (GCSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertFalse(storage.hasGcpCredential(), keys[0]);
+ Assertions.assertEquals("", storage.getCredentialProviderType(), keys[0]);
+ Assertions.assertInstanceOf(StaticCredentialsProvider.class, storage.getAwsCredentialsProvider());
+ Assertions.assertFalse(S3Properties.getObjStoreInfoPB(props).hasCredential());
+ Assertions.assertFalse(S3Properties.getS3TStorageParam(props).isSetCredential());
+ }
+ for (String provider : new String[] {"S3", "AZURE", "OSS"}) {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", provider);
+ Assertions.assertFalse(S3Properties.getObjStoreInfoPB(props).hasCredential(), provider);
+ Assertions.assertFalse(S3Properties.getS3TStorageParam(props).isSetCredential(), provider);
+ }
+ Assertions.assertFalse(S3Properties.getObjStoreInfoPB(new HashMap<>()).hasCredential());
+ Assertions.assertFalse(S3Properties.getS3TStorageParam(new HashMap<>()).isSetCredential());
+ }
+
+ @Test
+ public void testLegacyS3ProviderPreservesGcsHmacAndAnonymousAccess() {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "S3");
+ props.put("s3.endpoint", "https://storage.googleapis.com");
+ props.put("s3.access_key", "hmac-ak");
+ props.put("s3.secret_key", "hmac-sk");
+ GCSProperties storage = (GCSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertFalse(storage.hasGcpCredential());
+ Assertions.assertEquals("", storage.getCredentialProviderType());
+ Assertions.assertEquals("hmac-ak", storage.getAwsCredentialsProvider().resolveCredentials().accessKeyId());
+ Assertions.assertEquals("hmac-sk", storage.getAwsCredentialsProvider().resolveCredentials().secretAccessKey());
+ Assertions.assertEquals("hmac-ak", storage.getBackendConfigProperties().get("AWS_ACCESS_KEY"));
+ Assertions.assertFalse(S3Properties.getObjStoreInfoPB(props).hasCredential());
+ TS3StorageParam thrift = S3Properties.getS3TStorageParam(props);
+ Assertions.assertFalse(thrift.isSetCredential());
+ Assertions.assertEquals(TObjStorageType.AWS, thrift.getProvider());
+
+ props.remove("s3.access_key");
+ props.remove("s3.secret_key");
+ props.put(S3Properties.CREDENTIALS_PROVIDER_TYPE, "ANONYMOUS");
+ storage = (GCSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertTrue(storage.isAnonymous());
+ Assertions.assertInstanceOf(AnonymousCredentialsProvider.class, storage.getAwsCredentialsProvider());
+ Assertions.assertEquals(TCredProviderType.ANONYMOUS,
+ S3Properties.getS3TStorageParam(props).getCredProviderType());
+ }
+
+ @Test
+ public void testLegacyS3OmittedAuthenticationUsesUnsignedClients() {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "S3");
+ props.put("s3.endpoint", "https://storage.googleapis.com");
+ GCSProperties storage = (GCSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertTrue(storage.isAnonymous());
+ Assertions.assertFalse(storage.hasGcpCredential());
+ Assertions.assertEquals("ANONYMOUS", storage.getCredentialProviderType());
+ Assertions.assertInstanceOf(AnonymousCredentialsProvider.class, storage.getAwsCredentialsProvider());
+ Assertions.assertEquals("org.apache.hadoop.fs.s3a.S3AFileSystem",
+ storage.hadoopStorageConfig.get("fs.gs.impl"));
+ Assertions.assertEquals("org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider",
+ storage.hadoopStorageConfig.get(Constants.AWS_CREDENTIALS_PROVIDER));
+ Assertions.assertEquals("ANONYMOUS", storage.getBackendConfigProperties().get("AWS_CREDENTIALS_PROVIDER_TYPE"));
+ TS3StorageParam thrift = S3Properties.getS3TStorageParam(props);
+ Assertions.assertEquals(TCredProviderType.ANONYMOUS, thrift.getCredProviderType());
+ Assertions.assertFalse(thrift.isSetCredential());
+ Assertions.assertEquals(TObjStorageType.AWS, thrift.getProvider());
+ IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class,
+ () -> S3Properties.getObjStoreInfoPB(props));
+ Assertions.assertTrue(error.getMessage().contains("storage vaults"));
+ }
+
+ @Test
+ public void testResolvedAuthenticationIsSharedByClientsAndFingerprint() {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "GCP");
+ props.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE.toUpperCase(Locale.ROOT), " compute_engine ");
+ props.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT, "target@test.iam.gserviceaccount.com");
+ GCSProperties storage = (GCSProperties) StorageProperties.createPrimary(props);
+ GcpCredential credential = storage.getGcpCredential();
+ String fingerprint = storage.getFsCacheFingerprint();
+
+ // Changing the caller's map must not change the identity of an already initialized instance.
+ props.clear();
+ props.put("provider", "GCP");
+ props.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "ANONYMOUS");
+ Assertions.assertTrue(storage.hasGcpCredential());
+ Assertions.assertSame(credential, storage.getGcpCredential());
+ Assertions.assertThrows(IllegalStateException.class, storage::getAwsCredentialsProvider);
+ Assertions.assertEquals("COMPUTE_ENGINE", storage.getCredentialProviderType());
+ Assertions.assertEquals("COMPUTE_ENGINE", storage.getHadoopStorageConfig().get("fs.gs.auth.type"));
+ Assertions.assertEquals("target@test.iam.gserviceaccount.com",
+ storage.getHadoopStorageConfig().get("fs.gs.auth.impersonation.service.account"));
+ Assertions.assertEquals("COMPUTE_ENGINE",
+ storage.getBackendConfigProperties().get(GCSProperties.CREDENTIAL_PROVIDER_TYPE));
+ Assertions.assertEquals(fingerprint, storage.getFsCacheFingerprint());
+
+ GCSProperties anonymous = (GCSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertFalse(anonymous.hasGcpCredential());
+ Assertions.assertNotEquals(fingerprint, anonymous.getFsCacheFingerprint());
+ }
+
+ @Test
+ public void testLegacyAnonymousRejectsCredentialConflictsAcrossPaths() {
+ for (String anonymousKey : new String[] {S3Properties.CREDENTIALS_PROVIDER_TYPE,
+ S3Properties.Env.CREDENTIALS_PROVIDER_TYPE}) {
+ for (String conflictKey : new String[] {"gs.access_key", "gs.secret_key", "gs.session_token",
+ S3Properties.ROLE_ARN, S3Properties.EXTERNAL_ID}) {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "GCP");
+ props.put(anonymousKey, " anonymous ");
+ props.put(conflictKey, "conflicting-value");
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> StorageProperties.createPrimary(props), conflictKey);
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> S3Properties.getS3TStorageParam(props), conflictKey);
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> S3Properties.getObjStoreInfoPB(props), conflictKey);
+ }
+ }
+ }
+
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
index 6aff2ad8a89..822bd9eb76b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
@@ -23,6 +23,7 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.UserException;
import org.apache.doris.thrift.TCredProviderType;
+import org.apache.doris.thrift.TObjStorageType;
import org.apache.doris.thrift.TS3StorageParam;
import com.google.common.collect.Maps;
@@ -295,6 +296,30 @@ public class S3PropertiesTest {
Assertions.assertTrue(S3StorageVault.ALLOW_ALTER_PROPERTIES.contains(S3Properties.CREDENTIALS_PROVIDER_TYPE));
}
+ @Test
+ public void testThriftProviderMapping() {
+ Map<String, TObjStorageType> expected = new HashMap<>();
+ expected.put("S3", TObjStorageType.AWS);
+ expected.put("AZURE", TObjStorageType.AZURE);
+ expected.put("BOS", TObjStorageType.BOS);
+ expected.put("COS", TObjStorageType.COS);
+ expected.put("OBS", TObjStorageType.OBS);
+ expected.put("OSS", TObjStorageType.OSS);
+ expected.put("GCP", TObjStorageType.GCP);
+ expected.put("TOS", TObjStorageType.TOS);
+
+ expected.forEach((provider, thriftProvider) -> {
+ origProps.clear();
+ origProps.put(StorageProperties.FS_PROVIDER_KEY, provider.toLowerCase());
+ Assertions.assertEquals(thriftProvider,
+ S3Properties.getS3TStorageParam(origProps).getProvider(), provider);
+ });
+
+ origProps.clear();
+ origProps.put(StorageProperties.FS_PROVIDER_KEY, " ");
+ Assertions.assertFalse(S3Properties.getS3TStorageParam(origProps).isSetProvider());
+ }
+
@Test
public void testS3PropertiesIgnoreIcebergRestIamRoleAliases() throws UserException {
origProps.put("s3.endpoint", "s3.us-west-2.amazonaws.com");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/auth/GcpAuthTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/auth/GcpAuthTest.java
new file mode 100644
index 00000000000..90569608aa2
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/auth/GcpAuthTest.java
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage.auth;
+
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.ComputeEngineCredentials;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.auth.oauth2.ImpersonatedCredentials;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import software.amazon.awssdk.core.interceptor.Context;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.http.SdkHttpMethod;
+import software.amazon.awssdk.http.SdkHttpRequest;
+
+import java.net.URI;
+import java.util.Collections;
+import java.util.Date;
+
+public class GcpAuthTest {
+ @Test
+ public void testComputeEngineCredentialsUseMetadataProviderDirectly() throws Exception {
+ GoogleCredentials credentials = GcpAuth.createCredentials(
+ new GcpCredential(GcpCredentialProviderType.COMPUTE_ENGINE, ""));
+
+ Assertions.assertInstanceOf(ComputeEngineCredentials.class, credentials);
+ }
+
+ @Test
+ public void testComputeEngineImpersonationWrapsMetadataCredentials() throws Exception {
+ GoogleCredentials credentials = GcpAuth.createCredentials(
+ new GcpCredential(GcpCredentialProviderType.COMPUTE_ENGINE,
+ "target@my-project.iam.gserviceaccount.com"));
+
+ Assertions.assertInstanceOf(ImpersonatedCredentials.class, credentials);
+ }
+
+ @Test
+ public void testBearerTokenInterceptorUsesCredentialRequestMetadata() {
+ GoogleCredentials credentials = GoogleCredentials.create(
+ new AccessToken("metadata-token", new Date(System.currentTimeMillis() + 3_600_000)));
+ ExecutionInterceptor interceptor = GcpAuth.bearerTokenInterceptor(credentials);
+ Context.ModifyHttpRequest context = Mockito.mock(Context.ModifyHttpRequest.class);
+ SdkHttpRequest request = SdkHttpRequest.builder()
+ .uri(URI.create("https://storage.googleapis.com/bucket/key"))
+ .method(SdkHttpMethod.GET)
+ .build();
+ Mockito.when(context.httpRequest()).thenReturn(request);
+
+ SdkHttpRequest modified = interceptor.modifyHttpRequest(context, new ExecutionAttributes());
+
+ Assertions.assertEquals("Bearer metadata-token",
+ modified.firstMatchingHeader("Authorization").orElse(null));
+ }
+
+ @Test
+ public void testBearerTokenInterceptorRejectsMissingAuthorizationMetadata() throws Exception {
+ GoogleCredentials credentials = Mockito.mock(GoogleCredentials.class);
+ Mockito.when(credentials.getRequestMetadata()).thenReturn(Collections.emptyMap());
+ ExecutionInterceptor interceptor = GcpAuth.bearerTokenInterceptor(credentials);
+ Context.ModifyHttpRequest context = Mockito.mock(Context.ModifyHttpRequest.class);
+ SdkHttpRequest request = SdkHttpRequest.builder()
+ .uri(URI.create("https://storage.googleapis.com/bucket/key"))
+ .method(SdkHttpMethod.GET)
+ .build();
+ Mockito.when(context.httpRequest()).thenReturn(request);
+
+ RuntimeException exception = Assertions.assertThrows(RuntimeException.class,
+ () -> interceptor.modifyHttpRequest(context, new ExecutionAttributes()));
+ Assertions.assertTrue(exception.getMessage().contains("Failed to obtain a GCP access token"));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/auth/GcsAuthResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/auth/GcsAuthResolverTest.java
new file mode 100644
index 00000000000..50b512c3455
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/auth/GcsAuthResolverTest.java
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage.auth;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class GcsAuthResolverTest {
+ @Test
+ public void testDefaultAndExplicitModes() {
+ Map<String, String> props = new HashMap<>();
+ Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent());
+ props.put("provider", "GCP");
+ Assertions.assertEquals(GcsAuth.Mode.ADC, GcsAuthResolver.resolve(props).get().getMode());
+ for (String provider : new String[] {"DEFAULT", "COMPUTE_ENGINE", "ANONYMOUS"}) {
+ props.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, provider);
+ GcsAuth auth = GcsAuthResolver.resolve(props).get();
+ Assertions.assertEquals(provider.equals("DEFAULT") ? GcsAuth.Mode.ADC : GcsAuth.Mode.valueOf(provider),
+ auth.getMode());
+ Assertions.assertEquals(!provider.equals("ANONYMOUS"), auth.getNativeCredential().isPresent());
+ }
+ }
+
+ @Test
+ public void testImpersonationAndFactoryUseResolvedCredentials() {
+ Map<String, String> props = new HashMap<>();
+ props.put(GcpCredential.IMPERSONATION_SERVICE_ACCOUNT, "target@test.iam.gserviceaccount.com");
+ GcsAuth auth = GcsAuthResolver.resolve(props).get();
+ Assertions.assertEquals(GcsAuth.Mode.ADC, auth.getMode());
+ GcpCredential credential = (GcpCredential) ObjCredentialFactory.fromProperties(props).get();
+ Assertions.assertEquals(auth.getNativeCredential().get().getCredentialProviderType(),
+ credential.getCredentialProviderType());
+ Assertions.assertEquals("target@test.iam.gserviceaccount.com", credential.getImpersonationServiceAccount());
+ }
+
+ @Test
+ public void testHmacAliasesDoNotAcquireAdc() {
+ for (String[] keys : new String[][] {{"gs.access_key", "gs.secret_key"},
+ {"s3.access_key", "s3.secret_key"}, {"AWS_ACCESS_KEY", "AWS_SECRET_KEY"},
+ {"ACCESS_KEY", "SECRET_KEY"}}) {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "GCP");
+ props.put(keys[0], "ak");
+ props.put(keys[1], "sk");
+ Assertions.assertEquals(GcsAuth.Mode.HMAC, GcsAuthResolver.resolve(props).get().getMode());
+ Assertions.assertFalse(ObjCredentialFactory.fromProperties(props).isPresent());
+ // Vault ALTER may supply just one HMAC field; it must not inject native credentials.
+ props.remove(keys[1]);
+ Assertions.assertEquals(GcsAuth.Mode.HMAC, GcsAuthResolver.resolve(props).get().getMode());
+ }
+ }
+
+ @Test
+ public void testLegacyS3HmacRequiresGcsSelector() {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "S3");
+ props.put("s3.access_key", "ak");
+ props.put("s3.secret_key", "sk");
+ Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent());
+ props.put("s3.endpoint", "storage.googleapis.com");
+ Assertions.assertEquals(GcsAuth.Mode.HMAC, GcsAuthResolver.resolve(props).get().getMode());
+ for (String provider : new String[] {"AZURE", "OSS"}) {
+ props.put("provider", provider);
+ Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent());
+ }
+ props.put("provider", "S3");
+ props.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ Assertions.assertThrows(IllegalArgumentException.class, () -> GcsAuthResolver.resolve(props));
+ }
+
+ @Test
+ public void testLegacyS3OmittedAuthenticationUsesAnonymousOnlyForGcs() {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", "s3");
+ Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent());
+ props.put("s3.endpoint", "https://storage.googleapis.com");
+ GcsAuth auth = GcsAuthResolver.resolve(props).get();
+ Assertions.assertEquals(GcsAuth.Mode.ANONYMOUS, auth.getMode());
+ Assertions.assertFalse(auth.getNativeCredential().isPresent());
+
+ props.put("s3.credentials_provider_type", "DEFAULT");
+ Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent());
+ props.remove("s3.credentials_provider_type");
+ props.put("provider", "GCP");
+ Assertions.assertEquals(GcsAuth.Mode.ADC, GcsAuthResolver.resolve(props).get().getMode());
+ props.remove("provider");
+ Assertions.assertEquals(GcsAuth.Mode.ADC, GcsAuthResolver.resolve(props).get().getMode());
+ }
+
+ @Test
+ public void testOtherProvidersAndConflictingNativeProperties() {
+ for (String provider : new String[] {"S3", "AZURE", "OSS"}) {
+ Map<String, String> props = new HashMap<>();
+ props.put("provider", provider);
+ props.put("s3.endpoint", "storage.googleapis.com");
+ if ("S3".equals(provider)) {
+ Assertions.assertEquals(GcsAuth.Mode.ANONYMOUS, GcsAuthResolver.resolve(props).get().getMode());
+ } else {
+ Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent());
+ }
+ props.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ Assertions.assertThrows(IllegalArgumentException.class, () -> GcsAuthResolver.resolve(props));
+ }
+ Map<String, String> props = new HashMap<>();
+ props.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, "");
+ Assertions.assertThrows(IllegalArgumentException.class, () -> GcsAuthResolver.resolve(props));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java
index 5b6d4e4aaec..f380f7a69dd 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans;
+import org.apache.doris.datasource.property.storage.GCSProperties;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator;
@@ -35,8 +36,11 @@ import org.apache.doris.utframe.TestWithFeService;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import java.lang.reflect.Field;
+import java.util.Map;
public class OutFileTest extends TestWithFeService implements PlanPatternMatchSupported {
private final NereidsParser parser = new NereidsParser();
@@ -94,6 +98,7 @@ public class OutFileTest extends TestWithFeService implements PlanPatternMatchSu
Field field = ResultFileSink.class.getDeclaredField("fileSinkOptions");
field.setAccessible(true);
TResultFileSinkOptions sinkOptions = (TResultFileSinkOptions) field.get(fragment.getSink());
+ Assertions.assertEquals("hdfs://127.0.0.1:8020/tmp/outfile_test_", sinkOptions.getFilePath());
Assertions.assertEquals("hdfs://127.0.0.1:8020", sinkOptions.getBrokerProperties().get("fs.defaultFS"));
}
@@ -117,6 +122,105 @@ public class OutFileTest extends TestWithFeService implements PlanPatternMatchSu
Assertions.assertTrue(exception.getMessage().contains("should use binary"), exception.getMessage());
}
+ @Test
+ public void testGcsOutFileNormalizesPathAndPreservesCredentials() throws Exception {
+ for (String scheme : new String[] {"gs", "s3"}) {
+ for (String provider : new String[] {"DEFAULT", "COMPUTE_ENGINE"}) {
+ for (String serviceAccount : new String[] {"", "target@test-project.iam.gserviceaccount.com"}) {
+ String properties = "\"provider\" = \"GCP\","
+ + "\"s3.endpoint\" = \"https://storage.googleapis.com\","
+ + "\"s3.region\" = \"us-central1\","
+ + "\"" + GCSProperties.CREDENTIAL_PROVIDER_TYPE + "\" = \"" + provider + "\"";
+ if (!serviceAccount.isEmpty()) {
+ properties += ",\"" + GCSProperties.IMPERSONATION_SERVICE_ACCOUNT
+ + "\" = \"" + serviceAccount + "\"";
+ }
+ TResultFileSinkOptions options = getFileSinkOptions("select * from T1 into outfile '"
+ + scheme + "://outfile-bucket/nested/result_' format as csv properties ("
+ + properties + ")");
+ Assertions.assertEquals("s3://outfile-bucket/nested/result_", options.getFilePath());
+ Map<String, String> backendProperties = options.getBrokerProperties();
+ Assertions.assertEquals("GCP", backendProperties.get("provider"));
+ Assertions.assertEquals("https://storage.googleapis.com", backendProperties.get("AWS_ENDPOINT"));
+ // GCSProperties does not bind s3.region and retains its compatibility default.
+ Assertions.assertEquals("us-east1", backendProperties.get("AWS_REGION"));
+ Assertions.assertEquals(provider, backendProperties.get(GCSProperties.CREDENTIAL_PROVIDER_TYPE));
+ Assertions.assertEquals(serviceAccount,
+ backendProperties.getOrDefault(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT, ""));
+ }
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "http, false", "https, false", "http, true", "https, true",
+ "HTTP, false", "HTTPS, false"
+ })
+ public void testS3HttpOutFilePreservesPath(String scheme, boolean usePathStyle) throws Exception {
+ String path = scheme + "://s3.us-east-1.amazonaws.com/outfile-bucket/nested/result_";
+ TResultFileSinkOptions options = getFileSinkOptions("select * from T1 into outfile '"
+ + path + "' format as csv properties ("
+ + "\"s3.endpoint\" = \"https://s3.us-east-1.amazonaws.com\","
+ + "\"s3.region\" = \"us-east-1\","
+ + "\"s3.access_key\" = \"test-ak\",\"s3.secret_key\" = \"test-sk\","
+ + "\"use_path_style\" = \"" + usePathStyle + "\")");
+ Assertions.assertEquals(path, options.getFilePath());
+ }
+
+ @Test
+ public void testHdfsOutFilePreservesPathWithoutAuthority() throws Exception {
+ String path = "hdfs:/tmp/outfile_test_";
+ TResultFileSinkOptions options = getFileSinkOptions("select * from T1 into outfile '"
+ + path + "' format as csv properties (\"hadoop.username\" = \"doris\")");
+ Assertions.assertEquals(path, options.getFilePath());
+ }
+
+ @Test
+ public void testS3OutFilePreservesPath() throws Exception {
+ TResultFileSinkOptions options = getFileSinkOptions("select * from T1 into outfile '"
+ + "s3://outfile-bucket/nested/result_' format as csv properties ("
+ + "\"s3.endpoint\" = \"https://s3.us-east-1.amazonaws.com\","
+ + "\"s3.region\" = \"us-east-1\","
+ + "\"s3.access_key\" = \"test-ak\",\"s3.secret_key\" = \"test-sk\")");
+ Assertions.assertEquals("s3://outfile-bucket/nested/result_", options.getFilePath());
+ Assertions.assertEquals("test-ak", options.getBrokerProperties().get("AWS_ACCESS_KEY"));
+ Assertions.assertEquals("test-sk", options.getBrokerProperties().get("AWS_SECRET_KEY"));
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "oss://outfile-bucket, https://oss-cn-hangzhou.aliyuncs.com, cn-hangzhou",
+ "oss://outfile-bucket.oss-cn-hangzhou.aliyuncs.com, https://oss-cn-hangzhou.aliyuncs.com, cn-hangzhou",
+ "s3://outfile-bucket.oss-cn-hangzhou.aliyuncs.com, https://oss-cn-hangzhou.aliyuncs.com, cn-hangzhou",
+ "cos://outfile-bucket, https://cos.ap-guangzhou.myqcloud.com, ap-guangzhou",
+ "cosn://outfile-bucket, https://cos.ap-guangzhou.myqcloud.com, ap-guangzhou",
+ "obs://outfile-bucket, https://obs.cn-north-4.myhuaweicloud.com, cn-north-4",
+ "bos://outfile-bucket, https://s3.us-east-1.amazonaws.com, us-east-1",
+ "s3a://outfile-bucket, https://s3.us-east-1.amazonaws.com, us-east-1",
+ "s3n://outfile-bucket, https://s3.us-east-1.amazonaws.com, us-east-1"
+ })
+ public void testS3CompatibleOutFileNormalizesPath(String location, String endpoint, String region) throws Exception {
+ TResultFileSinkOptions options = getFileSinkOptions("select * from T1 into outfile '"
+ + location + "/nested/result_' format as csv properties ("
+ + "\"s3.endpoint\" = \"" + endpoint + "\","
+ + "\"s3.region\" = \"" + region + "\","
+ + "\"s3.access_key\" = \"test-ak\",\"s3.secret_key\" = \"test-sk\")");
+ Assertions.assertEquals("s3://outfile-bucket/nested/result_", options.getFilePath());
+ Assertions.assertEquals(endpoint, options.getBrokerProperties().get("AWS_ENDPOINT"));
+ Assertions.assertEquals(region, options.getBrokerProperties().get("AWS_REGION"));
+ Assertions.assertEquals("test-ak", options.getBrokerProperties().get("AWS_ACCESS_KEY"));
+ Assertions.assertEquals("test-sk", options.getBrokerProperties().get("AWS_SECRET_KEY"));
+ }
+
+ private TResultFileSinkOptions getFileSinkOptions(String sql) throws Exception {
+ PlanFragment fragment = getOutputFragment(sql);
+ Assertions.assertTrue(fragment.getSink() instanceof ResultFileSink);
+ Field field = ResultFileSink.class.getDeclaredField("fileSinkOptions");
+ field.setAccessible(true);
+ return (TResultFileSinkOptions) field.get(fragment.getSink());
+ }
+
private PlanFragment getOutputFragment(String sql) throws Exception {
StatementScopeIdGenerator.clear();
StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExportCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExportCommandTest.java
index 981b7be805b..c997aeca8c5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExportCommandTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExportCommandTest.java
@@ -17,18 +17,132 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.BrokerDesc;
+import org.apache.doris.analysis.StorageBackend.StorageType;
+import org.apache.doris.analysis.TableName;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.datasource.property.fileformat.ParquetFileFormatProperties;
+import org.apache.doris.datasource.property.storage.GCSProperties;
+import org.apache.doris.load.ExportJob;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.utframe.TestWithFeService;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import java.lang.reflect.Method;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
-public class ExportCommandTest {
+public class ExportCommandTest extends TestWithFeService {
+ @Override
+ protected void runBeforeAll() throws Exception {
+ createDatabase("export_test");
+ connectContext.setDatabase("export_test");
+ createTable("CREATE TABLE T1 (id bigint) DUPLICATE KEY(id) "
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\")");
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "s3://export-bucket, https://s3.us-east-1.amazonaws.com, us-east-1",
+ "oss://export-bucket, https://oss-cn-hangzhou.aliyuncs.com, cn-hangzhou",
+ "oss://export-bucket.oss-cn-hangzhou.aliyuncs.com, https://oss-cn-hangzhou.aliyuncs.com, cn-hangzhou",
+ "s3://export-bucket.oss-cn-hangzhou.aliyuncs.com, https://oss-cn-hangzhou.aliyuncs.com, cn-hangzhou",
+ "cos://export-bucket, https://cos.ap-guangzhou.myqcloud.com, ap-guangzhou",
+ "cosn://export-bucket, https://cos.ap-guangzhou.myqcloud.com, ap-guangzhou",
+ "obs://export-bucket, https://obs.cn-north-4.myhuaweicloud.com, cn-north-4",
+ "bos://export-bucket, https://s3.us-east-1.amazonaws.com, us-east-1",
+ "s3a://export-bucket, https://s3.us-east-1.amazonaws.com, us-east-1",
+ "s3n://export-bucket, https://s3.us-east-1.amazonaws.com, us-east-1"
+ })
+ public void testS3CompatibleExportNormalizesPath(String location, String endpoint, String region) throws Exception {
+ BrokerDesc broker = new BrokerDesc(null, s3Properties(endpoint, region));
+ ExportJob job = generateExportJob(location + "/nested/result_", broker);
+ Assertions.assertEquals("s3://export-bucket/nested/result_", job.getExportPath());
+ Map<String, String> backendProperties = job.getBrokerDesc().getBackendConfigProperties();
+ Assertions.assertEquals(endpoint, backendProperties.get("AWS_ENDPOINT"));
+ Assertions.assertEquals(region, backendProperties.get("AWS_REGION"));
+ Assertions.assertEquals("test-ak", backendProperties.get("AWS_ACCESS_KEY"));
+ Assertions.assertEquals("test-sk", backendProperties.get("AWS_SECRET_KEY"));
+ }
+
+ @Test
+ public void testGcsExportNormalizesPathAndPreservesCredentials() throws Exception {
+ for (String scheme : new String[] {"gs", "s3"}) {
+ for (String provider : new String[] {"DEFAULT", "COMPUTE_ENGINE"}) {
+ for (String serviceAccount : new String[] {"", "target@test-project.iam.gserviceaccount.com"}) {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("provider", "GCP");
+ properties.put("s3.endpoint", "https://storage.googleapis.com");
+ properties.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, provider);
+ if (!serviceAccount.isEmpty()) {
+ properties.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT, serviceAccount);
+ }
+ ExportJob job = generateExportJob(scheme + "://export-bucket/nested/result_",
+ new BrokerDesc(null, properties));
+ Assertions.assertEquals("s3://export-bucket/nested/result_", job.getExportPath());
+ Map<String, String> backendProperties = job.getBrokerDesc().getBackendConfigProperties();
+ Assertions.assertEquals("GCP", backendProperties.get("provider"));
+ Assertions.assertEquals("https://storage.googleapis.com", backendProperties.get("AWS_ENDPOINT"));
+ Assertions.assertEquals(provider, backendProperties.get(GCSProperties.CREDENTIAL_PROVIDER_TYPE));
+ Assertions.assertEquals(serviceAccount,
+ backendProperties.getOrDefault(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT, ""));
+ }
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "http, false", "https, false", "http, true", "https, true",
+ "HTTP, false", "HTTPS, false"
+ })
+ public void testHttpExportPreservesPath(String scheme, boolean usePathStyle) throws Exception {
+ Map<String, String> properties = s3Properties("https://s3.us-east-1.amazonaws.com", "us-east-1");
+ properties.put("use_path_style", Boolean.toString(usePathStyle));
+ String path = scheme + "://s3.us-east-1.amazonaws.com/export-bucket/nested/result_";
+ Assertions.assertEquals(path, generateExportJob(path, new BrokerDesc(null, properties)).getExportPath());
+ }
+
+ @Test
+ public void testNonS3ExportPreservesPath() throws Exception {
+ String hdfsPath = "hdfs:/tmp/export_test_";
+ Assertions.assertEquals(hdfsPath, generateExportJob(hdfsPath,
+ new BrokerDesc(null, ImmutableMap.of("hadoop.username", "doris"))).getExportPath());
+ String brokerPath = "oss://export-bucket/nested/result_";
+ Assertions.assertEquals(brokerPath, generateExportJob(brokerPath,
+ new BrokerDesc("broker", StorageType.BROKER, ImmutableMap.of())).getExportPath());
+ String localPath = "file:///tmp/export_test_";
+ Assertions.assertEquals(localPath, generateExportJob(localPath,
+ new BrokerDesc("local", StorageType.LOCAL, null)).getExportPath());
+ }
+
+ private Map<String, String> s3Properties(String endpoint, String region) {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("s3.endpoint", endpoint);
+ properties.put("s3.region", region);
+ properties.put("s3.access_key", "test-ak");
+ properties.put("s3.secret_key", "test-sk");
+ return properties;
+ }
+
+ private ExportJob generateExportJob(String path, BrokerDesc broker) throws Exception {
+ ExportCommand command = new ExportCommand(ImmutableList.of("T1"), ImmutableList.of(), Optional.empty(),
+ path, ImmutableMap.of(), Optional.of(broker));
+ // Inspect the job before registration so the test never starts an export task.
+ Method method = ExportCommand.class.getDeclaredMethod("generateExportJob",
+ ConnectContext.class, Map.class, TableName.class);
+ method.setAccessible(true);
+ return (ExportJob) method.invoke(command, connectContext, ImmutableMap.of(),
+ new TableName("internal", "export_test", "T1"));
+ }
@Test
void testEnableInt96TimestampsProperty() {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java
index 0f990e6aba7..8c5f850f23d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java
@@ -23,6 +23,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties;
import org.apache.doris.datasource.property.fileformat.DeferredFileFormatProperties;
+import org.apache.doris.datasource.property.storage.GCSProperties;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.analyzer.UnboundSlot;
import org.apache.doris.nereids.load.NereidsDataDescription;
@@ -84,6 +85,46 @@ public class LoadCommandTest extends TestWithFeService {
createTable(createTableBitmapSql);
}
+ @Test
+ public void testGcpNativeCredentialForS3Load() {
+ String loadSql = "LOAD LABEL gcp_native_load("
+ + " DATA INFILE(\"gs://load-bucket/customer\")"
+ + " INTO TABLE customer"
+ + ") WITH S3("
+ + " \"provider\" = \"GCP\","
+ + " \"gs.endpoint\" = \"https://storage.googleapis.com\","
+ + " \"gs.credential_provider_type\" = \"COMPUTE_ENGINE\","
+ + " \"gs.impersonation_service_account\" = "
+ + "\"target@my-project.iam.gserviceaccount.com\""
+ + ");";
+
+ LoadCommand command = (LoadCommand) new NereidsParser().parseSingle(loadSql);
+ BrokerDesc brokerDesc = command.getBrokerDesc();
+ Assertions.assertEquals("us-east1", brokerDesc.getBackendConfigProperties().get("AWS_REGION"));
+ Assertions.assertEquals("GCP", brokerDesc.getBackendConfigProperties().get("provider"));
+ Assertions.assertEquals("COMPUTE_ENGINE", brokerDesc.getBackendConfigProperties()
+ .get(GCSProperties.CREDENTIAL_PROVIDER_TYPE));
+ Assertions.assertEquals("target@my-project.iam.gserviceaccount.com",
+ brokerDesc.getBackendConfigProperties().get(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT));
+ }
+
+ @Test
+ public void testGcpNativeCredentialValidationHappensBeforeLoadSubmission() {
+ String loadSql = "LOAD LABEL invalid_gcp_native_load("
+ + " DATA INFILE(\"gs://load-bucket/customer\")"
+ + " INTO TABLE customer"
+ + ") WITH S3("
+ + " \"provider\" = \"GCP\","
+ + " \"gs.credential_provider_type\" = \"\""
+ + ");";
+
+ IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new NereidsParser().parseSingle(loadSql));
+ Assertions.assertTrue(exception.getMessage().contains(GCSProperties.CREDENTIAL_PROVIDER_TYPE));
+ Assertions.assertTrue(exception.getMessage().contains("DEFAULT"));
+ Assertions.assertTrue(exception.getMessage().contains("COMPUTE_ENGINE"));
+ }
+
@Test
public void testLoadCommandBitmap() {
String loadSql1 = "LOAD LABEL load_bitmap_table_test( "
diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java
index f08af573bef..ff9bbc17f0a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java
@@ -22,12 +22,14 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.FileFormatConstants;
import org.apache.doris.common.util.FileFormatUtils;
import org.apache.doris.datasource.lance.metadata.LanceTableAccess;
import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
import org.apache.doris.datasource.property.fileformat.FileFormatProperties;
import org.apache.doris.datasource.property.fileformat.LanceFileFormatProperties;
+import org.apache.doris.datasource.property.storage.GCSProperties;
import org.apache.doris.thrift.TFileFormatType;
import com.google.common.collect.Lists;
@@ -64,6 +66,53 @@ public class ExternalFileTableValuedFunctionTest {
}
}
+ @Test
+ public void testGcpNativeCredentialForS3Tvf() throws AnalysisException {
+ boolean previousRunningUnitTest = FeConstants.runningUnitTest;
+ FeConstants.runningUnitTest = true;
+ try {
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put("uri", "gs://tvf-bucket/path/file.parquet");
+ properties.put("format", "parquet");
+ properties.put("provider", "GCP");
+ properties.put("gs.endpoint", "https://storage.googleapis.com");
+ properties.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "COMPUTE_ENGINE");
+ properties.put(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT,
+ "target@my-project.iam.gserviceaccount.com");
+
+ S3TableValuedFunction tvf = new S3TableValuedFunction(properties);
+ Assert.assertEquals("s3://tvf-bucket/path/file.parquet", tvf.getFilePath());
+ Assert.assertEquals("us-east1", tvf.getBackendConnectProperties().get("AWS_REGION"));
+ Assert.assertEquals("GCP", tvf.getBackendConnectProperties().get("provider"));
+ Assert.assertEquals("COMPUTE_ENGINE", tvf.getBackendConnectProperties()
+ .get(GCSProperties.CREDENTIAL_PROVIDER_TYPE));
+ Assert.assertEquals("target@my-project.iam.gserviceaccount.com",
+ tvf.getBackendConnectProperties().get(GCSProperties.IMPERSONATION_SERVICE_ACCOUNT));
+ } finally {
+ FeConstants.runningUnitTest = previousRunningUnitTest;
+ }
+ }
+
+ @Test
+ public void testGcpNativeCredentialValidationHappensBeforeTvfAccess() {
+ boolean previousRunningUnitTest = FeConstants.runningUnitTest;
+ FeConstants.runningUnitTest = true;
+ try {
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put("uri", "gs://tvf-bucket/path/file.parquet");
+ properties.put("format", "parquet");
+ properties.put("provider", "GCP");
+ properties.put(GCSProperties.CREDENTIAL_PROVIDER_TYPE, "DEFAULT");
+ properties.put("s3.session_token", "session-token");
+
+ IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class,
+ () -> new S3TableValuedFunction(properties));
+ Assert.assertTrue(exception.getMessage().contains("session token"));
+ } finally {
+ FeConstants.runningUnitTest = previousRunningUnitTest;
+ }
+ }
+
@Test
public void testLanceIsAcceptedByFileFormatFactory() {
FileFormatProperties properties = FileFormatProperties.createFileFormatProperties("LaNcE");
diff --git a/fe/pom.xml b/fe/pom.xml
index c4c3ddf586c..4f0a24ee4ca 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -374,6 +374,8 @@ under the License.
<mariadb-java-client.version>3.0.9</mariadb-java-client.version>
<postgresql.version>42.7.13</postgresql.version>
<hadoop.version>3.4.2</hadoop.version>
+ <gcs-connector.version>3.1.18</gcs-connector.version>
+ <google-auth-library-oauth2-http.version>1.46.0</google-auth-library-oauth2-http.version>
<re2j.version>1.8</re2j.version>
<hadoop.thirdparty.guava.version>1.2.0</hadoop.thirdparty.guava.version>
<hadoop.thirdparty.protobuf_3_25.version>1.5.0</hadoop.thirdparty.protobuf_3_25.version>
@@ -480,6 +482,11 @@ under the License.
</profiles>
<dependencyManagement>
<dependencies>
+ <dependency>
+ <groupId>com.google.auth</groupId>
+ <artifactId>google-auth-library-oauth2-http</artifactId>
+ <version>${google-auth-library-oauth2-http.version}</version>
+ </dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
@@ -1316,6 +1323,19 @@ under the License.
</exclusion>
</exclusions>
</dependency>
+ <!-- Keep the connector's Google SDK dependencies isolated from FE's SDK versions. -->
+ <dependency>
+ <groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>gcs-connector</artifactId>
+ <version>${gcs-connector.version}</version>
+ <classifier>shaded</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index 030029bedf2..1110223cb1a 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -287,6 +287,24 @@ enum CredProviderTypePB {
ANONYMOUS = 8; // AnonymousAWSCredentialsProvider
}
+message GcpCredentialPB {
+ enum CredentialProviderType {
+ DEFAULT = 0;
+ COMPUTE_ENGINE = 1;
+ }
+
+ optional CredentialProviderType credential_provider_type = 1;
+ optional string impersonation_service_account = 2;
+}
+
+// Provider credentials are grouped in a regular message so older readers can
+// still observe that a credential envelope is present when a new provider is
+// added. Validation must reject multiple provider credentials and ensure the
+// selected credential matches ObjectStoreInfoPB.provider.
+message ObjectStoreCredentialPB {
+ optional GcpCredentialPB gcp_credential = 1;
+}
+
message ObjectStoreInfoPB {
// presigned url use
// oss,aws,cos,obs,bos
@@ -320,6 +338,8 @@ message ObjectStoreInfoPB {
optional CredProviderTypePB cred_provider_type = 17;
optional string role_arn = 18; // aws assumed role's arn
optional string external_id = 19; // aws assumed role's external_id if configure
+
+ optional ObjectStoreCredentialPB credential = 20;
}
// The legacy ObjectStoreInfoPB is stored in InstanceInfoPB
diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift
index 9973d6fa1f8..7ad0fe8d423 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -99,6 +99,23 @@ enum TCredProviderType {
ANONYMOUS = 7 // AnonymousAWSCredentialsProvider
}
+enum TGcpCredentialProviderType {
+ DEFAULT = 0,
+ COMPUTE_ENGINE = 1
+}
+
+struct TGcpCredential {
+ 1: optional TGcpCredentialProviderType credential_provider_type
+ 2: optional string impersonation_service_account
+}
+
+// Keep this struct aligned with ObjectStoreCredentialPB. Append new fields with
+// new IDs; validation rejects multiple provider credentials and ensures the
+// selected credential matches TS3StorageParam.provider.
+struct TCredential {
+ 1: optional TGcpCredential gcp_credential
+}
+
struct TS3StorageParam {
1: optional string endpoint
2: optional string region
@@ -116,6 +133,7 @@ struct TS3StorageParam {
13: optional TCredProviderType cred_provider_type
14: optional string role_arn // aws assumed role's arn
15: optional string external_id // aws assumed role's external_id if configure
+ 16: optional TCredential credential
}
struct TStoragePolicy {
diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy
index 385a523d33c..741bc07899c 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -319,6 +319,28 @@ anymousS3Region="eu-west-3"
anymousS3ExpectDataCount="8365"
awsInstanceProfileRegion="us-east-1"
+// Object storage IAM regression tests run one provider per invocation: AWS or GCP.
+objectStorageIamProvider=""
+objectStorageIamEndpoint=""
+objectStorageIamRegion=""
+objectStorageIamBucket=""
+objectStorageIamPrefix="regression/object_storage_iam_p0"
+objectStorageIamDataPath="regression/tpch/sf0.01/customer.csv.gz"
+// AWS authentication.
+objectStorageIamAwsRoleArn=""
+objectStorageIamAwsExternalId=""
+// GCP authentication. Credential provider types are comma-separated DEFAULT and/or
+// COMPUTE_ENGINE. DEFAULT also tests omitted authentication properties; leaving both GCP
+// settings empty tests only omitted properties. A non-empty account adds COMPUTE_ENGINE
+// impersonation. The IAM data fixture must be private so anonymous reads can be rejected.
+objectStorageIamGcpCredentialProviderTypes=""
+objectStorageIamGcpImpersonationServiceAccount=""
+// Optional public GCS fixture for anonymous TVF reads: gzip CSV with a '|' separator.
+// Configure a gs:// URI and its positive row count to enable test_gcp_tvf_anonymous.
+// This public fixture must differ from the private objectStorageIamDataPath fixture.
+objectStorageIamGcpAnonymousUri=""
+objectStorageIamGcpAnonymousExpectedRows=""
+
// hudi p0 external regression test config
// To enable hudi test, you need first start hudi container.
// See `docker/thirdparties/run-thirdparties-docker.sh -c hudi`
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
index a87744a86a3..69ab664ec2e 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
@@ -1027,25 +1027,15 @@ class Syncer {
"""
}
- void createS3RepositoryWithRole(String name, boolean readOnly = false) {
- String roleArn = suite.context.config.awsRoleArn
- String externalId = suite.context.config.awsExternalId
- String endpoint = suite.context.config.awsEndpoint
- String region = suite.context.config.awsRegion
- String bucket = suite.context.config.awsBucket
- String prefix = suite.context.config.awsPrefix
-
+ void createObjectStorageIamRepository(String name, Map iamConfig, Map authCase, boolean readOnly = false) {
suite.try_sql "DROP REPOSITORY `${name}`"
suite.sql """
CREATE ${readOnly ? "READ ONLY" : ""} REPOSITORY `${name}`
WITH S3
- ON LOCATION "s3://${bucket}/${prefix}/aws_iam_role_p0/${name}"
+ ON LOCATION "${iamConfig.scheme}://${iamConfig.bucket}/${iamConfig.prefix}/${name}"
PROPERTIES
(
- "s3.endpoint" = "${endpoint}",
- "s3.region" = "${region}",
- "s3.role_arn" = "${roleArn}",
- "s3.external_id" = "${externalId}"
+ ${authCase.storageSqlProperties}
)
"""
}
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectStorageIamTestUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectStorageIamTestUtils.groovy
new file mode 100644
index 00000000000..e3e94fbf366
--- /dev/null
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectStorageIamTestUtils.groovy
@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.util
+
+import java.util.Locale
+
+/**
+ * Builds the IAM authentication matrix shared by the object_storage_iam_p0 suites.
+ *
+ * <p>One regression-test invocation selects exactly one cloud with
+ * {@code objectStorageIamProvider=AWS} or {@code objectStorageIamProvider=GCP}. Common connection
+ * settings are reused by both providers. Leaving {@code objectStorageIamProvider} empty disables
+ * the shared cases; AWS-only suites also use {@link #isProvider(Map, String)} to skip themselves
+ * during a GCP run.
+ *
+ * <p>AWS role example in regression-conf.groovy:
+ * <pre>
+ * objectStorageIamProvider="AWS"
+ * objectStorageIamEndpoint="https://s3.us-east-1.amazonaws.com"
+ * objectStorageIamRegion="us-east-1"
+ * objectStorageIamBucket="my-bucket"
+ * objectStorageIamPrefix="regression/object_storage_iam_p0"
+ * objectStorageIamDataPath="regression/tpch/sf0.01/customer.csv.gz"
+ * objectStorageIamAwsRoleArn="arn:aws:iam::123456789012:role/my-role"
+ * objectStorageIamAwsExternalId="optional-external-id"
+ * </pre>
+ *
+ * <p>GCP example covering DEFAULT, COMPUTE_ENGINE, and service-account impersonation:
+ * <pre>
+ * objectStorageIamProvider="GCP"
+ * objectStorageIamEndpoint="https://storage.googleapis.com"
+ * objectStorageIamRegion="us-east1"
+ * objectStorageIamBucket="my-bucket"
+ * objectStorageIamPrefix="regression/object_storage_iam_p0"
+ * objectStorageIamDataPath="regression/tpch/sf0.01/customer.csv.gz"
+ * objectStorageIamGcpCredentialProviderTypes="DEFAULT,COMPUTE_ENGINE"
+ * objectStorageIamGcpImpersonationServiceAccount="target@project.iam.gserviceaccount.com"
+ * </pre>
+ *
+ * <p>The comma-separated GCP provider list controls which direct credential cases run. DEFAULT also
+ * tests omitted authentication properties against the same private fixture. If both the provider
+ * list and impersonation account are empty, only the omitted-authentication case runs. A non-empty
+ * impersonation service account adds one more COMPUTE_ENGINE case. Each shared suite iterates over
+ * the resulting {@code authCases}. The test selector uses AWS/GCP, while SQL properties use the
+ * public S3/GCP provider values.
+ */
+class ObjectStorageIamTestUtils {
+ private static final List<String> PROVIDERS = ["AWS", "GCP"]
+ private static final List<String> GCP_CREDENTIAL_PROVIDER_TYPES = ["DEFAULT", "COMPUTE_ENGINE"]
+ private static final List<String> REQUIRED_CONFIGS = [
+ "objectStorageIamEndpoint",
+ "objectStorageIamRegion",
+ "objectStorageIamBucket",
+ "objectStorageIamPrefix",
+ "objectStorageIamDataPath"
+ ]
+
+ static Map getConfig(Map configs) {
+ String provider = value(configs, "objectStorageIamProvider")?.toUpperCase(Locale.ROOT)
+ if (provider == null || provider.isEmpty()) {
+ return null
+ }
+ if (!PROVIDERS.contains(provider)) {
+ throw new IllegalArgumentException("Unsupported objectStorageIamProvider: ${provider}; "
+ + "supported values are ${PROVIDERS}")
+ }
+
+ List<String> missingConfigs = REQUIRED_CONFIGS.findAll { isBlank(value(configs, it)) }
+ if (!missingConfigs.isEmpty()) {
+ throw new IllegalArgumentException("Missing object storage IAM configs: ${missingConfigs}")
+ }
+
+ List<Map> authCases = provider == "AWS"
+ ? getAwsAuthCases(configs)
+ : getGcpAuthCases(configs)
+ String endpointProperty = provider == "AWS" ? "s3.endpoint" : "gs.endpoint"
+ String propertyProvider = provider == "AWS" ? "S3" : "GCP"
+ String endpoint = value(configs, "objectStorageIamEndpoint")
+ String region = value(configs, "objectStorageIamRegion")
+ authCases.each { authCase ->
+ Map<String, String> storageProperties = new LinkedHashMap<>()
+ storageProperties.put("provider", propertyProvider)
+ storageProperties.put(endpointProperty, endpoint)
+ storageProperties.put("s3.region", region)
+ storageProperties.putAll(authCase.properties)
+ authCase.storageProperties = storageProperties
+ authCase.storageSqlProperties = toSqlProperties(storageProperties)
+ }
+ return [
+ provider: provider,
+ propertyProvider: propertyProvider,
+ scheme: provider == "AWS" ? "s3" : "gs",
+ endpoint: endpoint,
+ region: region,
+ bucket: value(configs, "objectStorageIamBucket"),
+ prefix: value(configs, "objectStorageIamPrefix"),
+ dataPath: value(configs, "objectStorageIamDataPath"),
+ roleArn: value(configs, "objectStorageIamAwsRoleArn"),
+ externalId: value(configs, "objectStorageIamAwsExternalId"),
+ authCases: authCases
+ ]
+ }
+
+ static boolean isProvider(Map configs, String expectedProvider) {
+ String provider = value(configs, "objectStorageIamProvider")
+ return provider != null && provider.equalsIgnoreCase(expectedProvider)
+ }
+
+ private static List<Map> getAwsAuthCases(Map configs) {
+ String roleArn = value(configs, "objectStorageIamAwsRoleArn")
+ if (roleArn == null || roleArn.isEmpty()) {
+ throw new IllegalArgumentException("objectStorageIamAwsRoleArn must be configured for AWS")
+ }
+
+ Map<String, String> properties = ["s3.role_arn": roleArn]
+ String externalId = value(configs, "objectStorageIamAwsExternalId")
+ if (externalId != null && !externalId.isEmpty()) {
+ properties.put("s3.external_id", externalId)
+ }
+ return [authCase("aws_role", properties)]
+ }
+
+ private static List<Map> getGcpAuthCases(Map configs) {
+ String configuredTypes = value(configs, "objectStorageIamGcpCredentialProviderTypes")
+ List<String> providerTypes = (configuredTypes ?: "").split(",")
+ .collect { it.trim().toUpperCase(Locale.ROOT) }
+ .findAll { !it.isEmpty() }
+ .unique()
+ List<String> unsupportedTypes = providerTypes.findAll {
+ !GCP_CREDENTIAL_PROVIDER_TYPES.contains(it)
+ }
+ if (!unsupportedTypes.isEmpty()) {
+ throw new IllegalArgumentException(
+ "Unsupported objectStorageIamGcpCredentialProviderTypes: ${unsupportedTypes}; "
+ + "supported values are ${GCP_CREDENTIAL_PROVIDER_TYPES}")
+ }
+
+ List<Map> cases = providerTypes.collect { providerType ->
+ authCase("gcp_${providerType.toLowerCase(Locale.ROOT)}",
+ ["gs.credential_provider_type": providerType])
+ }
+ String serviceAccount = value(configs, "objectStorageIamGcpImpersonationServiceAccount")
+ if (serviceAccount != null && !serviceAccount.isEmpty()) {
+ cases.add(authCase("gcp_impersonation", [
+ "gs.credential_provider_type": "COMPUTE_ENGINE",
+ "gs.impersonation_service_account": serviceAccount
+ ]))
+ }
+ if (providerTypes.contains("DEFAULT") || cases.isEmpty()) {
+ // Exercise the VM-attached service account without either native authentication property.
+ // Shared TVF, Load, Outfile, Export, Resource, Catalog and Vault suites all consume this case.
+ cases.add(0, authCase("gcp_default_omitted", [:]))
+ }
+ return cases
+ }
+
+ private static Map authCase(String name, Map<String, String> properties) {
+ return [
+ name: name,
+ properties: properties,
+ authProperties: toSqlProperties(properties)
+ ]
+ }
+
+ static String toSqlProperties(Map<String, String> properties) {
+ return properties.collect { key, propertyValue ->
+ "\"${key}\" = \"${propertyValue}\""
+ }.join(",\n")
+ }
+
+ private static String value(Map configs, String key) {
+ return configs.get(key)?.toString()?.trim()
+ }
+
+ private static boolean isBlank(String value) {
+ return value == null || value.isEmpty()
+ }
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_alter_resource_with_role.groovy b/regression-test/suites/object_storage_iam_p0/test_alter_resource_with_role.groovy
new file mode 100644
index 00000000000..28793ec085e
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_alter_resource_with_role.groovy
@@ -0,0 +1,234 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import groovy.json.JsonSlurper
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_alter_resource_with_role") {
+ if (!ObjectStorageIamTestUtils.isProvider(context.config.otherConfigs, "AWS")) {
+ logger.info("skip ${name} because objectStorageIamProvider is not AWS")
+ return
+ }
+ def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs)
+
+
+ if (isCloudMode()) {
+ logger.info("skip ${name} case, because it is cloud mode")
+ return
+ }
+
+ def tableName = "test_alter_resource_with_role"
+ def randomStr = UUID.randomUUID().toString().replace("-", "")
+ def resourceName = "alter_resource_${randomStr}"
+ def policyName = "alter_policy_${randomStr}"
+
+ def awsEndpoint = config.endpoint
+ def region = config.region
+ def bucket = config.bucket
+ def roleArn = config.roleArn
+ def externalId = config.externalId ?: ""
+ def prefix = config.prefix
+
+ def awsAccessKey = context.config.awsAccessKey
+ def awsSecretKey = context.config.awsSecretKey
+
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "${resourceName}"
+ PROPERTIES(
+ "type"="s3",
+ "AWS_ENDPOINT" = "${awsEndpoint}",
+ "AWS_REGION" = "${region}",
+ "AWS_BUCKET" = "${bucket}",
+ "AWS_ROOT_PATH" = "${prefix}/test_alter_resource_with_role/${randomStr}",
+ "AWS_ACCESS_KEY" = "error_ak",
+ "AWS_SECRET_KEY" = "error_sk",
+ "s3_validity_check" = "false"
+ );
+ """
+
+ sql """
+ CREATE STORAGE POLICY IF NOT EXISTS ${policyName}
+ PROPERTIES(
+ "storage_resource" = "${resourceName}",
+ "cooldown_ttl" = "1"
+ )
+ """
+
+ sql """
+ DROP TABLE IF EXISTS ${tableName} FORCE;
+ """
+
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ siteid INT DEFAULT '10',
+ citycode SMALLINT NOT NULL,
+ username VARCHAR(32) DEFAULT '',
+ pv BIGINT SUM DEFAULT '0'
+ )
+ AGGREGATE KEY(siteid, citycode, username)
+ DISTRIBUTED BY HASH(siteid) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "storage_policy" = "${policyName}"
+ )
+ """
+
+ sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1),
+ (2, 2, "yyy", 2),
+ (3, 3, "zzz", 3)
+ """
+
+ def result = sql """ SHOW RESOURCES WHERE NAME = "${resourceName}"; """
+ log.info("result:${result}")
+ assertTrue(!result.toString().contains(roleArn))
+ assertTrue(!result.toString().contains(externalId));
+ assertTrue(result.toString().contains("error_ak"));
+
+ // data_sizes is one arrayList<Long>, t is tablet
+ def fetchDataSize = {List<Long> data_sizes, Map<String, Object> t ->
+ def tabletId = t.TabletId
+ def meta_url = t.MetaUrl
+ def clos = { respCode, body ->
+ logger.info("test ttl expired resp Code {}", "${respCode}".toString())
+ assertEquals("${respCode}".toString(), "200")
+ String out = "${body}".toString()
+ logger.info("test fetchBeHttp, resp body: ${out}")
+ def obj = new JsonSlurper().parseText(out)
+ data_sizes[0] = obj.local_data_size
+ data_sizes[1] = obj.remote_data_size
+ }
+ meta_url = meta_url.replace("header", "data_size")
+
+ def i = meta_url.indexOf("/api")
+ def endPoint = meta_url.substring(0, i)
+ def metaUri = meta_url.substring(i)
+ logger.info("test fetchBeHttp, endpoint:${endPoint}, metaUri:${metaUri}")
+ i = endPoint.lastIndexOf('/')
+ endPoint = endPoint.substring(i + 1)
+
+ httpTest {
+ endpoint {endPoint}
+ uri metaUri
+ op "get"
+ check clos
+ }
+ }
+
+ List<Long> sizes = [-1, -1]
+ def tablets = sql_return_maparray """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty:${tablets}")
+ fetchDataSize(sizes, tablets[0])
+ def retry = 12
+ while (sizes[1] == 0 && retry-- > 0) {
+ log.info( "test remote size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql_return_maparray """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ }
+ assertTrue(sizes[1] == 0, "remote size expected is zero, but got ${sizes[1]}")
+ assertTrue(tablets.size() > 0)
+
+ // alter resource to arn role
+ sql """ truncate table ${tableName} """
+ sql """ALTER RESOURCE "${resourceName}"
+ PROPERTIES(
+ "AWS_ROLE_ARN" = "${roleArn}",
+ "AWS_EXTERNAL_ID" = "${externalId}"
+ );"""
+
+ result = sql """ SHOW RESOURCES WHERE NAME = "${resourceName}"; """
+ log.info("result:${result}")
+ assertTrue(result.toString().contains(roleArn))
+ assertTrue(result.toString().contains(externalId));
+ assertTrue(!result.toString().contains("error_ak"));
+
+ // report_tablet_interval_seconds = 60 sleep wait for report policy
+ sleep(120 * 1000)
+
+ sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1),
+ (4, 4, "yyy", 5),
+ (5, 5, "zzz", 5),
+ (6, 6, "xxx", 6)
+ """
+
+ sizes = [-1, -1]
+ tablets = sql_return_maparray """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty:${tablets}")
+ fetchDataSize(sizes, tablets[0])
+ retry = 100
+ while (sizes[1] == 0 && retry-- > 0) {
+ log.info( "test remote size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql_return_maparray """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ }
+ assertTrue(sizes[1] != 0, "remote size is still zero, maybe some error occurred")
+ assertTrue(tablets.size() > 0)
+ log.info( "test remote size not zero")
+
+ // alter resource to correct ak/sk
+ sql """ truncate table ${tableName} """
+ sql """ALTER RESOURCE "${resourceName}"
+ PROPERTIES(
+ "AWS_ACCESS_KEY" = "${awsAccessKey}",
+ "AWS_SECRET_KEY" = "${awsSecretKey}"
+ );"""
+
+ result = sql """ SHOW RESOURCES WHERE NAME = "${resourceName}"; """
+ log.info("result:${result}")
+ assertTrue(!result.toString().contains(roleArn))
+ assertTrue(!result.toString().contains(externalId));
+ assertTrue(result.toString().contains(awsAccessKey));
+
+ // report_tablet_interval_seconds = 60 sleep wait for report policy
+ sleep(120 * 1000)
+
+ sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1),
+ (4, 4, "yyy", 5),
+ (5, 5, "zzz", 5),
+ (6, 6, "xxx", 6),
+ (7, 7, "aaa", 8)
+ """
+
+ sizes = [-1, -1]
+ tablets = sql_return_maparray """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty:${tablets}")
+ fetchDataSize(sizes, tablets[0])
+ retry = 100
+ while (sizes[1] == 0 && retry-- > 0) {
+ log.info( "test remote size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql_return_maparray """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ }
+ assertTrue(sizes[1] != 0, "remote size is still zero, maybe some error occurred")
+ assertTrue(tablets.size() > 0)
+ log.info( "test remote size not zero")
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_alter_storage_vault_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_alter_storage_vault_iam.groovy
new file mode 100644
index 00000000000..b6c80a6b0c9
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_alter_storage_vault_iam.groovy
@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_alter_storage_vault_iam") {
+ if (!isCloudMode()) {
+ logger.info("skip ${name} case, because not cloud mode")
+ return
+ }
+
+ if (!enableStoragevault()) {
+ logger.info("skip ${name} case, because storage vault not enabled")
+ return
+ }
+ def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs)
+ if (config == null) {
+ logger.info("skip ${name} because objectStorageIamProvider is not configured")
+ return
+ }
+
+ def randomStr = UUID.randomUUID().toString().replace("-", "")
+ def s3VaultName = "object_storage_iam_" + randomStr
+ def firstAuthCase = config.authCases[0]
+ def awsAccessKey = context.config.awsAccessKey
+ def awsSecretKey = context.config.awsSecretKey
+
+ sql """
+ CREATE STORAGE VAULT IF NOT EXISTS ${s3VaultName}
+ PROPERTIES (
+ "type"="S3",
+ ${firstAuthCase.storageSqlProperties},
+ "s3.root.path" = "${config.prefix}/test_alter_storage_vault_iam/${s3VaultName}",
+ "s3.bucket" = "${config.bucket}",
+ "s3.external_endpoint" = "",
+ "use_path_style" = "false"
+ );
+ """
+
+ sql """
+ CREATE TABLE ${s3VaultName} (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME INTEGER NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "storage_vault_name" = ${s3VaultName}
+ )
+ """
+ sql """ insert into ${s3VaultName} values(1, 1); """
+ sql """ sync;"""
+ def result = sql """ select * from ${s3VaultName}; """
+ assertEquals(result.size(), 1);
+
+ def assertVaultProperties = { Map<String, String> expected, List<String> unexpected ->
+ def vaultInfos = sql "SHOW STORAGE VAULTS"
+ def vaultInfo = vaultInfos.find { it[0].equals(s3VaultName) }
+ assertTrue(vaultInfo != null, "storage vault ${s3VaultName} was not found")
+ def newProperties = vaultInfo[2]
+ logger.info("newProperties: ${newProperties}")
+ expected.each { key, value ->
+ assertTrue(newProperties.contains(value), "${key} was not updated")
+ }
+ unexpected.findAll { it != null && !it.isEmpty() }.each { value ->
+ assertFalse(newProperties.contains(value), "stale credential property was not removed")
+ }
+ }
+
+ def insertAndAssert = { int rowCount ->
+ sql "insert into ${s3VaultName} values(${rowCount}, ${rowCount})"
+ sql "sync"
+ result = sql "select * from ${s3VaultName}"
+ assertEquals(result.size(), rowCount)
+ }
+
+ if (config.provider == "AWS") {
+ sql """
+ ALTER STORAGE VAULT ${s3VaultName}
+ PROPERTIES (
+ "type"="S3",
+ "s3.access_key" = "${awsAccessKey}",
+ "s3.secret_key" = "${awsSecretKey}"
+ );
+ """
+ assertVaultProperties(["s3.access_key": awsAccessKey], [config.roleArn])
+ insertAndAssert(2)
+
+ sql """
+ ALTER STORAGE VAULT ${s3VaultName}
+ PROPERTIES (
+ "type"="S3",
+ ${firstAuthCase.authProperties}
+ );
+ """
+ assertVaultProperties(firstAuthCase.properties, [awsAccessKey])
+ insertAndAssert(3)
+ } else {
+ def impersonationAccounts = config.authCases.collect {
+ it.properties.get("gs.impersonation_service_account")
+ }.findAll { it != null }
+ // Omitted authentication is a CREATE default, not a request to reset credentials on ALTER.
+ // Exercise a storage-only update before switching to the explicitly configured providers.
+ sql """
+ ALTER STORAGE VAULT ${s3VaultName} PROPERTIES (
+ "type" = "S3",
+ "use_path_style" = "true"
+ )
+ """
+ assertVaultProperties(firstAuthCase.properties ?: ["gs.credential_provider_type": "DEFAULT"], [])
+ insertAndAssert(2)
+
+ def explicitCases = config.authCases.findAll { !it.properties.isEmpty() }
+ def transitionCases = explicitCases.isEmpty() ? [] : explicitCases + [explicitCases[0]]
+ transitionCases.eachWithIndex { authCase, index ->
+ logger.info("alter ${name} to ${authCase.name}")
+ def alterProperties = new LinkedHashMap(authCase.properties)
+ // A direct-credential transition must explicitly clear the old impersonation target.
+ alterProperties.putIfAbsent("gs.impersonation_service_account", "")
+ sql """
+ ALTER STORAGE VAULT ${s3VaultName}
+ PROPERTIES (
+ "type"="S3",
+ ${ObjectStorageIamTestUtils.toSqlProperties(alterProperties)}
+ );
+ """
+ def unexpected = authCase.properties.containsKey("gs.impersonation_service_account")
+ ? [] : impersonationAccounts
+ assertVaultProperties(authCase.properties, unexpected)
+ insertAndAssert(index * 2 + 3)
+
+ // Unrelated ALTERs must preserve COMPUTE_ENGINE and impersonation as well as DEFAULT.
+ sql """
+ ALTER STORAGE VAULT ${s3VaultName} PROPERTIES (
+ "type" = "S3",
+ "use_path_style" = "${index % 2 == 0 ? 'false' : 'true'}"
+ )
+ """
+ assertVaultProperties(authCase.properties, unexpected)
+ insertAndAssert(index * 2 + 4)
+ }
+ }
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_backup_restore_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_backup_restore_iam.groovy
new file mode 100644
index 00000000000..1d14c957054
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_backup_restore_iam.groovy
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_backup_restore_iam") {
+ if (isCloudMode()) {
+ logger.info("skip ${name} case, because cloud mode not support")
+ return
+ }
+ def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs)
+ if (config == null) {
+ logger.info("skip ${name} because objectStorageIamProvider is not configured")
+ return
+ }
+
+ String suiteName = "test_backup_restore_iam"
+
+ String dbName = "${suiteName}_db"
+ String tableName = "${suiteName}_table"
+
+ def syncer = getSyncer()
+
+ sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+ sql "DROP TABLE IF EXISTS ${dbName}.${tableName} FORCE;"
+ sql """
+ CREATE TABLE ${dbName}.${tableName} (
+ `id` LARGEINT NOT NULL,
+ `count` LARGEINT SUM DEFAULT "0")
+ AGGREGATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 2
+ PROPERTIES
+ (
+ "replication_num" = "1"
+ )
+ """
+
+ List<String> values = []
+ for (int i = 1; i <= 10; ++i) {
+ values.add("(${i}, ${i})")
+ }
+ sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}"
+ def result = sql "SELECT * FROM ${dbName}.${tableName}"
+ assertEquals(result.size(), values.size());
+
+ config.authCases.each { authCase ->
+ String suffix = UUID.randomUUID().toString().replace("-", "")
+ String repoName = "obj_iam_${authCase.name}_${suffix}"
+ String snapshotName = "obj_iam_${authCase.name}_${suffix.substring(0, 8)}"
+
+ logger.info("run ${name} with ${authCase.name}")
+ syncer.createObjectStorageIamRepository(repoName, config, authCase)
+
+ sql """
+ BACKUP SNAPSHOT ${dbName}.${snapshotName}
+ TO `${repoName}`
+ ON (${tableName})
+ """
+
+ syncer.waitSnapshotFinish(dbName)
+
+ def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+ assertTrue(snapshot != null)
+
+ sql "TRUNCATE TABLE ${dbName}.${tableName}"
+
+ sql """
+ RESTORE SNAPSHOT ${dbName}.${snapshotName}
+ FROM `${repoName}`
+ ON (`${tableName}`)
+ PROPERTIES
+ (
+ "backup_timestamp" = "${snapshot}",
+ "reserve_replica" = "true"
+ )
+ """
+
+ syncer.waitAllRestoreFinish(dbName)
+
+ result = sql "SELECT * FROM ${dbName}.${tableName}"
+ assertEquals(values.size(), result.size(), authCase.name)
+ sql "DROP REPOSITORY `${repoName}`"
+ }
+
+ sql "DROP TABLE ${dbName}.${tableName} FORCE"
+ sql "DROP DATABASE ${dbName} FORCE"
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_catalog_instance_profile.groovy b/regression-test/suites/object_storage_iam_p0/test_catalog_instance_profile.groovy
new file mode 100644
index 00000000000..a617fe34ffb
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_catalog_instance_profile.groovy
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import com.google.common.base.Strings;
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_catalog_instance_profile") {
+ if (!ObjectStorageIamTestUtils.isProvider(context.config.otherConfigs, "AWS")) {
+ logger.info("skip ${name} because objectStorageIamProvider is not AWS")
+ return
+ }
+
+ if (Strings.isNullOrEmpty(context.config.otherConfigs.get("hiveGlueInstanceProfileQueryTableName"))) {
+ return
+ }
+ sql """ ADMIN SET FRONTEND CONFIG ("aws_credentials_provider_version"="v2"); """
+ String hiveGlueQueryTableName = context.config.otherConfigs.get("hiveGlueInstanceProfileQueryTableName")
+ String hiveGlueExpectCounts = context.config.otherConfigs.get("hiveGlueInstanceProfileExpectCounts")
+ String icebergFsQueryTableName = context.config.otherConfigs.get("icebergFsInstanceProfileQueryTableName")
+ String icebergFsExpectCounts = context.config.otherConfigs.get("icebergFsInstanceProfileExpectCounts")
+ String icebergFsWarehouse = context.config.otherConfigs.get("icebergFsInstanceProfileWarehouse")
+ String region = context.config.otherConfigs.get("awsInstanceProfileRegion")
+ //query method
+ def createCatalogAndQuery = { catalogProps, catalogName, queryTableName, expectCounts ->
+ sql """drop catalog if exists ${catalogName}"""
+ sql """
+ ${catalogProps}
+ """
+ def result = sql """
+ select count(1) from ${catalogName}.${queryTableName};
+ """
+ println("result: ${result}")
+ def countValue = result[0][0]
+ assertTrue(countValue == expectCounts.toInteger())
+ sql """drop catalog if exists ${catalogName}"""
+ }
+ def assertCatalogAndQueryException = { catalogProps, catalogName, queryTableName ->
+ sql """drop catalog if exists ${catalogName}"""
+ sql """
+ ${catalogProps}
+ """
+ boolean failed = false
+ try {
+ sql """
+ switch ${catalogName};
+ """
+ sql """
+ show databases;
+ """
+ sql """
+ select count(1) from ${catalogName}.${queryTableName};
+ """
+ }catch (Exception e){
+ failed = true
+ }
+ assertTrue(failed)
+ }
+ String hiveGlueCatalogProps = """
+ create catalog hive_glue_catalog_instance_profile properties(
+ "type"="hms",
+ "hive.metastore.type"="glue",
+ "glue.region"="${region}",
+ "glue.endpoint" = "https://glue.${region}.amazonaws.com"
+ );
+ """
+ createCatalogAndQuery(hiveGlueCatalogProps, "hive_glue_catalog_instance_profile", hiveGlueQueryTableName, hiveGlueExpectCounts)
+ hiveGlueCatalogProps = """
+ create catalog hive_glue_catalog_instance_profile properties(
+ "type"="hms",
+ "hive.metastore.type"="glue",
+ "glue.credentials_provider_type"="INSTANCE_PROFILE",
+ "glue.region"="${region}",
+ "glue.endpoint" = "https://glue.${region}.amazonaws.com"
+ );
+ """
+ createCatalogAndQuery(hiveGlueCatalogProps, "hive_glue_catalog_instance_profile", hiveGlueQueryTableName, hiveGlueExpectCounts)
+ hiveGlueCatalogProps = """
+ create catalog hive_glue_catalog_instance_profile properties(
+ "type"="hms",
+ "hive.metastore.type"="glue",
+ "glue.credentials_provider_type"="CONTAINER",
+ "glue.region"="${region}",
+ "glue.endpoint" = "https://glue.${region}.amazonaws.com"
+ );
+ """
+ assertCatalogAndQueryException(hiveGlueCatalogProps,"hive_glue_catalog_instance_profile", hiveGlueQueryTableName)
+ String icebergFsCatalogProps = """
+ create catalog iceberg_fs_catalog_instance_profile properties(
+ "type"="iceberg",
+ "warehouse"="${icebergFsWarehouse}",
+ "iceberg.catalog.type"="hadoop",
+ "s3.region" = "${region}",
+ "s3.endpoint" = "https://s3.${region}.amazonaws.com"
+ );
+ """
+ createCatalogAndQuery(icebergFsCatalogProps, "iceberg_fs_catalog_instance_profile", icebergFsQueryTableName, icebergFsExpectCounts)
+ icebergFsCatalogProps = """
+ create catalog iceberg_fs_catalog_instance_profile properties(
+ "type"="iceberg",
+ "warehouse"="${icebergFsWarehouse}",
+ "iceberg.catalog.type"="hadoop",
+ "s3.credentials_provider_type"="INSTANCE_PROFILE",
+ "s3.region" = "${region}",
+ "s3.endpoint" = "https://s3.${region}.amazonaws.com"
+ );
+ """
+ createCatalogAndQuery(icebergFsCatalogProps, "iceberg_fs_catalog_instance_profile", icebergFsQueryTableName, icebergFsExpectCounts)
+ icebergFsCatalogProps = """
+ create catalog iceberg_fs_catalog_instance_profile properties(
+ "type"="iceberg",
+ "warehouse"="${icebergFsWarehouse}",
+ "iceberg.catalog.type"="hadoop",
+ "s3.credentials_provider_type"="CONTAINER",
+ "s3.region" = "${region}",
+ "s3.endpoint" = "https://s3.${region}.amazonaws.com"
+ );
+ """
+ assertCatalogAndQueryException(icebergFsCatalogProps,"iceberg_fs_catalog_instance_profile",icebergFsQueryTableName)
+
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_catalog_with_role.groovy b/regression-test/suites/object_storage_iam_p0/test_catalog_with_role.groovy
new file mode 100644
index 00000000000..d6eda63c8ef
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_catalog_with_role.groovy
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import com.google.common.base.Strings;
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_catalog_with_role") {
+ if (!ObjectStorageIamTestUtils.isProvider(context.config.otherConfigs, "AWS")) {
+ logger.info("skip ${name} because objectStorageIamProvider is not AWS")
+ return
+ }
+ if (Strings.isNullOrEmpty(context.config.otherConfigs.get("awsLakesRoleArn"))) {
+ logger.info("skip ${name} case, because awsRoleArn is null or empty")
+ return
+ }
+ //query method
+ def createCatalogAndQuery = { catalogProps, catalogName, queryTableName, expectCounts ->
+ sql """drop catalog if exists ${catalogName}"""
+ sql """
+ ${catalogProps}
+ """
+ def result = sql """
+ select count(1) from ${catalogName}.${queryTableName};
+ """
+ println("result: ${result}")
+ def countValue = result[0][0]
+ assertTrue(countValue == expectCounts.toInteger())
+ sql """drop catalog if exists ${catalogName}"""
+ }
+ String suiteName = "test_catalog_with_role"
+ String awsLakersRoleArn = context.config.otherConfigs.get("awsLakesRoleArn")
+ String awsLakesRegion = context.config.otherConfigs.get("awsLakesRegion")
+ String icebergFSOnS3CatalogWarehouse = context.config.otherConfigs.get("icebergFSOnS3CatalogWarehouse")
+ //String hmsMetastoreHost = context.config.hmsMetastoreHost
+ //String icebergHmsTableName = context.config.icebergHmsTableName
+ String icebergGlueTableName = context.config.otherConfigs.get("icebergGlueTableName")
+
+ String icebergFSOnS3TableName = context.config.otherConfigs.get("icebergFSOnS3TableName")
+ //String hiveHmsTableName = context.config.hiveHmsTableName
+ String hiveGlueTableName = context.config.otherConfigs.get("hiveGlueTableName")
+ String expectCounts = context.config.otherConfigs.get("awsLakesRoleArnQueryExpectCounts")
+
+ String awsS3Property = """
+ "s3.region" = "${awsLakesRegion}",
+ "s3.role_arn" = "${awsLakersRoleArn}"
+ """
+ String awsGlueProperties = """
+ "glue.region"="${awsLakesRegion}",
+ "glue.endpoint" = "https://glue.${awsLakesRegion}.amazonaws.com",
+ "glue.role_arn" = "${awsLakersRoleArn}"
+ """
+ // start test
+ String catalogName = "${suiteName}_iceberg_hadoop_on_s3"
+ //1. iceberg hadoop catalog on s
+ String icebergFSCatalogProps = """
+ create catalog if not exists ${catalogName} properties(
+ "type"="iceberg",
+ "iceberg.catalog.type"="hadoop",
+ "warehouse"="${icebergFSOnS3CatalogWarehouse}",
+ ${awsS3Property}
+ );
+ """
+ createCatalogAndQuery(icebergFSCatalogProps, catalogName, icebergFSOnS3TableName, expectCounts)
+ catalogName = "${suiteName}_iceberg_glue"
+ String icebergGlueCatalogProps = """
+ create catalog if not exists ${catalogName} properties(
+ "type"="iceberg",
+ "iceberg.catalog.type"="glue",
+ ${awsGlueProperties}
+ );
+ """
+ createCatalogAndQuery(icebergGlueCatalogProps, catalogName, icebergGlueTableName, expectCounts)
+ catalogName = "${suiteName}_hive_glue"
+ String hiveGlueCatalogProps = """
+ create catalog if not exists ${catalogName} properties (
+ "type"="hms",
+ "hive.metastore.type" = "glue",
+ ${awsGlueProperties}
+ );
+ """
+ createCatalogAndQuery(hiveGlueCatalogProps, catalogName, hiveGlueTableName, expectCounts)
+/*********** HMS ***************/
+// The following HMS-related cases are commented out because
+// they require a Docker environment for the Hive Metastore service.
+
+/*
+String catalogName = "${suiteName}iceberg_hive_on_s3"
+String icebergHmsCatalogProps = """
+ create catalog if not exists ${catalogName} properties (
+ "type" = "iceberg",
+ "iceberg.catalog.type" = "hive",
+ "hive.metastore.uris" = "thrift://${hmsMetastoreHost}:9083",
+ ${awsS3Property}
+ );
+"""
+createCatalogAndQuery(icebergHmsCatalogProps, catalogName, icebergHmsTableName, expectCounts)
+
+String hiveHmsCatalogProps = """
+ create catalog if not exists ${catalogName} properties (
+ "type" = "hms",
+ "hive.metastore.uris" = "thrift://${hmsMetastoreHost}:9083",
+ ${awsS3Property}
+ );
+"""
+createCatalogAndQuery(hiveHmsCatalogProps, catalogName, hiveHmsTableName, expectCounts)
+*/
+
+
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_export_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_export_iam.groovy
new file mode 100644
index 00000000000..046f909b22f
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_export_iam.groovy
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import com.google.common.base.Strings
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_export_iam") {
+ def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs)
+ if (config == null) {
+ logger.info("skip ${name} because objectStorageIamProvider is not configured")
+ return
+ }
+
+ def cases = config.authCases.collect { authCase ->
+ return [
+ name: authCase.name,
+ properties: """
+ ${authCase.storageSqlProperties}
+ """
+ ]
+ }
+
+ def tableName = "test_export_iam"
+ sql "DROP TABLE IF EXISTS ${tableName} FORCE"
+ sql """
+ CREATE TABLE ${tableName} (
+ siteid INT DEFAULT '10',
+ citycode SMALLINT NOT NULL,
+ username VARCHAR(32) DEFAULT '',
+ pv BIGINT SUM DEFAULT '0'
+ )
+ AGGREGATE KEY(siteid, citycode, username)
+ DISTRIBUTED BY HASH(siteid) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+ sql """INSERT INTO ${tableName}(siteid, citycode, username, pv) VALUES
+ (1, 1, "xxx", 1), (2, 2, "yyy", 2), (3, 3, "zzz", 3)"""
+
+ cases.each { testCase ->
+ def randomStr = UUID.randomUUID().toString().replace("-", "")
+ def label = "iam_export_${testCase.name}_${randomStr}"
+ // Keep cleanup inside this run's UUID directory.
+ def exportPrefix = "${config.prefix}/test_export_iam/${testCase.name}/${randomStr}/"
+
+ logger.info("run ${name} with ${testCase.name}")
+ sql """
+ EXPORT TABLE ${tableName} TO "${config.scheme}://${config.bucket}/${exportPrefix}"
+ PROPERTIES(
+ "label" = "${label}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "delete_existing_files" = "true"
+ )
+ WITH S3 (
+ ${testCase.properties}
+ )
+ """
+
+ def maxTryMs = 600000
+ def exportedUrl = ""
+ while (maxTryMs > 0) {
+ String[][] exportResult = sql "SHOW EXPORT WHERE LABEL = '${label}'"
+ if (exportResult[0][2] == "FINISHED") {
+ def json = parseJson(exportResult[0][11])
+ assert json instanceof List
+ assertEquals("1", json.fileNumber[0][0], testCase.name)
+ exportedUrl = json.url[0][0]
+ break
+ }
+ if (exportResult[0][2] == "CANCELLED") {
+ assertTrue(false, "Export ${label} cancelled for ${testCase.name}: ${exportResult}")
+ }
+ Thread.sleep(5000)
+ maxTryMs -= 5000
+ }
+ assertTrue(maxTryMs > 0, "Export ${label} timeout for ${testCase.name}")
+ assertFalse(Strings.isNullOrEmpty(exportedUrl), "Export returned no URL for ${testCase.name}")
+
+ def s3Prefix = "s3://${config.bucket}/"
+ assertTrue(exportedUrl.startsWith(s3Prefix), "Unexpected export URL: ${exportedUrl}")
+ def exportedFile = "${config.scheme}://${config.bucket}/" +
+ "${exportedUrl.substring(s3Prefix.length())}0.csv"
+ def result = sql """
+ SELECT COUNT(*) FROM s3(
+ "uri" = "${exportedFile}",
+ ${testCase.properties},
+ "format" = "csv",
+ "column_separator" = ",",
+ "use_path_style" = "false"
+ )
+ """
+ assertEquals(3, result[0][0], testCase.name)
+ }
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_external_catalog_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_external_catalog_iam.groovy
new file mode 100644
index 00000000000..941db827f36
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_external_catalog_iam.groovy
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_external_catalog_iam") {
+ def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs)
+ if (config == null) {
+ logger.info("skip ${name} because objectStorageIamProvider is not configured")
+ return
+ }
+
+ def cases = config.authCases.collect { authCase ->
+ return [
+ name: authCase.name,
+ warehouse: "${config.scheme}://${config.bucket}/${config.prefix}" +
+ "/test_external_catalog_iam/${authCase.name}",
+ properties: """
+ ${authCase.storageSqlProperties}
+ """
+ ]
+ }
+
+ cases.eachWithIndex { testCase, index ->
+ def randomStr = UUID.randomUUID().toString().replace("-", "")
+ def catalogName = "iam_iceberg_${index}_${randomStr}"
+ def databaseName = "iam_db"
+ def tableName = "iam_table"
+
+ logger.info("run ${name} with ${testCase.name}")
+ try {
+ sql """
+ CREATE CATALOG ${catalogName} PROPERTIES (
+ "type" = "iceberg",
+ "iceberg.catalog.type" = "hadoop",
+ "warehouse" = "${testCase.warehouse}/${randomStr}",
+ ${testCase.properties}
+ )
+ """
+ sql "CREATE DATABASE ${catalogName}.${databaseName}"
+ def databases = sql "SHOW DATABASES FROM ${catalogName}"
+ assertTrue(databases.any { it[0] == databaseName }, testCase.name)
+ sql """
+ CREATE TABLE ${catalogName}.${databaseName}.${tableName} (
+ id INT,
+ value STRING
+ )
+ PROPERTIES("file_format" = "parquet")
+ """
+ sql "INSERT INTO ${catalogName}.${databaseName}.${tableName} VALUES (1, 'one'), (2, 'two')"
+ def result = sql "SELECT COUNT(*) FROM ${catalogName}.${databaseName}.${tableName}"
+ assertEquals(2, result[0][0], testCase.name)
+ def rows = sql "SELECT id, value FROM ${catalogName}.${databaseName}.${tableName} ORDER BY id"
+ assertEquals([[1, 'one'], [2, 'two']], rows, testCase.name)
+ sql "DROP TABLE ${catalogName}.${databaseName}.${tableName}"
+ sql "DROP DATABASE ${catalogName}.${databaseName}"
+ // Exercise namespace deletion and recreation through Hadoop FileSystem as well.
+ sql "CREATE DATABASE ${catalogName}.${databaseName}"
+ sql "DROP DATABASE ${catalogName}.${databaseName}"
+ } finally {
+ sql "DROP CATALOG IF EXISTS ${catalogName}"
+ }
+ }
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_gcp_iam_property_validation.groovy b/regression-test/suites/object_storage_iam_p0/test_gcp_iam_property_validation.groovy
new file mode 100644
index 00000000000..2233a0b8811
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_gcp_iam_property_validation.groovy
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_gcp_iam_property_validation") {
+ ["ADC", "", "unknown"].each { providerType ->
+ test {
+ sql """
+ SELECT * FROM s3(
+ "uri" = "gs://validation-only-bucket/file.csv",
+ "provider" = "GCP",
+ "gs.credential_provider_type" = "${providerType}",
+ "format" = "csv"
+ ) LIMIT 1
+ """
+ exception "gs.credential_provider_type"
+ }
+ }
+
+ [" ", "invalid-account"].each { account ->
+ test {
+ sql """
+ SELECT * FROM s3(
+ "uri" = "gs://validation-only-bucket/file.csv",
+ "provider" = "GCP",
+ "gs.impersonation_service_account" = "${account}",
+ "format" = "csv"
+ ) LIMIT 1
+ """
+ exception "Invalid GCP service account email"
+ }
+ }
+
+ [
+ "gs.access_key": "gcp-access-key",
+ "gs.secret_key": "gcp-secret-key",
+ "gs.session_token": "gcp-session-token",
+ "s3.access_key": "access-key",
+ "s3.secret_key": "secret-key",
+ "s3.session_token": "session-token",
+ "s3.role_arn": "arn:aws:iam::123456789012:role/test-role",
+ "s3.external_id": "external-id",
+ "s3.credentials_provider_type": "INSTANCE_PROFILE"
+ ].each { key, value ->
+ test {
+ sql """
+ SELECT * FROM s3(
+ "uri" = "gs://validation-only-bucket/file.csv",
+ "provider" = "GCP",
+ "gs.credential_provider_type" = "DEFAULT",
+ "${key}" = "${value}",
+ "format" = "csv"
+ ) LIMIT 1
+ """
+ exception "cannot be used together"
+ }
+ }
+
+ test {
+ sql """
+ SELECT * FROM s3(
+ "uri" = "gs://validation-only-bucket/file.csv",
+ "provider" = "S3",
+ "gs.credential_provider_type" = "DEFAULT",
+ "format" = "csv"
+ ) LIMIT 1
+ """
+ exception "requires provider=GCP"
+ }
+
+ [
+ "gs.access_key": "access-key",
+ "gs.secret_key": "secret-key",
+ "gs.session_token": "session-token",
+ "s3.role_arn": "arn:aws:iam::123456789012:role/test-role"
+ ].each { key, value ->
+ test {
+ sql """
+ SELECT * FROM s3(
+ "uri" = "gs://validation-only-bucket/file.csv",
+ "provider" = "GCP",
+ "gs.credential_provider_type" = "ANONYMOUS",
+ "${key}" = "${value}",
+ "format" = "csv"
+ ) LIMIT 1
+ """
+ exception "cannot be used together"
+ }
+ }
+
+ test {
+ sql """
+ SELECT * FROM s3(
+ "uri" = "gs://validation-only-bucket/file.csv",
+ "gs.credential_provider_type" = "ANONYMOUS",
+ "gs.impersonation_service_account" = "target@project.iam.gserviceaccount.com",
+ "format" = "csv"
+ ) LIMIT 1
+ """
+ exception "ANONYMOUS cannot be used with gs.impersonation_service_account"
+ }
+
+ // Implicit DEFAULT must enforce the same conflicts as explicit DEFAULT, before any remote I/O.
+ ["gs.session_token", "s3.role_arn", "s3.credentials_provider_type"].each { key ->
+ test {
+ sql """
+ SELECT * FROM s3(
+ "uri" = "gs://validation-only-bucket/file.csv",
+ "provider" = "GCP",
+ "${key}" = "conflicting-credential",
+ "format" = "csv"
+ ) LIMIT 1
+ """
+ exception "cannot be used together"
+ }
+ }
+
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_gcp_tvf_anonymous.groovy b/regression-test/suites/object_storage_iam_p0/test_gcp_tvf_anonymous.groovy
new file mode 100644
index 00000000000..459bcb49592
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_gcp_tvf_anonymous.groovy
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_gcp_tvf_anonymous") {
+ def uri = context.config.otherConfigs.get("objectStorageIamGcpAnonymousUri")?.toString()?.trim()
+ if (!uri) {
+ logger.info("skip ${name} because objectStorageIamGcpAnonymousUri is not configured")
+ return
+ }
+ assertTrue(uri.startsWith("gs://"), "The anonymous fixture must be a gs:// URI")
+ def expectedRows = context.config.otherConfigs.get("objectStorageIamGcpAnonymousExpectedRows")
+ ?.toString()?.trim()
+ assertTrue(expectedRows != null && expectedRows.isLong() && expectedRows.toLong() > 0,
+ "Configure a positive objectStorageIamGcpAnonymousExpectedRows")
+
+ // Each query exercises FE object discovery/schema inference and a BE scan of the public file.
+ // Include the legacy S3-compatible spelling and provider-value case/whitespace normalization.
+ [
+ ["gs.credential_provider_type": "ANONYMOUS"],
+ ["gs.credential_provider_type": " anonymous "],
+ ["s3.credentials_provider_type": "ANONYMOUS"]
+ ].each { authProperties ->
+ def result = sql """
+ SELECT COUNT(*) FROM s3(
+ "uri" = "${uri}",
+ "provider" = "GCP",
+ ${ObjectStorageIamTestUtils.toSqlProperties(authProperties)},
+ "format" = "csv",
+ "compress_type" = "gz",
+ "column_separator" = "|"
+ )
+ """
+ assertEquals(expectedRows.toLong(), result[0][0], authProperties.toString())
+ }
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_gcp_vault_anonymous_validation.groovy b/regression-test/suites/object_storage_iam_p0/test_gcp_vault_anonymous_validation.groovy
new file mode 100644
index 00000000000..f3d699e678d
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_gcp_vault_anonymous_validation.groovy
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_gcp_vault_anonymous_validation") {
+ if (!isCloudMode() || !enableStoragevault()) {
+ logger.info("skip ${name} because cloud storage vaults are not enabled")
+ return
+ }
+
+ ["gs.credential_provider_type", "s3.credentials_provider_type", "AWS_CREDENTIALS_PROVIDER_TYPE"].each { key ->
+ def vaultName = "gcp_anonymous_${UUID.randomUUID().toString().replace('-', '')}"
+ test {
+ sql """
+ CREATE STORAGE VAULT ${vaultName} PROPERTIES (
+ "type" = "S3",
+ "provider" = "GCP",
+ "s3.endpoint" = "storage.googleapis.com",
+ "s3.region" = "us-east1",
+ "s3.bucket" = "validation-only-bucket",
+ "s3.root.path" = "validation-only-root",
+ "s3_validity_check" = "false",
+ "${key}" = " anonymous "
+ )
+ """
+ exception "Anonymous GCS authentication is not supported for storage vaults"
+ }
+ }
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_iceberg_s3tables_catalog_credentials_provider.groovy b/regression-test/suites/object_storage_iam_p0/test_iceberg_s3tables_catalog_credentials_provider.groovy
new file mode 100644
index 00000000000..bbef874246c
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_iceberg_s3tables_catalog_credentials_provider.groovy
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import com.google.common.base.Strings;
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_iceberg_s3tables_catalog_credentials_provider") {
+ if (!ObjectStorageIamTestUtils.isProvider(context.config.otherConfigs, "AWS")) {
+ logger.info("skip ${name} because objectStorageIamProvider is not AWS")
+ return
+ }
+
+ List<String> requiredConfigNames = [
+ "icebergS3TablesCatalog1Props",
+ "icebergS3TablesCatalog2Props",
+ "icebergS3TablesCatalog3Props",
+ "icebergS3TablesPartitionDbTableName",
+ "icebergS3TablesNonPartitionDbTableName",
+ "icebergS3TablesExpectCounts"
+ ]
+ if (requiredConfigNames.any { Strings.isNullOrEmpty(context.config.otherConfigs.get(it)) }) {
+ return
+ }
+
+ sql """ ADMIN SET FRONTEND CONFIG ("aws_credentials_provider_version"="v2"); """
+
+ String catalog1Props = context.config.otherConfigs.get("icebergS3TablesCatalog1Props")
+ String catalog2Props = context.config.otherConfigs.get("icebergS3TablesCatalog2Props")
+ String catalog3Props = context.config.otherConfigs.get("icebergS3TablesCatalog3Props")
+ String partitionDbTableName = context.config.otherConfigs.get("icebergS3TablesPartitionDbTableName")
+ String nonPartitionDbTableName = context.config.otherConfigs.get("icebergS3TablesNonPartitionDbTableName")
+ String expectCounts = context.config.otherConfigs.get("icebergS3TablesExpectCounts")
+ String expectedWebIdentityError = "Either the environment variable AWS_WEB_IDENTITY_TOKEN_FILE or the javaproperty aws.webIdentityTokenFile must be set."
+
+ def createCatalogAndQueryTables = {
+ catalogProps, catalogName, partitionQueryDbTableName, nonPartitionQueryDbTableName, expectedCounts ->
+ sql """drop catalog if exists ${catalogName}"""
+ try {
+ sql """
+ create catalog ${catalogName} properties (
+ ${catalogProps}
+ );
+ """
+ def partitionResult = sql """
+ select count(1) from ${catalogName}.${partitionQueryDbTableName};
+ """
+ def nonPartitionResult = sql """
+ select count(1) from ${catalogName}.${nonPartitionQueryDbTableName};
+ """
+ def partitionCount = partitionResult[0][0]
+ def nonPartitionCount = nonPartitionResult[0][0]
+ assertTrue(partitionCount == expectedCounts.toInteger())
+ assertTrue(nonPartitionCount == expectedCounts.toInteger())
+ assertTrue(partitionCount == nonPartitionCount)
+ } finally {
+ sql """drop catalog if exists ${catalogName}"""
+ }
+ }
+
+ def createCatalogAndShowDatabasesException = { catalogProps, catalogName, expectedMessage ->
+ sql """drop catalog if exists ${catalogName}"""
+ try {
+ sql """
+ create catalog ${catalogName} properties (
+ ${catalogProps}
+ );
+ """
+ sql """switch ${catalogName};"""
+ try {
+ sql """show databases;"""
+ assertTrue(false)
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains(expectedMessage))
+ }
+ } finally {
+ sql """drop catalog if exists ${catalogName}"""
+ }
+ }
+
+ createCatalogAndQueryTables(
+ catalog1Props,
+ "iceberg_s3tables_catalog_1",
+ partitionDbTableName,
+ nonPartitionDbTableName,
+ expectCounts)
+ createCatalogAndQueryTables(
+ catalog2Props,
+ "iceberg_s3tables_catalog_2",
+ partitionDbTableName,
+ nonPartitionDbTableName,
+ expectCounts)
+ createCatalogAndShowDatabasesException(
+ catalog3Props,
+ "iceberg_s3tables_catalog_3",
+ expectedWebIdentityError)
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_load_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_load_iam.groovy
new file mode 100644
index 00000000000..e9b1517305f
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_load_iam.groovy
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_load_iam") {
+ def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs)
+ if (config == null) {
+ logger.info("skip ${name} because objectStorageIamProvider is not configured")
+ return
+ }
+
+ def cases = config.authCases.collect { authCase ->
+ return [
+ name: authCase.name,
+ properties: """
+ ${authCase.storageSqlProperties}
+ """
+ ]
+ }
+ if (config.provider == "AWS") {
+ cases.add(0, [
+ name: "aws_role_legacy_properties",
+ properties: """
+ "AWS_ENDPOINT" = "${config.endpoint}",
+ "AWS_REGION" = "${config.region}",
+ "AWS_ROLE_ARN" = "${config.roleArn}",
+ "AWS_EXTERNAL_ID" = "${config.externalId ?: ''}"
+ """
+ ])
+ cases[1].name = "aws_role_s3_properties"
+ }
+
+ cases.eachWithIndex { testCase, index ->
+ def randomStr = UUID.randomUUID().toString().replace("-", "")
+ def loadLabel = "iam_load_${index}_${randomStr}"
+ def tableName = "test_iam_load_${index}_${randomStr}"
+
+ sql "DROP TABLE IF EXISTS ${tableName} FORCE"
+ sql """
+ CREATE TABLE ${tableName} (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME VARCHAR(25) NOT NULL,
+ C_ADDRESS VARCHAR(40) NOT NULL,
+ C_NATIONKEY INTEGER NOT NULL,
+ C_PHONE CHAR(15) NOT NULL,
+ C_ACCTBAL DECIMAL(15, 2) NOT NULL,
+ C_MKTSEGMENT CHAR(10) NOT NULL,
+ C_COMMENT VARCHAR(117) NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ logger.info("run ${name} with ${testCase.name}")
+ sql """
+ LOAD LABEL ${loadLabel} (
+ DATA INFILE("${config.scheme}://${config.bucket}/${config.dataPath}")
+ INTO TABLE ${tableName}
+ COLUMNS TERMINATED BY "|"
+ (c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_mktsegment, c_comment)
+ )
+ WITH S3 (
+ ${testCase.properties},
+ "compress_type" = "GZ"
+ )
+ PROPERTIES(
+ "timeout" = "600",
+ "exec_mem_limit" = "8589934592"
+ )
+ """
+
+ def maxTryMs = 600000
+ while (maxTryMs > 0) {
+ String[][] loadResult = sql """SHOW LOAD WHERE LABEL = "${loadLabel}"
+ ORDER BY CREATETIME DESC LIMIT 1"""
+ if (loadResult[0][2] == "FINISHED") {
+ break
+ }
+ if (loadResult[0][2] == "CANCELLED") {
+ assertTrue(false, "Load ${loadLabel} cancelled for ${testCase.name}: ${loadResult}")
+ }
+ Thread.sleep(5000)
+ maxTryMs -= 5000
+ }
+ assertTrue(maxTryMs > 0, "Load ${loadLabel} timeout for ${testCase.name}")
+
+ def result = sql "SELECT COUNT(*) FROM ${tableName}"
+ assertEquals(1500, result[0][0], testCase.name)
+ }
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_resource_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_resource_iam.groovy
new file mode 100644
index 00000000000..7433179e902
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_resource_iam.groovy
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import groovy.json.JsonSlurper
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_resource_iam") {
+ if (isCloudMode()) {
+ logger.info("skip ${name} case, because cloud mode not support")
+ return
+ }
+ def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs)
+ if (config == null) {
+ logger.info("skip ${name} because objectStorageIamProvider is not configured")
+ return
+ }
+
+ def randomStr = UUID.randomUUID().toString().replace("-", "")
+ def testObjects = []
+ config.authCases.eachWithIndex { authCase, index ->
+ def tableName = "test_resource_iam_${index}"
+ def resourceName = "resource_${authCase.name}_${randomStr}"
+ def policyName = "policy_${authCase.name}_${randomStr}"
+
+ logger.info("run ${name} with ${authCase.name}")
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "${resourceName}"
+ PROPERTIES(
+ "type" = "s3",
+ ${authCase.storageSqlProperties},
+ "s3.bucket" = "${config.bucket}",
+ "s3.root.path" = "${config.prefix}/test_resource_iam/${authCase.name}/${randomStr}",
+ "s3.connection.maximum" = "50",
+ "s3.connection.request.timeout" = "3000",
+ "s3.connection.timeout" = "1000",
+ "s3_validity_check" = "true"
+ );
+ """
+
+ sql """
+ CREATE STORAGE POLICY IF NOT EXISTS ${policyName}
+ PROPERTIES(
+ "storage_resource" = "${resourceName}",
+ "cooldown_ttl" = "1"
+ )
+ """
+
+ sql "DROP TABLE IF EXISTS ${tableName} FORCE"
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ siteid INT DEFAULT '10',
+ citycode SMALLINT NOT NULL,
+ username VARCHAR(32) DEFAULT '',
+ pv BIGINT SUM DEFAULT '0'
+ )
+ AGGREGATE KEY(siteid, citycode, username)
+ DISTRIBUTED BY HASH(siteid) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "storage_policy" = "${policyName}"
+ )
+ """
+
+ sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1),
+ (2, 2, "yyy", 2),
+ (3, 3, "zzz", 3)
+ """
+ testObjects.add([name: authCase.name, tableName: tableName])
+ }
+
+ // data_sizes is one arrayList<Long>, t is tablet
+ def fetchDataSize = {List<Long> data_sizes, Map<String, Object> t ->
+ def tabletId = t.TabletId
+ def meta_url = t.MetaUrl
+ def clos = { respCode, body ->
+ logger.info("test ttl expired resp Code {}", "${respCode}".toString())
+ assertEquals("${respCode}".toString(), "200")
+ String out = "${body}".toString()
+ def obj = new JsonSlurper().parseText(out)
+ data_sizes[0] = obj.local_data_size
+ data_sizes[1] = obj.remote_data_size
+ }
+ meta_url = meta_url.replace("header", "data_size")
+
+ def i = meta_url.indexOf("/api")
+ def endPoint = meta_url.substring(0, i)
+ def metaUri = meta_url.substring(i)
+ logger.info("test fetchBeHttp, endpoint:${endPoint}, metaUri:${metaUri}")
+ i = endPoint.lastIndexOf('/')
+ endPoint = endPoint.substring(i + 1)
+
+ httpTest {
+ endpoint {endPoint}
+ uri metaUri
+ op "get"
+ check clos
+ }
+ }
+
+ sleep(60000)
+
+ testObjects.each { testObject ->
+ List<Long> sizes = [-1, -1]
+ def tablets = sql_return_maparray "SHOW TABLETS FROM ${testObject.tableName}"
+ log.info("test tablets not empty for ${testObject.name}: ${tablets}")
+ fetchDataSize(sizes, tablets[0])
+ def retry = 100
+ while (sizes[1] == 0 && retry-- > 0) {
+ log.info("test remote size is zero for ${testObject.name}, sleep 10s")
+ sleep(10000)
+ tablets = sql_return_maparray "SHOW TABLETS FROM ${testObject.tableName}"
+ fetchDataSize(sizes, tablets[0])
+ }
+ assertTrue(sizes[1] != 0,
+ "remote size is still zero for ${testObject.name}, maybe some error occurred")
+ assertTrue(tablets.size() > 0)
+ log.info("test remote size not zero for ${testObject.name}")
+ }
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_select_into_outfile_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_select_into_outfile_iam.groovy
new file mode 100644
index 00000000000..e71067ab287
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_select_into_outfile_iam.groovy
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_select_into_outfile_iam") {
+ def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs)
+ if (config == null) {
+ logger.info("skip ${name} because objectStorageIamProvider is not configured")
+ return
+ }
+
+ def randomStr = UUID.randomUUID().toString().replace("-", "")
+ def tableName = "test_select_into_outfile_iam"
+
+ sql """ drop table if exists ${tableName} force;"""
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ siteid INT DEFAULT '10',
+ citycode SMALLINT NOT NULL,
+ username VARCHAR(32) DEFAULT '',
+ pv BIGINT SUM DEFAULT '0'
+ )
+ AGGREGATE KEY(siteid, citycode, username)
+ DISTRIBUTED BY HASH(siteid) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ )
+ """
+
+ sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1),
+ (2, 2, "yyy", 2),
+ (3, 3, "zzz", 3)
+ """
+ sql """sync;"""
+
+ def expectedRows = sql """
+ SELECT CAST(siteid AS STRING), CAST(citycode AS STRING), username, CAST(pv AS STRING)
+ FROM ${tableName} ORDER BY siteid
+ """
+
+ config.authCases.each { authCase ->
+ logger.info("run ${name} with ${authCase.name}")
+ def outfilePath = "${config.scheme}://${config.bucket}/${config.prefix}/" +
+ "test_select_into_outfile_iam/${authCase.name}/${randomStr}"
+ sql """
+ SELECT * FROM ${tableName}
+ INTO OUTFILE "${outfilePath}"
+ FORMAT AS CSV
+ PROPERTIES(
+ "column_separator" = ",",
+ ${authCase.storageSqlProperties}
+ );
+ """
+ def exportedRows = sql """
+ SELECT c1, c2, c3, c4 FROM s3(
+ "uri" = "${outfilePath}*.csv",
+ ${authCase.storageSqlProperties},
+ "format" = "csv",
+ "column_separator" = ","
+ ) ORDER BY CAST(c1 AS INT)
+ """
+ assertEquals(expectedRows, exportedRows, authCase.name)
+ }
+}
diff --git a/regression-test/suites/object_storage_iam_p0/test_storage_vault_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_storage_vault_iam.groovy
new file mode 100644
index 00000000000..9f89873ef2c
--- /dev/null
+++ b/regression-test/suites/object_storage_iam_p0/test_storage_vault_iam.groovy
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.util.ObjectStorageIamTestUtils
+
+suite("test_storage_vault_iam") {
+ if (!isCloudMode()) {
+ logger.info("skip ${name} because it requires cloud mode")
+ return
+ }
+ if (!enableStoragevault()) {
+ logger.info("skip ${name} because storage vault is disabled")
+ return
+ }
+
+ def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs)
+ if (config == null) {
+ logger.info("skip ${name} because objectStorageIamProvider is not configured")
+ return
+ }
+
+ def cases = config.authCases.collect { authCase ->
+ return [
+ name: authCase.name,
+ properties: """
+ ${authCase.storageSqlProperties},
+ "s3.bucket" = "${config.bucket}",
+ "s3.external_endpoint" = "",
+ "use_path_style" = "false"
+ """
+ ]
+ }
+
+ cases.eachWithIndex { testCase, index ->
+ def randomStr = UUID.randomUUID().toString().replace("-", "")
+ def vaultName = "iam_vault_${index}_${randomStr}"
+ def tableName = "test_iam_vault_${index}_${randomStr}"
+
+ logger.info("run ${name} with ${testCase.name}")
+ sql """
+ CREATE STORAGE VAULT ${vaultName}
+ PROPERTIES (
+ "type" = "S3",
+ "s3.root.path" = "${config.prefix}/test_storage_vault_iam/${testCase.name}/${randomStr}",
+ ${testCase.properties}
+ )
+ """
+ sql """
+ CREATE TABLE ${tableName} (
+ id INT NOT NULL,
+ value INT NOT NULL
+ )
+ DUPLICATE KEY(id, value)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES(
+ "replication_num" = "1",
+ "storage_vault_name" = "${vaultName}"
+ )
+ """
+ sql "INSERT INTO ${tableName} VALUES (1, 1)"
+ sql "SYNC"
+ def result = sql "SELECT * FROM ${tableName}"
+ assertEquals(1, result.size(), testCase.name)
+ assertEquals(1, result[0][0], testCase.name)
... 157 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org