You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by nj...@apache.org on 2022/12/19 01:51:47 UTC
[arrow-ballista] branch master updated: Support Alibaba Cloud OSS with ObjectStore (#567)
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-ballista.git
The following commit(s) were added to refs/heads/master by this push:
new a2661e49 Support Alibaba Cloud OSS with ObjectStore (#567)
a2661e49 is described below
commit a2661e49235526f7421ba86fe23776ff7975197b
Author: r.4ntix <an...@antix.blue>
AuthorDate: Mon Dec 19 09:51:41 2022 +0800
Support Alibaba Cloud OSS with ObjectStore (#567)
---
ballista/core/Cargo.toml | 2 +-
ballista/core/src/utils.rs | 13 ++++++++++++-
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml
index 1166c99e..0d8621f0 100644
--- a/ballista/core/Cargo.toml
+++ b/ballista/core/Cargo.toml
@@ -58,7 +58,7 @@ hashbrown = "0.13"
itertools = "0.10"
libloading = "0.7.3"
log = "0.4"
-object_store = "0.5.0"
+object_store = "0.5.2"
once_cell = "1.9.0"
parking_lot = "0.12"
diff --git a/ballista/core/src/utils.rs b/ballista/core/src/utils.rs
index 8eb77226..9fb62cd2 100644
--- a/ballista/core/src/utils.rs
+++ b/ballista/core/src/utils.rs
@@ -99,13 +99,24 @@ impl ObjectStoreProvider for FeatureBasedObjectStoreProvider {
#[cfg(feature = "s3")]
{
- if url.to_string().starts_with("s3://") {
+ if url.as_str().starts_with("s3://") {
if let Some(bucket_name) = url.host_str() {
let store = AmazonS3Builder::from_env()
.with_bucket_name(bucket_name)
.build()?;
return Ok(Arc::new(store));
}
+ // Support Alibaba Cloud OSS
+ // Use S3 compatibility mode to access Alibaba Cloud OSS
+ // The `AWS_ENDPOINT` should have bucket name included
+ } else if url.as_str().starts_with("oss://") {
+ if let Some(bucket_name) = url.host_str() {
+ let store = AmazonS3Builder::from_env()
+ .with_virtual_hosted_style_request(true)
+ .with_bucket_name(bucket_name)
+ .build()?;
+ return Ok(Arc::new(store));
+ }
}
}