You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2022/10/01 15:09:06 UTC
[arrow-rs] branch master updated: Handle S3 virtual host request type (#2782)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 0052d2572 Handle S3 virtual host request type (#2782)
0052d2572 is described below
commit 0052d2572ba9d70d6832221d48feab0286d0312a
Author: askoa <11...@users.noreply.github.com>
AuthorDate: Sat Oct 1 11:09:00 2022 -0400
Handle S3 virtual host request type (#2782)
* include s2 virtual host request type
* formatting changes
* fix issues highlighted in PR comments
* initialize bucket_endpoint
* some imporments on endpoint initialization
* fix issue in initalizing bucket_endpoint
* incorporating PR comments
* incorporate PR comments
* fix typo in comment
Co-authored-by: askoa <as...@local>
---
object_store/src/aws/client.rs | 12 ++++------
object_store/src/aws/mod.rs | 54 ++++++++++++++++++++++++++++++++++++++----
2 files changed, 54 insertions(+), 12 deletions(-)
diff --git a/object_store/src/aws/client.rs b/object_store/src/aws/client.rs
index 5ec9390ec..29621626c 100644
--- a/object_store/src/aws/client.rs
+++ b/object_store/src/aws/client.rs
@@ -197,6 +197,7 @@ pub struct S3Config {
pub region: String,
pub endpoint: String,
pub bucket: String,
+ pub bucket_endpoint: String,
pub credentials: CredentialProvider,
pub retry_config: RetryConfig,
pub allow_http: bool,
@@ -204,7 +205,7 @@ pub struct S3Config {
impl S3Config {
fn path_url(&self, path: &Path) -> String {
- format!("{}/{}/{}", self.endpoint, self.bucket, encode_path(path))
+ format!("{}/{}", self.bucket_endpoint, encode_path(path))
}
}
@@ -342,7 +343,7 @@ impl S3Client {
token: Option<&str>,
) -> Result<(ListResult, Option<String>)> {
let credential = self.get_credential().await?;
- let url = format!("{}/{}", self.config.endpoint, self.config.bucket);
+ let url = self.config.bucket_endpoint.clone();
let mut query = Vec::with_capacity(4);
@@ -398,12 +399,7 @@ impl S3Client {
pub async fn create_multipart(&self, location: &Path) -> Result<MultipartId> {
let credential = self.get_credential().await?;
- let url = format!(
- "{}/{}/{}?uploads=",
- self.config.endpoint,
- self.config.bucket,
- encode_path(location)
- );
+ let url = format!("{}?uploads=", self.config.path_url(location),);
let response = self
.client
diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs
index a6026032e..e3510b3e2 100644
--- a/object_store/src/aws/mod.rs
+++ b/object_store/src/aws/mod.rs
@@ -357,6 +357,7 @@ pub struct AmazonS3Builder {
retry_config: RetryConfig,
allow_http: bool,
imdsv1_fallback: bool,
+ virtual_hosted_style_request: bool,
metadata_endpoint: Option<String>,
}
@@ -446,10 +447,13 @@ impl AmazonS3Builder {
}
/// Sets the endpoint for communicating with AWS S3. Default value
- /// is based on region.
+ /// is based on region. The `endpoint` field should be consistent with
+ /// the field `virtual_hosted_style_request'.
///
/// For example, this might be set to `"http://localhost:4566:`
/// for testing against a localstack instance.
+ /// If `virtual_hosted_style_request` is set to true then `endpoint`
+ /// should have bucket name included.
pub fn with_endpoint(mut self, endpoint: impl Into<String>) -> Self {
self.endpoint = Some(endpoint.into());
self
@@ -469,6 +473,23 @@ impl AmazonS3Builder {
self
}
+ /// Sets if virtual hosted style request has to be used.
+ /// If `virtual_hosted_style_request` is :
+ /// * false (default): Path style request is used
+ /// * true: Virtual hosted style request is used
+ ///
+ /// If the `endpoint` is provided then it should be
+ /// consistent with `virtual_hosted_style_request`.
+ /// i.e. if `virtual_hosted_style_request` is set to true
+ /// then `endpoint` should have bucket name included.
+ pub fn with_virtual_hosted_style_request(
+ mut self,
+ virtual_hosted_style_request: bool,
+ ) -> Self {
+ self.virtual_hosted_style_request = virtual_hosted_style_request;
+ self
+ }
+
/// Set the retry configuration
pub fn with_retry(mut self, retry_config: RetryConfig) -> Self {
self.retry_config = retry_config;
@@ -568,14 +589,29 @@ impl AmazonS3Builder {
},
};
- let endpoint = self
- .endpoint
- .unwrap_or_else(|| format!("https://s3.{}.amazonaws.com", region));
+ let endpoint: String;
+ let bucket_endpoint: String;
+
+ //If `endpoint` is provided then its assumed to be consistent with
+ // `virutal_hosted_style_request`. i.e. if `virtual_hosted_style_request` is true then
+ // `endpoint` should have bucket name included.
+ if self.virtual_hosted_style_request {
+ endpoint = self.endpoint.unwrap_or_else(|| {
+ format!("https://{}.s3.{}.amazonaws.com", bucket, region)
+ });
+ bucket_endpoint = endpoint.clone();
+ } else {
+ endpoint = self
+ .endpoint
+ .unwrap_or_else(|| format!("https://s3.{}.amazonaws.com", region));
+ bucket_endpoint = format!("{}/{}", endpoint, bucket);
+ }
let config = S3Config {
region,
endpoint,
bucket,
+ bucket_endpoint,
credentials,
retry_config: self.retry_config,
allow_http: self.allow_http,
@@ -674,6 +710,16 @@ mod tests {
config
};
+ let config = if let Some(virtual_hosted_style_request) =
+ env::var("OBJECT_STORE_VIRTUAL_HOSTED_STYLE_REQUEST").ok()
+ {
+ config.with_virtual_hosted_style_request(
+ virtual_hosted_style_request.trim().parse().unwrap(),
+ )
+ } else {
+ config
+ };
+
config
}
}};