You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2023/05/18 07:51:03 UTC
[arrow-rs] branch master updated: Remove AWS_PROFILE support (#4238)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new a5c1a33af Remove AWS_PROFILE support (#4238)
a5c1a33af is described below
commit a5c1a33af88d56bcd9a297d77305f033598e2428
Author: Raphael Taylor-Davies <17...@users.noreply.github.com>
AuthorDate: Thu May 18 08:50:56 2023 +0100
Remove AWS_PROFILE support (#4238)
---
.github/workflows/object_store.yml | 2 -
object_store/Cargo.toml | 8 ---
object_store/src/aws/mod.rs | 127 +----------------------------------
object_store/src/aws/profile.rs | 133 -------------------------------------
4 files changed, 1 insertion(+), 269 deletions(-)
diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml
index df43ae3bf..5ae9d2d9c 100644
--- a/.github/workflows/object_store.yml
+++ b/.github/workflows/object_store.yml
@@ -56,8 +56,6 @@ jobs:
run: cargo clippy -- -D warnings
- name: Run clippy with aws feature
run: cargo clippy --features aws -- -D warnings
- - name: Run clippy with aws_profile feature
- run: cargo clippy --features aws_profile -- -D warnings
- name: Run clippy with gcp feature
run: cargo clippy --features gcp -- -D warnings
- name: Run clippy with azure feature
diff --git a/object_store/Cargo.toml b/object_store/Cargo.toml
index c6b89fa23..bd9c973e0 100644
--- a/object_store/Cargo.toml
+++ b/object_store/Cargo.toml
@@ -53,11 +53,6 @@ reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"
ring = { version = "0.16", default-features = false, features = ["std"], optional = true }
rustls-pemfile = { version = "1.0", default-features = false, optional = true }
-# AWS Profile support
-aws-types = { version = "0.55", optional = true }
-aws-credential-types = { version = "0.55", optional = true }
-aws-config = { version = "0.55", optional = true }
-
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
tokio = { version = "1.25.0", features = ["sync", "macros", "rt", "time", "io-util", "fs"] }
@@ -74,9 +69,6 @@ gcp = ["cloud", "rustls-pemfile"]
aws = ["cloud"]
http = ["cloud"]
-# Experimental support for AWS_PROFILE
-aws_profile = ["aws", "aws-config", "aws-types", "aws-credential-types"]
-
[dev-dependencies] # In alphabetical order
dotenv = "0.15.0"
tempfile = "3.1.0"
diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs
index a10561ba6..a7f43d153 100644
--- a/object_store/src/aws/mod.rs
+++ b/object_store/src/aws/mod.rs
@@ -64,9 +64,6 @@ mod checksum;
mod client;
mod credential;
-#[cfg(feature = "aws_profile")]
-mod profile;
-
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
//
// Do not URI-encode any of the unreserved characters that RFC 3986 defines:
@@ -106,9 +103,6 @@ enum Error {
#[snafu(display("Missing SecretAccessKey"))]
MissingSecretAccessKey,
- #[snafu(display("Profile support requires aws_profile feature"))]
- MissingProfileFeature,
-
#[snafu(display("ETag Header missing from response"))]
MissingEtag,
@@ -427,8 +421,6 @@ pub struct AmazonS3Builder {
checksum_algorithm: Option<ConfigValue<Checksum>>,
/// Metadata endpoint, see <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html>
metadata_endpoint: Option<String>,
- /// Profile name, see <https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html>
- profile: Option<String>,
/// Client options
client_options: ClientOptions,
/// Credentials
@@ -559,13 +551,6 @@ pub enum AmazonS3ConfigKey {
/// - `metadata_endpoint`
MetadataEndpoint,
- /// AWS profile name
- ///
- /// Supported keys:
- /// - `aws_profile`
- /// - `profile`
- Profile,
-
/// Client options
Client(ClientConfigKey),
}
@@ -583,7 +568,6 @@ impl AsRef<str> for AmazonS3ConfigKey {
Self::VirtualHostedStyleRequest => "aws_virtual_hosted_style_request",
Self::DefaultRegion => "aws_default_region",
Self::MetadataEndpoint => "aws_metadata_endpoint",
- Self::Profile => "aws_profile",
Self::UnsignedPayload => "aws_unsigned_payload",
Self::Checksum => "aws_checksum_algorithm",
Self::Client(opt) => opt.as_ref(),
@@ -612,7 +596,6 @@ impl FromStr for AmazonS3ConfigKey {
"aws_virtual_hosted_style_request" | "virtual_hosted_style_request" => {
Ok(Self::VirtualHostedStyleRequest)
}
- "aws_profile" | "profile" => Ok(Self::Profile),
"aws_imdsv1_fallback" | "imdsv1_fallback" => Ok(Self::ImdsV1Fallback),
"aws_metadata_endpoint" | "metadata_endpoint" => Ok(Self::MetadataEndpoint),
"aws_unsigned_payload" | "unsigned_payload" => Ok(Self::UnsignedPayload),
@@ -643,7 +626,6 @@ impl AmazonS3Builder {
/// * `AWS_SESSION_TOKEN` -> token
/// * `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` -> <https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html>
/// * `AWS_ALLOW_HTTP` -> set to "true" to permit HTTP connections without TLS
- /// * `AWS_PROFILE` -> set profile name, requires `aws_profile` feature enabled
/// # Example
/// ```
/// use object_store::aws::AmazonS3Builder;
@@ -727,7 +709,6 @@ impl AmazonS3Builder {
AmazonS3ConfigKey::MetadataEndpoint => {
self.metadata_endpoint = Some(value.into())
}
- AmazonS3ConfigKey::Profile => self.profile = Some(value.into()),
AmazonS3ConfigKey::UnsignedPayload => self.unsigned_payload.parse(value),
AmazonS3ConfigKey::Checksum => {
self.checksum_algorithm = Some(ConfigValue::Deferred(value.into()))
@@ -794,7 +775,6 @@ impl AmazonS3Builder {
Some(self.virtual_hosted_style_request.to_string())
}
AmazonS3ConfigKey::MetadataEndpoint => self.metadata_endpoint.clone(),
- AmazonS3ConfigKey::Profile => self.profile.clone(),
AmazonS3ConfigKey::UnsignedPayload => Some(self.unsigned_payload.to_string()),
AmazonS3ConfigKey::Checksum => {
self.checksum_algorithm.as_ref().map(ToString::to_string)
@@ -982,24 +962,6 @@ impl AmazonS3Builder {
self
}
- /// Set the AWS profile name, see <https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html>
- ///
- /// This makes use of [aws-config] to provide credentials and therefore requires
- /// the `aws-profile` feature to be enabled
- ///
- /// It is strongly encouraged that users instead make use of a credential manager
- /// such as [aws-vault] not only to avoid the significant additional dependencies,
- /// but also to avoid storing credentials in [plain text on disk]
- ///
- /// [aws-config]: https://docs.rs/aws-config
- /// [aws-vault]: https://github.com/99designs/aws-vault
- /// [plain text on disk]: https://99designs.com.au/blog/engineering/aws-vault/
- #[cfg(feature = "aws_profile")]
- pub fn with_profile(mut self, profile: impl Into<String>) -> Self {
- self.profile = Some(profile.into());
- self
- }
-
/// Create a [`AmazonS3`] instance from the provided values,
/// consuming `self`.
pub fn build(mut self) -> Result<AmazonS3> {
@@ -1007,14 +969,8 @@ impl AmazonS3Builder {
self.parse_url(&url)?;
}
- let region = match (self.region, self.profile.clone()) {
- (Some(region), _) => Some(region),
- (None, Some(profile)) => profile_region(profile),
- (None, None) => None,
- };
-
let bucket = self.bucket_name.context(MissingBucketNameSnafu)?;
- let region = region.context(MissingRegionSnafu)?;
+ let region = self.region.context(MissingRegionSnafu)?;
let checksum = self.checksum_algorithm.map(|x| x.get()).transpose()?;
let credentials = if let Some(credentials) = self.credentials {
@@ -1065,9 +1021,6 @@ impl AmazonS3Builder {
client,
self.retry_config.clone(),
)) as _
- } else if let Some(profile) = self.profile {
- info!("Using profile \"{}\" credential provider", profile);
- profile_credentials(profile, region.clone())?
} else {
info!("Using Instance credential provider");
@@ -1123,37 +1076,6 @@ impl AmazonS3Builder {
}
}
-#[cfg(feature = "aws_profile")]
-fn profile_region(profile: String) -> Option<String> {
- use tokio::runtime::Handle;
-
- let handle = Handle::current();
- let provider = profile::ProfileProvider::new(profile, None);
-
- handle.block_on(provider.get_region())
-}
-
-#[cfg(feature = "aws_profile")]
-fn profile_credentials(profile: String, region: String) -> Result<AwsCredentialProvider> {
- Ok(Arc::new(profile::ProfileProvider::new(
- profile,
- Some(region),
- )))
-}
-
-#[cfg(not(feature = "aws_profile"))]
-fn profile_region(_profile: String) -> Option<String> {
- None
-}
-
-#[cfg(not(feature = "aws_profile"))]
-fn profile_credentials(
- _profile: String,
- _region: String,
-) -> Result<AwsCredentialProvider> {
- Err(Error::MissingProfileFeature.into())
-}
-
#[cfg(test)]
mod tests {
use super::*;
@@ -1638,50 +1560,3 @@ mod s3_resolve_bucket_region_tests {
assert!(result.is_err());
}
}
-
-#[cfg(all(test, feature = "aws_profile"))]
-mod profile_tests {
- use super::*;
- use std::env;
-
- use super::profile::{TEST_PROFILE_NAME, TEST_PROFILE_REGION};
-
- #[tokio::test]
- async fn s3_test_region_from_profile() {
- let s3_url = "s3://bucket/prefix".to_owned();
-
- let s3 = AmazonS3Builder::new()
- .with_url(s3_url)
- .with_profile(TEST_PROFILE_NAME)
- .build()
- .unwrap();
-
- let region = &s3.client.config().region;
-
- assert_eq!(region, TEST_PROFILE_REGION);
- }
-
- #[test]
- fn s3_test_region_override() {
- let s3_url = "s3://bucket/prefix".to_owned();
-
- let aws_profile =
- env::var("AWS_PROFILE").unwrap_or_else(|_| TEST_PROFILE_NAME.into());
-
- let aws_region =
- env::var("AWS_REGION").unwrap_or_else(|_| "object_store:fake_region".into());
-
- env::set_var("AWS_PROFILE", aws_profile);
-
- let s3 = AmazonS3Builder::from_env()
- .with_url(s3_url)
- .with_region(aws_region.clone())
- .build()
- .unwrap();
-
- let actual = &s3.client.config().region;
- let expected = &aws_region;
-
- assert_eq!(actual, expected);
- }
-}
diff --git a/object_store/src/aws/profile.rs b/object_store/src/aws/profile.rs
deleted file mode 100644
index 3fc080564..000000000
--- a/object_store/src/aws/profile.rs
+++ /dev/null
@@ -1,133 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#![cfg(feature = "aws_profile")]
-
-use async_trait::async_trait;
-use aws_config::meta::region::ProvideRegion;
-use aws_config::profile::profile_file::ProfileFiles;
-use aws_config::profile::ProfileFileCredentialsProvider;
-use aws_config::profile::ProfileFileRegionProvider;
-use aws_config::provider_config::ProviderConfig;
-use aws_credential_types::provider::ProvideCredentials;
-use aws_types::region::Region;
-use std::sync::Arc;
-use std::time::Instant;
-use std::time::SystemTime;
-
-use crate::aws::AwsCredential;
-use crate::client::token::{TemporaryToken, TokenCache};
-use crate::client::CredentialProvider;
-use crate::Result;
-
-#[cfg(test)]
-pub static TEST_PROFILE_NAME: &str = "object_store:fake_profile";
-
-#[cfg(test)]
-pub static TEST_PROFILE_REGION: &str = "object_store:fake_region_from_profile";
-
-#[derive(Debug)]
-pub struct ProfileProvider {
- name: String,
- region: Option<String>,
- cache: TokenCache<Arc<AwsCredential>>,
-}
-
-impl ProfileProvider {
- pub fn new(name: String, region: Option<String>) -> Self {
- Self {
- name,
- region,
- cache: Default::default(),
- }
- }
-
- #[cfg(test)]
- fn profile_files(&self) -> ProfileFiles {
- use aws_config::profile::profile_file::ProfileFileKind;
-
- let config = format!(
- "[profile {}]\nregion = {}",
- TEST_PROFILE_NAME, TEST_PROFILE_REGION
- );
-
- ProfileFiles::builder()
- .with_contents(ProfileFileKind::Config, config)
- .build()
- }
-
- #[cfg(not(test))]
- fn profile_files(&self) -> ProfileFiles {
- ProfileFiles::default()
- }
-
- pub async fn get_region(&self) -> Option<String> {
- if let Some(region) = self.region.clone() {
- return Some(region);
- }
-
- let provider = ProfileFileRegionProvider::builder()
- .profile_files(self.profile_files())
- .profile_name(&self.name)
- .build();
-
- let region = provider.region().await;
-
- region.map(|r| r.as_ref().to_owned())
- }
-}
-
-#[async_trait]
-impl CredentialProvider for ProfileProvider {
- type Credential = AwsCredential;
-
- async fn get_credential(&self) -> Result<Arc<AwsCredential>> {
- self.cache
- .get_or_insert_with(move || async move {
- let region = self.region.clone().map(Region::new);
-
- let config = ProviderConfig::default().with_region(region);
-
- let credentials = ProfileFileCredentialsProvider::builder()
- .configure(&config)
- .profile_name(&self.name)
- .build();
-
- let c = credentials.provide_credentials().await.map_err(|source| {
- crate::Error::Generic {
- store: "S3",
- source: Box::new(source),
- }
- })?;
- let t_now = SystemTime::now();
- let expiry = c
- .expiry()
- .and_then(|e| e.duration_since(t_now).ok())
- .map(|ttl| Instant::now() + ttl);
-
- Ok(TemporaryToken {
- token: Arc::new(AwsCredential {
- key_id: c.access_key_id().to_string(),
- secret_key: c.secret_access_key().to_string(),
- token: c.session_token().map(ToString::to_string),
- }),
- expiry,
- })
- })
- .await
- }
-}