You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2023/01/04 20:03:51 UTC
[arrow-rs] branch master updated: object_store: builder configuration api (#3436)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 28a04db03 object_store: builder configuration api (#3436)
28a04db03 is described below
commit 28a04db03cea376991e7efb04c3cf4f71f6d05bf
Author: Robert Pack <42...@users.noreply.github.com>
AuthorDate: Wed Jan 4 21:03:45 2023 +0100
object_store: builder configuration api (#3436)
* feat: draf configuration api for azure
* feat: add configuration for aws and gcp
* fix: clippy
* feat: allow passing typed config keys
* refactor: implement try_from for config keys
* chore: PR feedback
* refactor: make options api fallible
* fix: docs errors
* chore: remove helpers
* test: test sas key splitting and un-nit nits
---
object_store/src/aws/mod.rs | 348 ++++++++++++++++++++++++++++++++++++++----
object_store/src/azure/mod.rs | 339 +++++++++++++++++++++++++++++++++++++---
object_store/src/gcp/mod.rs | 174 ++++++++++++++++++++-
object_store/src/lib.rs | 7 +
object_store/src/util.rs | 9 ++
5 files changed, 822 insertions(+), 55 deletions(-)
diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs
index 786ccd20f..4b633d9f5 100644
--- a/object_store/src/aws/mod.rs
+++ b/object_store/src/aws/mod.rs
@@ -37,9 +37,11 @@ use chrono::{DateTime, Utc};
use futures::stream::BoxStream;
use futures::TryStreamExt;
use itertools::Itertools;
+use serde::{Deserialize, Serialize};
use snafu::{OptionExt, ResultExt, Snafu};
use std::collections::BTreeSet;
use std::ops::Range;
+use std::str::FromStr;
use std::sync::Arc;
use tokio::io::AsyncWrite;
use tracing::info;
@@ -51,6 +53,7 @@ use crate::aws::credential::{
StaticCredentialProvider, WebIdentityProvider,
};
use crate::multipart::{CloudMultiPartUpload, CloudMultiPartUploadImpl, UploadPart};
+use crate::util::str_is_truthy;
use crate::{
ClientOptions, GetResult, ListResult, MultipartId, ObjectMeta, ObjectStore, Path,
Result, RetryConfig, StreamExt,
@@ -133,13 +136,21 @@ enum Error {
#[snafu(display("URL did not match any known pattern for scheme: {}", url))]
UrlNotRecognised { url: String },
+
+ #[snafu(display("Configuration key: '{}' is not known.", key))]
+ UnknownConfigurationKey { key: String },
}
impl From<Error> for super::Error {
- fn from(err: Error) -> Self {
- Self::Generic {
- store: "S3",
- source: Box::new(err),
+ fn from(source: Error) -> Self {
+ match source {
+ Error::UnknownConfigurationKey { key } => {
+ Self::UnknownConfigurationKey { store: "S3", key }
+ }
+ _ => Self::Generic {
+ store: "S3",
+ source: Box::new(source),
+ },
}
}
}
@@ -379,6 +390,184 @@ pub struct AmazonS3Builder {
client_options: ClientOptions,
}
+/// Configuration keys for [`AmazonS3Builder`]
+///
+/// Configuration via keys can be dome via the [`try_with_option`](AmazonS3Builder::try_with_option)
+/// or [`with_options`](AmazonS3Builder::try_with_options) methods on the builder.
+///
+/// # Example
+/// ```
+/// use std::collections::HashMap;
+/// use object_store::aws::{AmazonS3Builder, AmazonS3ConfigKey};
+///
+/// let options = HashMap::from([
+/// ("aws_access_key_id", "my-access-key-id"),
+/// ("aws_secret_access_key", "my-secret-access-key"),
+/// ]);
+/// let typed_options = vec![
+/// (AmazonS3ConfigKey::DefaultRegion, "my-default-region"),
+/// ];
+/// let azure = AmazonS3Builder::new()
+/// .try_with_options(options)
+/// .unwrap()
+/// .try_with_options(typed_options)
+/// .unwrap()
+/// .try_with_option(AmazonS3ConfigKey::Region, "my-region")
+/// .unwrap();
+/// ```
+#[derive(PartialEq, Eq, Hash, Clone, Debug, Copy, Serialize, Deserialize)]
+pub enum AmazonS3ConfigKey {
+ /// AWS Access Key
+ ///
+ /// See [`AmazonS3Builder::with_access_key_id`] for details.
+ ///
+ /// Supported keys:
+ /// - `aws_access_key_id`
+ /// - `access_key_id`
+ AccessKeyId,
+
+ /// Secret Access Key
+ ///
+ /// See [`AmazonS3Builder::with_secret_access_key`] for details.
+ ///
+ /// Supported keys:
+ /// - `aws_secret_access_key`
+ /// - `secret_access_key`
+ SecretAccessKey,
+
+ /// Region
+ ///
+ /// See [`AmazonS3Builder::with_region`] for details.
+ ///
+ /// Supported keys:
+ /// - `aws_region`
+ /// - `region`
+ Region,
+
+ /// Default region
+ ///
+ /// See [`AmazonS3Builder::with_region`] for details.
+ ///
+ /// Supported keys:
+ /// - `aws_default_region`
+ /// - `default_region`
+ DefaultRegion,
+
+ /// Bucket name
+ ///
+ /// See [`AmazonS3Builder::with_bucket_name`] for details.
+ ///
+ /// Supported keys:
+ /// - `aws_bucket`
+ /// - `aws_bucket_name`
+ /// - `bucket`
+ /// - `bucket_name`
+ Bucket,
+
+ /// Sets custom endpoint for communicating with AWS S3.
+ ///
+ /// See [`AmazonS3Builder::with_endpoint`] for details.
+ ///
+ /// Supported keys:
+ /// - `aws_endpoint`
+ /// - `aws_endpoint_url`
+ /// - `endpoint`
+ /// - `endpoint_url`
+ Endpoint,
+
+ /// Token to use for requests (passed to underlying provider)
+ ///
+ /// See [`AmazonS3Builder::with_token`] for details.
+ ///
+ /// Supported keys:
+ /// - `aws_session_token`
+ /// - `aws_token`
+ /// - `session_token`
+ /// - `token`
+ Token,
+
+ /// Fall back to ImdsV1
+ ///
+ /// See [`AmazonS3Builder::with_imdsv1_fallback`] for details.
+ ///
+ /// Supported keys:
+ /// - `aws_imdsv1_fallback`
+ /// - `imdsv1_fallback`
+ ImdsV1Fallback,
+
+ /// If virtual hosted style request has to be used
+ ///
+ /// See [`AmazonS3Builder::with_virtual_hosted_style_request`] for details.
+ ///
+ /// Supported keys:
+ /// - `aws_virtual_hosted_style_request`
+ /// - `virtual_hosted_style_request`
+ VirtualHostedStyleRequest,
+
+ /// Set the instance metadata endpoint
+ ///
+ /// See [`AmazonS3Builder::with_metadata_endpoint`] for details.
+ ///
+ /// Supported keys:
+ /// - `aws_metadata_endpoint`
+ /// - `metadata_endpoint`
+ MetadataEndpoint,
+
+ /// AWS profile name
+ ///
+ /// Supported keys:
+ /// - `aws_profile`
+ /// - `profile`
+ Profile,
+}
+
+impl AsRef<str> for AmazonS3ConfigKey {
+ fn as_ref(&self) -> &str {
+ match self {
+ Self::AccessKeyId => "aws_access_key_id",
+ Self::SecretAccessKey => "aws_secret_access_key",
+ Self::Region => "aws_region",
+ Self::Bucket => "aws_bucket",
+ Self::Endpoint => "aws_endpoint",
+ Self::Token => "aws_session_token",
+ Self::ImdsV1Fallback => "aws_imdsv1_fallback",
+ Self::VirtualHostedStyleRequest => "aws_virtual_hosted_style_request",
+ Self::DefaultRegion => "aws_default_region",
+ Self::MetadataEndpoint => "aws_metadata_endpoint",
+ Self::Profile => "aws_profile",
+ }
+ }
+}
+
+impl FromStr for AmazonS3ConfigKey {
+ type Err = super::Error;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s {
+ "aws_access_key_id" | "access_key_id" => Ok(Self::AccessKeyId),
+ "aws_secret_access_key" | "secret_access_key" => Ok(Self::SecretAccessKey),
+ "aws_default_region" | "default_region" => Ok(Self::DefaultRegion),
+ "aws_region" | "region" => Ok(Self::Region),
+ "aws_bucket" | "aws_bucket_name" | "bucket_name" | "bucket" => {
+ Ok(Self::Bucket)
+ }
+ "aws_endpoint_url" | "aws_endpoint" | "endpoint_url" | "endpoint" => {
+ Ok(Self::Endpoint)
+ }
+ "aws_session_token" | "aws_token" | "session_token" | "token" => {
+ Ok(Self::Token)
+ }
+ "aws_virtual_hosted_style_request" | "virtual_hosted_style_request" => {
+ Ok(Self::VirtualHostedStyleRequest)
+ }
+ "aws_profile" | "profile" => Ok(Self::Profile),
+ "aws_imdsv1_fallback" | "imdsv1_fallback" => Ok(Self::ImdsV1Fallback),
+ "aws_metadata_endpoint" | "metadata_endpoint" => Ok(Self::MetadataEndpoint),
+ _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+ }
+ }
+}
+
impl AmazonS3Builder {
/// Create a new [`AmazonS3Builder`] with default values.
pub fn new() -> Self {
@@ -407,28 +596,16 @@ impl AmazonS3Builder {
pub fn from_env() -> Self {
let mut builder: Self = Default::default();
- if let Ok(access_key_id) = std::env::var("AWS_ACCESS_KEY_ID") {
- builder.access_key_id = Some(access_key_id);
- }
-
- if let Ok(secret_access_key) = std::env::var("AWS_SECRET_ACCESS_KEY") {
- builder.secret_access_key = Some(secret_access_key);
- }
-
- if let Ok(secret) = std::env::var("AWS_DEFAULT_REGION") {
- builder.region = Some(secret);
- }
-
- if let Ok(endpoint) = std::env::var("AWS_ENDPOINT") {
- builder.endpoint = Some(endpoint);
- }
-
- if let Ok(token) = std::env::var("AWS_SESSION_TOKEN") {
- builder.token = Some(token);
- }
-
- if let Ok(profile) = std::env::var("AWS_PROFILE") {
- builder.profile = Some(profile);
+ for (os_key, os_value) in std::env::vars_os() {
+ if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) {
+ if key.starts_with("AWS_") {
+ if let Ok(config_key) =
+ AmazonS3ConfigKey::from_str(&key.to_ascii_lowercase())
+ {
+ builder = builder.try_with_option(config_key, value).unwrap();
+ }
+ }
+ }
}
// This env var is set in ECS
@@ -442,7 +619,7 @@ impl AmazonS3Builder {
if let Ok(text) = std::env::var("AWS_ALLOW_HTTP") {
builder.client_options =
- builder.client_options.with_allow_http(text == "true");
+ builder.client_options.with_allow_http(str_is_truthy(&text));
}
builder
@@ -472,6 +649,55 @@ impl AmazonS3Builder {
self
}
+ /// Set an option on the builder via a key - value pair.
+ ///
+ /// This method will return an `UnknownConfigKey` error if key cannot be parsed into [`AmazonS3ConfigKey`].
+ pub fn try_with_option(
+ mut self,
+ key: impl AsRef<str>,
+ value: impl Into<String>,
+ ) -> Result<Self> {
+ match AmazonS3ConfigKey::from_str(key.as_ref())? {
+ AmazonS3ConfigKey::AccessKeyId => self.access_key_id = Some(value.into()),
+ AmazonS3ConfigKey::SecretAccessKey => {
+ self.secret_access_key = Some(value.into())
+ }
+ AmazonS3ConfigKey::Region => self.region = Some(value.into()),
+ AmazonS3ConfigKey::Bucket => self.bucket_name = Some(value.into()),
+ AmazonS3ConfigKey::Endpoint => self.endpoint = Some(value.into()),
+ AmazonS3ConfigKey::Token => self.token = Some(value.into()),
+ AmazonS3ConfigKey::ImdsV1Fallback => {
+ self.imdsv1_fallback = str_is_truthy(&value.into())
+ }
+ AmazonS3ConfigKey::VirtualHostedStyleRequest => {
+ self.virtual_hosted_style_request = str_is_truthy(&value.into())
+ }
+ AmazonS3ConfigKey::DefaultRegion => {
+ self.region = self.region.or_else(|| Some(value.into()))
+ }
+ AmazonS3ConfigKey::MetadataEndpoint => {
+ self.metadata_endpoint = Some(value.into())
+ }
+ AmazonS3ConfigKey::Profile => self.profile = Some(value.into()),
+ };
+ Ok(self)
+ }
+
+ /// Hydrate builder from key value pairs
+ ///
+ /// This method will return an `UnknownConfigKey` error if any key cannot be parsed into [`AmazonS3ConfigKey`].
+ pub fn try_with_options<
+ I: IntoIterator<Item = (impl AsRef<str>, impl Into<String>)>,
+ >(
+ mut self,
+ options: I,
+ ) -> Result<Self> {
+ for (key, value) in options {
+ self = self.try_with_option(key, value)?;
+ }
+ Ok(self)
+ }
+
/// Sets properties on this builder based on a URL
///
/// This is a separate member function to allow fallible computation to
@@ -773,6 +999,7 @@ mod tests {
put_get_delete_list_opts, rename_and_copy, stream_get,
};
use bytes::Bytes;
+ use std::collections::HashMap;
use std::env;
const NON_EXISTENT_NAME: &str = "nonexistentname";
@@ -915,6 +1142,73 @@ mod tests {
assert_eq!(builder.metadata_endpoint.unwrap(), metadata_uri);
}
+ #[test]
+ fn s3_test_config_from_map() {
+ let aws_access_key_id = "object_store:fake_access_key_id".to_string();
+ let aws_secret_access_key = "object_store:fake_secret_key".to_string();
+ let aws_default_region = "object_store:fake_default_region".to_string();
+ let aws_endpoint = "object_store:fake_endpoint".to_string();
+ let aws_session_token = "object_store:fake_session_token".to_string();
+ let options = HashMap::from([
+ ("aws_access_key_id", aws_access_key_id.clone()),
+ ("aws_secret_access_key", aws_secret_access_key),
+ ("aws_default_region", aws_default_region.clone()),
+ ("aws_endpoint", aws_endpoint.clone()),
+ ("aws_session_token", aws_session_token.clone()),
+ ]);
+
+ let builder = AmazonS3Builder::new()
+ .try_with_options(&options)
+ .unwrap()
+ .try_with_option("aws_secret_access_key", "new-secret-key")
+ .unwrap();
+ assert_eq!(builder.access_key_id.unwrap(), aws_access_key_id.as_str());
+ assert_eq!(builder.secret_access_key.unwrap(), "new-secret-key");
+ assert_eq!(builder.region.unwrap(), aws_default_region);
+ assert_eq!(builder.endpoint.unwrap(), aws_endpoint);
+ assert_eq!(builder.token.unwrap(), aws_session_token);
+ }
+
+ #[test]
+ fn s3_test_config_from_typed_map() {
+ let aws_access_key_id = "object_store:fake_access_key_id".to_string();
+ let aws_secret_access_key = "object_store:fake_secret_key".to_string();
+ let aws_default_region = "object_store:fake_default_region".to_string();
+ let aws_endpoint = "object_store:fake_endpoint".to_string();
+ let aws_session_token = "object_store:fake_session_token".to_string();
+ let options = HashMap::from([
+ (AmazonS3ConfigKey::AccessKeyId, aws_access_key_id.clone()),
+ (AmazonS3ConfigKey::SecretAccessKey, aws_secret_access_key),
+ (AmazonS3ConfigKey::DefaultRegion, aws_default_region.clone()),
+ (AmazonS3ConfigKey::Endpoint, aws_endpoint.clone()),
+ (AmazonS3ConfigKey::Token, aws_session_token.clone()),
+ ]);
+
+ let builder = AmazonS3Builder::new()
+ .try_with_options(&options)
+ .unwrap()
+ .try_with_option(AmazonS3ConfigKey::SecretAccessKey, "new-secret-key")
+ .unwrap();
+ assert_eq!(builder.access_key_id.unwrap(), aws_access_key_id.as_str());
+ assert_eq!(builder.secret_access_key.unwrap(), "new-secret-key");
+ assert_eq!(builder.region.unwrap(), aws_default_region);
+ assert_eq!(builder.endpoint.unwrap(), aws_endpoint);
+ assert_eq!(builder.token.unwrap(), aws_session_token);
+ }
+
+ #[test]
+ fn s3_test_config_fallible_options() {
+ let aws_access_key_id = "object_store:fake_access_key_id".to_string();
+ let aws_secret_access_key = "object_store:fake_secret_key".to_string();
+ let options = HashMap::from([
+ ("aws_access_key_id", aws_access_key_id),
+ ("invalid-key", aws_secret_access_key),
+ ]);
+
+ let builder = AmazonS3Builder::new().try_with_options(&options);
+ assert!(builder.is_err());
+ }
+
#[tokio::test]
async fn s3_test() {
let config = maybe_skip_integration!();
diff --git a/object_store/src/azure/mod.rs b/object_store/src/azure/mod.rs
index 7cf369de3..416883ac9 100644
--- a/object_store/src/azure/mod.rs
+++ b/object_store/src/azure/mod.rs
@@ -37,16 +37,18 @@ use async_trait::async_trait;
use bytes::Bytes;
use chrono::{TimeZone, Utc};
use futures::{stream::BoxStream, StreamExt, TryStreamExt};
+use percent_encoding::percent_decode_str;
+use serde::{Deserialize, Serialize};
use snafu::{OptionExt, ResultExt, Snafu};
-use std::collections::BTreeSet;
use std::fmt::{Debug, Formatter};
use std::io;
use std::ops::Range;
use std::sync::Arc;
+use std::{collections::BTreeSet, str::FromStr};
use tokio::io::AsyncWrite;
use url::Url;
-use crate::util::RFC1123_FMT;
+use crate::util::{str_is_truthy, RFC1123_FMT};
pub use credential::authority_hosts;
mod client;
@@ -124,13 +126,28 @@ enum Error {
#[snafu(display("URL did not match any known pattern for scheme: {}", url))]
UrlNotRecognised { url: String },
+
+ #[snafu(display("Failed parsing an SAS key"))]
+ DecodeSasKey { source: std::str::Utf8Error },
+
+ #[snafu(display("Missing component in SAS query pair"))]
+ MissingSasComponent {},
+
+ #[snafu(display("Configuration key: '{}' is not known.", key))]
+ UnknownConfigurationKey { key: String },
}
impl From<Error> for super::Error {
fn from(source: Error) -> Self {
- Self::Generic {
- store: "MicrosoftAzure",
- source: Box::new(source),
+ match source {
+ Error::UnknownConfigurationKey { key } => Self::UnknownConfigurationKey {
+ store: "MicrosoftAzure",
+ key,
+ },
+ _ => Self::Generic {
+ store: "MicrosoftAzure",
+ source: Box::new(source),
+ },
}
}
}
@@ -367,6 +384,7 @@ pub struct MicrosoftAzureBuilder {
client_secret: Option<String>,
tenant_id: Option<String>,
sas_query_pairs: Option<Vec<(String, String)>>,
+ sas_key: Option<String>,
authority_host: Option<String>,
url: Option<String>,
use_emulator: bool,
@@ -374,6 +392,157 @@ pub struct MicrosoftAzureBuilder {
client_options: ClientOptions,
}
+/// Configuration keys for [`MicrosoftAzureBuilder`]
+///
+/// Configuration via keys can be dome via the [`try_with_option`](MicrosoftAzureBuilder::try_with_option)
+/// or [`with_options`](MicrosoftAzureBuilder::try_with_options) methods on the builder.
+///
+/// # Example
+/// ```
+/// use std::collections::HashMap;
+/// use object_store::azure::{MicrosoftAzureBuilder, AzureConfigKey};
+///
+/// let options = HashMap::from([
+/// ("azure_client_id", "my-client-id"),
+/// ("azure_client_secret", "my-account-name"),
+/// ]);
+/// let typed_options = vec![
+/// (AzureConfigKey::AccountName, "my-account-name"),
+/// ];
+/// let azure = MicrosoftAzureBuilder::new()
+/// .try_with_options(options)
+/// .unwrap()
+/// .try_with_options(typed_options)
+/// .unwrap()
+/// .try_with_option(AzureConfigKey::AuthorityId, "my-tenant-id")
+/// .unwrap();
+/// ```
+#[derive(PartialEq, Eq, Hash, Clone, Debug, Copy, Deserialize, Serialize)]
+pub enum AzureConfigKey {
+ /// The name of the azure storage account
+ ///
+ /// Supported keys:
+ /// - `azure_storage_account_name`
+ /// - `account_name`
+ AccountName,
+
+ /// Master key for accessing storage account
+ ///
+ /// Supported keys:
+ /// - `azure_storage_account_key`
+ /// - `azure_storage_access_key`
+ /// - `azure_storage_master_key`
+ /// - `access_key`
+ /// - `account_key`
+ /// - `master_key`
+ AccessKey,
+
+ /// Service principal client id for authorizing requests
+ ///
+ /// Supported keys:
+ /// - `azure_storage_client_id`
+ /// - `azure_client_id`
+ /// - `client_id`
+ ClientId,
+
+ /// Service principal client secret for authorizing requests
+ ///
+ /// Supported keys:
+ /// - `azure_storage_client_secret`
+ /// - `azure_client_secret`
+ /// - `client_secret`
+ ClientSecret,
+
+ /// Tenant id used in oauth flows
+ ///
+ /// Supported keys:
+ /// - `azure_storage_tenant_id`
+ /// - `azure_storage_authority_id`
+ /// - `azure_tenant_id`
+ /// - `azure_authority_id`
+ /// - `tenant_id`
+ /// - `authority_id`
+ AuthorityId,
+
+ /// Shared access signature.
+ ///
+ /// The signature is expected to be percent-encoded, much like they are provided
+ /// in the azure storage explorer or azure portal.
+ ///
+ /// Supported keys:
+ /// - `azure_storage_sas_key`
+ /// - `azure_storage_sas_token`
+ /// - `sas_key`
+ /// - `sas_token`
+ SasKey,
+
+ /// Bearer token
+ ///
+ /// Supported keys:
+ /// - `azure_storage_token`
+ /// - `bearer_token`
+ /// - `token`
+ Token,
+
+ /// Use object store with azurite storage emulator
+ ///
+ /// Supported keys:
+ /// - `azure_storage_use_emulator`
+ /// - `object_store_use_emulator`
+ /// - `use_emulator`
+ UseEmulator,
+}
+
+impl AsRef<str> for AzureConfigKey {
+ fn as_ref(&self) -> &str {
+ match self {
+ Self::AccountName => "azure_storage_account_name",
+ Self::AccessKey => "azure_storage_account_key",
+ Self::ClientId => "azure_storage_client_id",
+ Self::ClientSecret => "azure_storage_client_secret",
+ Self::AuthorityId => "azure_storage_tenant_id",
+ Self::SasKey => "azure_storage_sas_key",
+ Self::Token => "azure_storage_token",
+ Self::UseEmulator => "azure_storage_use_emulator",
+ }
+ }
+}
+
+impl FromStr for AzureConfigKey {
+ type Err = super::Error;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s {
+ "azure_storage_account_key"
+ | "azure_storage_access_key"
+ | "azure_storage_master_key"
+ | "master_key"
+ | "account_key"
+ | "access_key" => Ok(Self::AccessKey),
+ "azure_storage_account_name" | "account_name" => Ok(Self::AccountName),
+ "azure_storage_client_id" | "azure_client_id" | "client_id" => {
+ Ok(Self::ClientId)
+ }
+ "azure_storage_client_secret" | "azure_client_secret" | "client_secret" => {
+ Ok(Self::ClientSecret)
+ }
+ "azure_storage_tenant_id"
+ | "azure_storage_authority_id"
+ | "azure_tenant_id"
+ | "azure_authority_id"
+ | "tenant_id"
+ | "authority_id" => Ok(Self::AuthorityId),
+ "azure_storage_sas_key"
+ | "azure_storage_sas_token"
+ | "sas_key"
+ | "sas_token" => Ok(Self::SasKey),
+ "azure_storage_token" | "bearer_token" | "token" => Ok(Self::Token),
+ "azure_storage_use_emulator" | "use_emulator" => Ok(Self::UseEmulator),
+ _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+ }
+ }
+}
+
impl Debug for MicrosoftAzureBuilder {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
@@ -409,27 +578,21 @@ impl MicrosoftAzureBuilder {
/// ```
pub fn from_env() -> Self {
let mut builder = Self::default();
-
- if let Ok(account_name) = std::env::var("AZURE_STORAGE_ACCOUNT_NAME") {
- builder.account_name = Some(account_name);
- }
-
- if let Ok(access_key) = std::env::var("AZURE_STORAGE_ACCOUNT_KEY") {
- builder.access_key = Some(access_key);
- } else if let Ok(access_key) = std::env::var("AZURE_STORAGE_ACCESS_KEY") {
- builder.access_key = Some(access_key);
- }
-
- if let Ok(client_id) = std::env::var("AZURE_STORAGE_CLIENT_ID") {
- builder.client_id = Some(client_id);
- }
-
- if let Ok(client_secret) = std::env::var("AZURE_STORAGE_CLIENT_SECRET") {
- builder.client_secret = Some(client_secret);
+ for (os_key, os_value) in std::env::vars_os() {
+ if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) {
+ if key.starts_with("AZURE_") {
+ if let Ok(config_key) =
+ AzureConfigKey::from_str(&key.to_ascii_lowercase())
+ {
+ builder = builder.try_with_option(config_key, value).unwrap();
+ }
+ }
+ }
}
- if let Ok(tenant_id) = std::env::var("AZURE_STORAGE_TENANT_ID") {
- builder.tenant_id = Some(tenant_id);
+ if let Ok(text) = std::env::var("AZURE_ALLOW_HTTP") {
+ builder.client_options =
+ builder.client_options.with_allow_http(str_is_truthy(&text));
}
builder
@@ -462,6 +625,40 @@ impl MicrosoftAzureBuilder {
self
}
+ /// Set an option on the builder via a key - value pair.
+ pub fn try_with_option(
+ mut self,
+ key: impl AsRef<str>,
+ value: impl Into<String>,
+ ) -> Result<Self> {
+ match AzureConfigKey::from_str(key.as_ref())? {
+ AzureConfigKey::AccessKey => self.access_key = Some(value.into()),
+ AzureConfigKey::AccountName => self.account_name = Some(value.into()),
+ AzureConfigKey::ClientId => self.client_id = Some(value.into()),
+ AzureConfigKey::ClientSecret => self.client_secret = Some(value.into()),
+ AzureConfigKey::AuthorityId => self.tenant_id = Some(value.into()),
+ AzureConfigKey::SasKey => self.sas_key = Some(value.into()),
+ AzureConfigKey::Token => self.bearer_token = Some(value.into()),
+ AzureConfigKey::UseEmulator => {
+ self.use_emulator = str_is_truthy(&value.into())
+ }
+ };
+ Ok(self)
+ }
+
+ /// Hydrate builder from key value pairs
+ pub fn try_with_options<
+ I: IntoIterator<Item = (impl AsRef<str>, impl Into<String>)>,
+ >(
+ mut self,
+ options: I,
+ ) -> Result<Self> {
+ for (key, value) in options {
+ self = self.try_with_option(key, value)?;
+ }
+ Ok(self)
+ }
+
/// Sets properties on this builder based on a URL
///
/// This is a separate member function to allow fallible computation to
@@ -636,6 +833,8 @@ impl MicrosoftAzureBuilder {
))
} else if let Some(query_pairs) = self.sas_query_pairs {
Ok(credential::CredentialProvider::SASToken(query_pairs))
+ } else if let Some(sas) = self.sas_key {
+ Ok(credential::CredentialProvider::SASToken(split_sas(&sas)?))
} else {
Err(Error::MissingCredentials {})
}?;
@@ -673,6 +872,25 @@ fn url_from_env(env_name: &str, default_url: &str) -> Result<Url> {
Ok(url)
}
+fn split_sas(sas: &str) -> Result<Vec<(String, String)>, Error> {
+ let sas = percent_decode_str(sas)
+ .decode_utf8()
+ .context(DecodeSasKeySnafu {})?;
+ let kv_str_pairs = sas
+ .trim_start_matches('?')
+ .split('&')
+ .filter(|s| !s.chars().all(char::is_whitespace));
+ let mut pairs = Vec::new();
+ for kv_pair_str in kv_str_pairs {
+ let (k, v) = kv_pair_str
+ .trim()
+ .split_once('=')
+ .ok_or(Error::MissingSasComponent {})?;
+ pairs.push((k.into(), v.into()))
+ }
+ Ok(pairs)
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -680,6 +898,7 @@ mod tests {
copy_if_not_exists, list_uses_directories_correctly, list_with_delimiter,
put_get_delete_list, put_get_delete_list_opts, rename_and_copy, stream_get,
};
+ use std::collections::HashMap;
use std::env;
// Helper macro to skip tests if TEST_INTEGRATION and the Azure environment
@@ -832,4 +1051,76 @@ mod tests {
builder.parse_url(case).unwrap_err();
}
}
+
+ #[test]
+ fn azure_test_config_from_map() {
+ let azure_client_id = "object_store:fake_access_key_id";
+ let azure_storage_account_name = "object_store:fake_secret_key";
+ let azure_storage_token = "object_store:fake_default_region";
+ let options = HashMap::from([
+ ("azure_client_id", azure_client_id),
+ ("azure_storage_account_name", azure_storage_account_name),
+ ("azure_storage_token", azure_storage_token),
+ ]);
+
+ let builder = MicrosoftAzureBuilder::new()
+ .try_with_options(options)
+ .unwrap();
+ assert_eq!(builder.client_id.unwrap(), azure_client_id);
+ assert_eq!(builder.account_name.unwrap(), azure_storage_account_name);
+ assert_eq!(builder.bearer_token.unwrap(), azure_storage_token);
+ }
+
+ #[test]
+ fn azure_test_config_from_typed_map() {
+ let azure_client_id = "object_store:fake_access_key_id".to_string();
+ let azure_storage_account_name = "object_store:fake_secret_key".to_string();
+ let azure_storage_token = "object_store:fake_default_region".to_string();
+ let options = HashMap::from([
+ (AzureConfigKey::ClientId, azure_client_id.clone()),
+ (
+ AzureConfigKey::AccountName,
+ azure_storage_account_name.clone(),
+ ),
+ (AzureConfigKey::Token, azure_storage_token.clone()),
+ ]);
+
+ let builder = MicrosoftAzureBuilder::new()
+ .try_with_options(&options)
+ .unwrap();
+ assert_eq!(builder.client_id.unwrap(), azure_client_id);
+ assert_eq!(builder.account_name.unwrap(), azure_storage_account_name);
+ assert_eq!(builder.bearer_token.unwrap(), azure_storage_token);
+ }
+
+ #[test]
+ fn azure_test_config_fallible_options() {
+ let azure_client_id = "object_store:fake_access_key_id".to_string();
+ let azure_storage_token = "object_store:fake_default_region".to_string();
+ let options = HashMap::from([
+ ("azure_client_id", azure_client_id),
+ ("invalid-key", azure_storage_token),
+ ]);
+
+ let builder = MicrosoftAzureBuilder::new().try_with_options(&options);
+ assert!(builder.is_err());
+ }
+
+ #[test]
+ fn azure_test_split_sas() {
+ let raw_sas = "?sv=2021-10-04&st=2023-01-04T17%3A48%3A57Z&se=2023-01-04T18%3A15%3A00Z&sr=c&sp=rcwl&sig=C7%2BZeEOWbrxPA3R0Cw%2Fw1EZz0%2B4KBvQexeKZKe%2BB6h0%3D";
+ let expected = vec![
+ ("sv".to_string(), "2021-10-04".to_string()),
+ ("st".to_string(), "2023-01-04T17:48:57Z".to_string()),
+ ("se".to_string(), "2023-01-04T18:15:00Z".to_string()),
+ ("sr".to_string(), "c".to_string()),
+ ("sp".to_string(), "rcwl".to_string()),
+ (
+ "sig".to_string(),
+ "C7+ZeEOWbrxPA3R0Cw/w1EZz0+4KBvQexeKZKe+B6h0=".to_string(),
+ ),
+ ];
+ let pairs = split_sas(raw_sas).unwrap();
+ assert_eq!(expected, pairs);
+ }
}
diff --git a/object_store/src/gcp/mod.rs b/object_store/src/gcp/mod.rs
index f2638748f..177812fa8 100644
--- a/object_store/src/gcp/mod.rs
+++ b/object_store/src/gcp/mod.rs
@@ -33,6 +33,7 @@ use std::collections::BTreeSet;
use std::fs::File;
use std::io::{self, BufReader};
use std::ops::Range;
+use std::str::FromStr;
use std::sync::Arc;
use async_trait::async_trait;
@@ -42,6 +43,7 @@ use futures::{stream::BoxStream, StreamExt, TryStreamExt};
use percent_encoding::{percent_encode, NON_ALPHANUMERIC};
use reqwest::header::RANGE;
use reqwest::{header, Client, Method, Response, StatusCode};
+use serde::{Deserialize, Serialize};
use snafu::{OptionExt, ResultExt, Snafu};
use tokio::io::AsyncWrite;
use url::Url;
@@ -145,6 +147,9 @@ enum Error {
#[snafu(display("URL did not match any known pattern for scheme: {}", url))]
UrlNotRecognised { url: String },
+
+ #[snafu(display("Configuration key: '{}' is not known.", key))]
+ UnknownConfigurationKey { key: String },
}
impl From<Error> for super::Error {
@@ -164,6 +169,9 @@ impl From<Error> for super::Error {
source: Box::new(source),
path,
},
+ Error::UnknownConfigurationKey { key } => {
+ Self::UnknownConfigurationKey { store: "GCS", key }
+ }
_ => Self::Generic {
store: "GCS",
source: Box::new(err),
@@ -796,6 +804,74 @@ pub struct GoogleCloudStorageBuilder {
client_options: ClientOptions,
}
+/// Configuration keys for [`GoogleCloudStorageBuilder`]
+///
+/// Configuration via keys can be dome via the [`try_with_option`](GoogleCloudStorageBuilder::try_with_option)
+/// or [`with_options`](GoogleCloudStorageBuilder::try_with_options) methods on the builder.
+///
+/// # Example
+/// ```
+/// use std::collections::HashMap;
+/// use object_store::gcp::{GoogleCloudStorageBuilder, GoogleConfigKey};
+///
+/// let options = HashMap::from([
+/// ("google_service_account", "my-service-account"),
+/// ]);
+/// let typed_options = vec![
+/// (GoogleConfigKey::Bucket, "my-bucket"),
+/// ];
+/// let azure = GoogleCloudStorageBuilder::new()
+/// .try_with_options(options)
+/// .unwrap()
+/// .try_with_options(typed_options)
+/// .unwrap()
+/// .try_with_option(GoogleConfigKey::Bucket, "my-new-bucket")
+/// .unwrap();
+/// ```
+#[derive(PartialEq, Eq, Hash, Clone, Debug, Copy, Serialize, Deserialize)]
+pub enum GoogleConfigKey {
+ /// Path to the service account file
+ ///
+ /// Supported keys:
+ /// - `google_service_account`
+ /// - `service_account`
+ ServiceAccount,
+
+ /// Bucket name
+ ///
+ /// See [`GoogleCloudStorageBuilder::with_bucket_name`] for details.
+ ///
+ /// Supported keys:
+ /// - `google_bucket`
+ /// - `google_bucket_name`
+ /// - `bucket`
+ /// - `bucket_name`
+ Bucket,
+}
+
+impl AsRef<str> for GoogleConfigKey {
+ fn as_ref(&self) -> &str {
+ match self {
+ Self::ServiceAccount => "google_service_account",
+ Self::Bucket => "google_bucket",
+ }
+ }
+}
+
+impl FromStr for GoogleConfigKey {
+ type Err = super::Error;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s {
+ "google_service_account" | "service_account" => Ok(Self::ServiceAccount),
+ "google_bucket" | "google_bucket_name" | "bucket" | "bucket_name" => {
+ Ok(Self::Bucket)
+ }
+ _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+ }
+ }
+}
+
impl Default for GoogleCloudStorageBuilder {
fn default() -> Self {
Self {
@@ -835,8 +911,16 @@ impl GoogleCloudStorageBuilder {
builder.service_account_path = Some(service_account_path);
}
- if let Ok(service_account_path) = std::env::var("GOOGLE_SERVICE_ACCOUNT") {
- builder.service_account_path = Some(service_account_path);
+ for (os_key, os_value) in std::env::vars_os() {
+ if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) {
+ if key.starts_with("GOOGLE_") {
+ if let Ok(config_key) =
+ GoogleConfigKey::from_str(&key.to_ascii_lowercase())
+ {
+ builder = builder.try_with_option(config_key, value).unwrap();
+ }
+ }
+ }
}
builder
@@ -863,6 +947,34 @@ impl GoogleCloudStorageBuilder {
self
}
+ /// Set an option on the builder via a key - value pair.
+ pub fn try_with_option(
+ mut self,
+ key: impl AsRef<str>,
+ value: impl Into<String>,
+ ) -> Result<Self> {
+ match GoogleConfigKey::from_str(key.as_ref())? {
+ GoogleConfigKey::ServiceAccount => {
+ self.service_account_path = Some(value.into())
+ }
+ GoogleConfigKey::Bucket => self.bucket_name = Some(value.into()),
+ };
+ Ok(self)
+ }
+
+ /// Hydrate builder from key value pairs
+ pub fn try_with_options<
+ I: IntoIterator<Item = (impl AsRef<str>, impl Into<String>)>,
+ >(
+ mut self,
+ options: I,
+ ) -> Result<Self> {
+ for (key, value) in options {
+ self = self.try_with_option(key, value)?;
+ }
+ Ok(self)
+ }
+
/// Sets properties on this builder based on a URL
///
/// This is a separate member function to allow fallible computation to
@@ -995,9 +1107,9 @@ fn convert_object_meta(object: &Object) -> Result<ObjectMeta> {
#[cfg(test)]
mod test {
- use std::env;
-
use bytes::Bytes;
+ use std::collections::HashMap;
+ use std::env;
use crate::{
tests::{
@@ -1205,4 +1317,58 @@ mod test {
builder.parse_url(case).unwrap_err();
}
}
+
+ #[test]
+ fn gcs_test_config_from_map() {
+ let google_service_account = "object_store:fake_service_account".to_string();
+ let google_bucket_name = "object_store:fake_bucket".to_string();
+ let options = HashMap::from([
+ ("google_service_account", google_service_account.clone()),
+ ("google_bucket_name", google_bucket_name.clone()),
+ ]);
+
+ let builder = GoogleCloudStorageBuilder::new()
+ .try_with_options(&options)
+ .unwrap();
+ assert_eq!(
+ builder.service_account_path.unwrap(),
+ google_service_account.as_str()
+ );
+ assert_eq!(builder.bucket_name.unwrap(), google_bucket_name.as_str());
+ }
+
+ #[test]
+ fn gcs_test_config_from_typed_map() {
+ let google_service_account = "object_store:fake_service_account".to_string();
+ let google_bucket_name = "object_store:fake_bucket".to_string();
+ let options = HashMap::from([
+ (
+ GoogleConfigKey::ServiceAccount,
+ google_service_account.clone(),
+ ),
+ (GoogleConfigKey::Bucket, google_bucket_name.clone()),
+ ]);
+
+ let builder = GoogleCloudStorageBuilder::new()
+ .try_with_options(&options)
+ .unwrap();
+ assert_eq!(
+ builder.service_account_path.unwrap(),
+ google_service_account.as_str()
+ );
+ assert_eq!(builder.bucket_name.unwrap(), google_bucket_name.as_str());
+ }
+
+ #[test]
+ fn gcs_test_config_fallible_options() {
+ let google_service_account = "object_store:fake_service_account".to_string();
+ let google_bucket_name = "object_store:fake_bucket".to_string();
+ let options = HashMap::from([
+ ("google_service_account", google_service_account),
+ ("invalid-key", google_bucket_name),
+ ]);
+
+ let builder = GoogleCloudStorageBuilder::new().try_with_options(&options);
+ assert!(builder.is_err());
+ }
}
diff --git a/object_store/src/lib.rs b/object_store/src/lib.rs
index 425c5cdba..4ec58c387 100644
--- a/object_store/src/lib.rs
+++ b/object_store/src/lib.rs
@@ -555,6 +555,13 @@ pub enum Error {
#[snafu(display("Operation not yet implemented."))]
NotImplemented,
+
+ #[snafu(display(
+ "Configuration key: '{}' is not valid for store '{}'.",
+ key,
+ store
+ ))]
+ UnknownConfigurationKey { store: &'static str, key: String },
}
impl From<Error> for std::io::Error {
diff --git a/object_store/src/util.rs b/object_store/src/util.rs
index e592e7b64..08bfd86d9 100644
--- a/object_store/src/util.rs
+++ b/object_store/src/util.rs
@@ -185,6 +185,15 @@ fn merge_ranges(
ret
}
+#[allow(dead_code)]
+pub(crate) fn str_is_truthy(val: &str) -> bool {
+ val.eq_ignore_ascii_case("1")
+ | val.eq_ignore_ascii_case("true")
+ | val.eq_ignore_ascii_case("on")
+ | val.eq_ignore_ascii_case("yes")
+ | val.eq_ignore_ascii_case("y")
+}
+
#[cfg(test)]
mod tests {
use super::*;