You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2023/05/11 12:19:51 UTC
[arrow-rs] branch master updated: Allow setting ClientOptions with Options API (#4202)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 1f714fa77 Allow setting ClientOptions with Options API (#4202)
1f714fa77 is described below
commit 1f714fa77dc037e561dcfb701e2e906e01640142
Author: Raphael Taylor-Davies <17...@users.noreply.github.com>
AuthorDate: Thu May 11 13:19:45 2023 +0100
Allow setting ClientOptions with Options API (#4202)
* Allow setting ClientOptions with options API
* More clippy
---
object_store/src/aws/mod.rs | 25 +++++++++++++++----------
object_store/src/azure/mod.rs | 25 +++++++++++++++----------
object_store/src/client/mod.rs | 23 +++++++++++++++++++++++
object_store/src/gcp/mod.rs | 18 ++++++++++++++----
4 files changed, 67 insertions(+), 24 deletions(-)
diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs
index fe49471c4..17d779ff6 100644
--- a/object_store/src/aws/mod.rs
+++ b/object_store/src/aws/mod.rs
@@ -604,6 +604,9 @@ pub enum AmazonS3ConfigKey {
/// - `aws_profile`
/// - `profile`
Profile,
+
+ /// Client options
+ Client(ClientConfigKey),
}
impl AsRef<str> for AmazonS3ConfigKey {
@@ -622,6 +625,7 @@ impl AsRef<str> for AmazonS3ConfigKey {
Self::Profile => "aws_profile",
Self::UnsignedPayload => "aws_unsigned_payload",
Self::Checksum => "aws_checksum_algorithm",
+ Self::Client(opt) => opt.as_ref(),
}
}
}
@@ -652,7 +656,12 @@ impl FromStr for AmazonS3ConfigKey {
"aws_metadata_endpoint" | "metadata_endpoint" => Ok(Self::MetadataEndpoint),
"aws_unsigned_payload" | "unsigned_payload" => Ok(Self::UnsignedPayload),
"aws_checksum_algorithm" | "checksum_algorithm" => Ok(Self::Checksum),
- _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+ // Backwards compatibility
+ "aws_allow_http" => Ok(Self::Client(ClientConfigKey::AllowHttp)),
+ _ => match s.parse() {
+ Ok(key) => Ok(Self::Client(key)),
+ Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+ },
}
}
}
@@ -688,9 +697,7 @@ impl AmazonS3Builder {
for (os_key, os_value) in std::env::vars_os() {
if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) {
if key.starts_with("AWS_") {
- if let Ok(config_key) =
- AmazonS3ConfigKey::from_str(&key.to_ascii_lowercase())
- {
+ if let Ok(config_key) = key.to_ascii_lowercase().parse() {
builder = builder.with_config(config_key, value);
}
}
@@ -706,12 +713,6 @@ impl AmazonS3Builder {
Some(format!("{METADATA_ENDPOINT}{metadata_relative_uri}"));
}
- if let Ok(text) = std::env::var("AWS_ALLOW_HTTP") {
- builder.client_options = builder
- .client_options
- .with_config(ClientConfigKey::AllowHttp, text);
- }
-
builder
}
@@ -770,6 +771,9 @@ impl AmazonS3Builder {
AmazonS3ConfigKey::Checksum => {
self.checksum_algorithm = Some(ConfigValue::Deferred(value.into()))
}
+ AmazonS3ConfigKey::Client(key) => {
+ self.client_options = self.client_options.with_config(key, value)
+ }
};
self
}
@@ -834,6 +838,7 @@ impl AmazonS3Builder {
AmazonS3ConfigKey::Checksum => {
self.checksum_algorithm.as_ref().map(ToString::to_string)
}
+ AmazonS3ConfigKey::Client(key) => self.client_options.get_config_value(key),
}
}
diff --git a/object_store/src/azure/mod.rs b/object_store/src/azure/mod.rs
index 2b5b43ada..c2cfdfe6a 100644
--- a/object_store/src/azure/mod.rs
+++ b/object_store/src/azure/mod.rs
@@ -559,6 +559,9 @@ pub enum AzureConfigKey {
/// - `azure_use_azure_cli`
/// - `use_azure_cli`
UseAzureCli,
+
+ /// Client options
+ Client(ClientConfigKey),
}
impl AsRef<str> for AzureConfigKey {
@@ -577,6 +580,7 @@ impl AsRef<str> for AzureConfigKey {
Self::MsiResourceId => "azure_msi_resource_id",
Self::FederatedTokenFile => "azure_federated_token_file",
Self::UseAzureCli => "azure_use_azure_cli",
+ Self::Client(key) => key.as_ref(),
}
}
}
@@ -621,7 +625,12 @@ impl FromStr for AzureConfigKey {
Ok(Self::FederatedTokenFile)
}
"azure_use_azure_cli" | "use_azure_cli" => Ok(Self::UseAzureCli),
- _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+ // Backwards compatibility
+ "azure_allow_http" => Ok(Self::Client(ClientConfigKey::AllowHttp)),
+ _ => match s.parse() {
+ Ok(key) => Ok(Self::Client(key)),
+ Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+ },
}
}
}
@@ -664,21 +673,13 @@ impl MicrosoftAzureBuilder {
for (os_key, os_value) in std::env::vars_os() {
if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) {
if key.starts_with("AZURE_") {
- if let Ok(config_key) =
- AzureConfigKey::from_str(&key.to_ascii_lowercase())
- {
+ if let Ok(config_key) = key.to_ascii_lowercase().parse() {
builder = builder.with_config(config_key, value);
}
}
}
}
- if let Ok(text) = std::env::var("AZURE_ALLOW_HTTP") {
- builder.client_options = builder
- .client_options
- .with_config(ClientConfigKey::AllowHttp, text)
- }
-
if let Ok(text) = std::env::var(MSI_ENDPOINT_ENV_KEY) {
builder = builder.with_msi_endpoint(text);
}
@@ -731,6 +732,9 @@ impl MicrosoftAzureBuilder {
}
AzureConfigKey::UseAzureCli => self.use_azure_cli.parse(value),
AzureConfigKey::UseEmulator => self.use_emulator.parse(value),
+ AzureConfigKey::Client(key) => {
+ self.client_options = self.client_options.with_config(key, value)
+ }
};
self
}
@@ -786,6 +790,7 @@ impl MicrosoftAzureBuilder {
AzureConfigKey::MsiResourceId => self.msi_resource_id.clone(),
AzureConfigKey::FederatedTokenFile => self.federated_token_file.clone(),
AzureConfigKey::UseAzureCli => Some(self.use_azure_cli.to_string()),
+ AzureConfigKey::Client(key) => self.client_options.get_config_value(key),
}
}
diff --git a/object_store/src/client/mod.rs b/object_store/src/client/mod.rs
index d7b0b86d9..d2242dd41 100644
--- a/object_store/src/client/mod.rs
+++ b/object_store/src/client/mod.rs
@@ -31,6 +31,7 @@ use reqwest::header::{HeaderMap, HeaderValue};
use reqwest::{Client, ClientBuilder, Proxy};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
+use std::str::FromStr;
use std::time::Duration;
use crate::path::Path;
@@ -53,6 +54,28 @@ pub enum ClientConfigKey {
AllowHttp,
}
+impl AsRef<str> for ClientConfigKey {
+ fn as_ref(&self) -> &str {
+ match self {
+ Self::AllowHttp => "allow_http",
+ }
+ }
+}
+
+impl FromStr for ClientConfigKey {
+ type Err = super::Error;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s {
+ "allow_http" => Ok(Self::AllowHttp),
+ _ => Err(super::Error::UnknownConfigurationKey {
+ store: "HTTP",
+ key: s.into(),
+ }),
+ }
+ }
+}
+
/// HTTP client configuration for remote object stores
#[derive(Debug, Clone, Default)]
pub struct ClientOptions {
diff --git a/object_store/src/gcp/mod.rs b/object_store/src/gcp/mod.rs
index 6f3d53d42..375b4d8f8 100644
--- a/object_store/src/gcp/mod.rs
+++ b/object_store/src/gcp/mod.rs
@@ -49,6 +49,7 @@ use url::Url;
use crate::client::pagination::stream_paginated;
use crate::client::retry::RetryExt;
+use crate::client::ClientConfigKey;
use crate::{
client::token::TokenCache,
multipart::{CloudMultiPartUpload, CloudMultiPartUploadImpl, UploadPart},
@@ -829,6 +830,9 @@ pub enum GoogleConfigKey {
///
/// See [`GoogleCloudStorageBuilder::with_application_credentials`].
ApplicationCredentials,
+
+ /// Client options
+ Client(ClientConfigKey),
}
impl AsRef<str> for GoogleConfigKey {
@@ -838,6 +842,7 @@ impl AsRef<str> for GoogleConfigKey {
Self::ServiceAccountKey => "google_service_account_key",
Self::Bucket => "google_bucket",
Self::ApplicationCredentials => "google_application_credentials",
+ Self::Client(key) => key.as_ref(),
}
}
}
@@ -858,7 +863,10 @@ impl FromStr for GoogleConfigKey {
Ok(Self::Bucket)
}
"google_application_credentials" => Ok(Self::ApplicationCredentials),
- _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+ _ => match s.parse() {
+ Ok(key) => Ok(Self::Client(key)),
+ Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+ },
}
}
}
@@ -911,9 +919,7 @@ impl GoogleCloudStorageBuilder {
for (os_key, os_value) in std::env::vars_os() {
if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) {
if key.starts_with("GOOGLE_") {
- if let Ok(config_key) =
- GoogleConfigKey::from_str(&key.to_ascii_lowercase())
- {
+ if let Ok(config_key) = key.to_ascii_lowercase().parse() {
builder = builder.with_config(config_key, value);
}
}
@@ -957,6 +963,9 @@ impl GoogleCloudStorageBuilder {
GoogleConfigKey::ApplicationCredentials => {
self.application_credentials_path = Some(value.into())
}
+ GoogleConfigKey::Client(key) => {
+ self.client_options = self.client_options.with_config(key, value)
+ }
};
self
}
@@ -1005,6 +1014,7 @@ impl GoogleCloudStorageBuilder {
GoogleConfigKey::ApplicationCredentials => {
self.application_credentials_path.clone()
}
+ GoogleConfigKey::Client(key) => self.client_options.get_config_value(key),
}
}