You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2022/07/29 17:27:42 UTC

[arrow-rs] branch master updated: Add Builder style config objects for object_store (#2204)

This is an automated email from the ASF dual-hosted git repository.

alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git


The following commit(s) were added to refs/heads/master by this push:
     new 41d96b216 Add Builder style config objects for object_store (#2204)
41d96b216 is described below

commit 41d96b21628a9035623b72e80d7ea06e5fb904f9
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Fri Jul 29 13:27:37 2022 -0400

    Add Builder style config objects for object_store (#2204)
    
    * Add AmazonS3Config, MicrosoftAzureBuilder, GoogleCloudStorageBuilder
    
    * fix: improve docs
    
    * review feedback: remove old code, make with_client test only
---
 object_store/src/aws.rs   | 370 +++++++++++++++++++++++++++++-----------------
 object_store/src/azure.rs | 220 ++++++++++++++++-----------
 object_store/src/gcp.rs   | 230 +++++++++++++++++-----------
 3 files changed, 516 insertions(+), 304 deletions(-)

diff --git a/object_store/src/aws.rs b/object_store/src/aws.rs
index 3606a3806..89a218512 100644
--- a/object_store/src/aws.rs
+++ b/object_store/src/aws.rs
@@ -228,6 +228,14 @@ enum Error {
         source: rusoto_core::region::ParseRegionError,
     },
 
+    #[snafu(display(
+        "Region must be specified for AWS S3. Regions should look like `us-east-2`"
+    ))]
+    MissingRegion {},
+
+    #[snafu(display("Missing bucket name"))]
+    MissingBucketName {},
+
     #[snafu(display("Missing aws-access-key"))]
     MissingAccessKey,
 
@@ -584,99 +592,195 @@ fn convert_object_meta(object: rusoto_s3::Object, bucket: &str) -> Result<Object
 
 /// Configure a connection to Amazon S3 using the specified credentials in
 /// the specified Amazon region and bucket.
-#[allow(clippy::too_many_arguments)]
-pub fn new_s3(
-    access_key_id: Option<impl Into<String>>,
-    secret_access_key: Option<impl Into<String>>,
-    region: impl Into<String>,
-    bucket_name: impl Into<String>,
-    endpoint: Option<impl Into<String>>,
-    session_token: Option<impl Into<String>>,
+///
+/// # Example
+/// ```
+/// # let REGION = "foo";
+/// # let BUCKET_NAME = "foo";
+/// # let ACCESS_KEY_ID = "foo";
+/// # let SECRET_KEY = "foo";
+/// let s3 = object_store::aws::AmazonS3Builder::new()
+///  .with_region(REGION)
+///  .with_bucket_name(BUCKET_NAME)
+///  .with_access_key_id(ACCESS_KEY_ID)
+///  .with_secret_access_key(SECRET_KEY)
+///  .build();
+/// ```
+#[derive(Debug)]
+pub struct AmazonS3Builder {
+    access_key_id: Option<String>,
+    secret_access_key: Option<String>,
+    region: Option<String>,
+    bucket_name: Option<String>,
+    endpoint: Option<String>,
+    token: Option<String>,
     max_connections: NonZeroUsize,
     allow_http: bool,
-) -> Result<AmazonS3> {
-    let region = region.into();
-    let region: rusoto_core::Region = match endpoint {
-        None => region.parse().context(InvalidRegionSnafu { region })?,
-        Some(endpoint) => rusoto_core::Region::Custom {
-            name: region,
-            endpoint: endpoint.into(),
-        },
-    };
+}
 
-    let mut builder = HyperBuilder::default();
-    builder.pool_max_idle_per_host(max_connections.get());
-
-    let connector = if allow_http {
-        hyper_rustls::HttpsConnectorBuilder::new()
-            .with_webpki_roots()
-            .https_or_http()
-            .enable_http1()
-            .enable_http2()
-            .build()
-    } else {
-        hyper_rustls::HttpsConnectorBuilder::new()
-            .with_webpki_roots()
-            .https_only()
-            .enable_http1()
-            .enable_http2()
-            .build()
-    };
+impl Default for AmazonS3Builder {
+    fn default() -> Self {
+        Self {
+            access_key_id: None,
+            secret_access_key: None,
+            region: None,
+            bucket_name: None,
+            endpoint: None,
+            token: None,
+            max_connections: NonZeroUsize::new(16).unwrap(),
+            allow_http: false,
+        }
+    }
+}
 
-    let http_client = rusoto_core::request::HttpClient::from_builder(builder, connector);
+impl AmazonS3Builder {
+    /// Create a new [`AmazonS3Builder`] with default values.
+    pub fn new() -> Self {
+        Default::default()
+    }
 
-    let client = match (access_key_id, secret_access_key, session_token) {
-        (Some(access_key_id), Some(secret_access_key), Some(session_token)) => {
-            let credentials_provider = StaticProvider::new(
-                access_key_id.into(),
-                secret_access_key.into(),
-                Some(session_token.into()),
-                None,
-            );
-            rusoto_s3::S3Client::new_with(http_client, credentials_provider, region)
-        }
-        (Some(access_key_id), Some(secret_access_key), None) => {
-            let credentials_provider = StaticProvider::new_minimal(
-                access_key_id.into(),
-                secret_access_key.into(),
-            );
-            rusoto_s3::S3Client::new_with(http_client, credentials_provider, region)
-        }
-        (None, Some(_), _) => return Err(Error::MissingAccessKey.into()),
-        (Some(_), None, _) => return Err(Error::MissingSecretAccessKey.into()),
-        _ if std::env::var_os("AWS_WEB_IDENTITY_TOKEN_FILE").is_some() => {
-            rusoto_s3::S3Client::new_with(
-                http_client,
-                WebIdentityProvider::from_k8s_env(),
-                region,
-            )
-        }
-        _ => rusoto_s3::S3Client::new_with(
-            http_client,
-            InstanceMetadataProvider::new(),
+    /// Set the AWS Access Key (required)
+    pub fn with_access_key_id(mut self, access_key_id: impl Into<String>) -> Self {
+        self.access_key_id = Some(access_key_id.into());
+        self
+    }
+
+    /// Set the AWS Secret Access Key (required)
+    pub fn with_secret_access_key(
+        mut self,
+        secret_access_key: impl Into<String>,
+    ) -> Self {
+        self.secret_access_key = Some(secret_access_key.into());
+        self
+    }
+
+    /// Set the region (e.g. `us-east-1`) (required)
+    pub fn with_region(mut self, region: impl Into<String>) -> Self {
+        self.region = Some(region.into());
+        self
+    }
+
+    /// Set the bucket_name (required)
+    pub fn with_bucket_name(mut self, bucket_name: impl Into<String>) -> Self {
+        self.bucket_name = Some(bucket_name.into());
+        self
+    }
+
+    /// Sets the endpoint for communicating with AWS S3. Default value
+    /// is based on region.
+    ///
+    /// For example, this might be set to `"http://localhost:4566:`
+    /// for testing against a localstack instance.
+    pub fn with_endpoint(mut self, endpoint: impl Into<String>) -> Self {
+        self.endpoint = Some(endpoint.into());
+        self
+    }
+
+    /// Set the token to use for requests (passed to underlying provider)
+    pub fn with_token(mut self, token: impl Into<String>) -> Self {
+        self.token = Some(token.into());
+        self
+    }
+
+    /// Sets the maximum number of concurrent outstanding
+    /// connectons. Default is `16`.
+    pub fn with_max_connections(mut self, max_connections: NonZeroUsize) -> Self {
+        self.max_connections = max_connections;
+        self
+    }
+
+    /// Sets what protocol is allowed. If `allow_http` is :
+    /// * false (default):  Only HTTPS are allowed
+    /// * true:  HTTP and HTTPS are allowed
+    pub fn with_allow_http(mut self, allow_http: bool) -> Self {
+        self.allow_http = allow_http;
+        self
+    }
+
+    /// Create a [`AmazonS3`] instance from the provided values,
+    /// consuming `self`.
+    pub fn build(self) -> Result<AmazonS3> {
+        let Self {
+            access_key_id,
+            secret_access_key,
             region,
-        ),
-    };
+            bucket_name,
+            endpoint,
+            token,
+            max_connections,
+            allow_http,
+        } = self;
+
+        let region = region.ok_or(Error::MissingRegion {})?;
+        let bucket_name = bucket_name.ok_or(Error::MissingBucketName {})?;
+
+        let region: rusoto_core::Region = match endpoint {
+            None => region.parse().context(InvalidRegionSnafu { region })?,
+            Some(endpoint) => rusoto_core::Region::Custom {
+                name: region,
+                endpoint,
+            },
+        };
 
-    Ok(AmazonS3 {
-        client_unrestricted: client,
-        connection_semaphore: Arc::new(Semaphore::new(max_connections.get())),
-        bucket_name: bucket_name.into(),
-    })
-}
+        let mut builder = HyperBuilder::default();
+        builder.pool_max_idle_per_host(max_connections.get());
+
+        let connector = if allow_http {
+            hyper_rustls::HttpsConnectorBuilder::new()
+                .with_webpki_roots()
+                .https_or_http()
+                .enable_http1()
+                .enable_http2()
+                .build()
+        } else {
+            hyper_rustls::HttpsConnectorBuilder::new()
+                .with_webpki_roots()
+                .https_only()
+                .enable_http1()
+                .enable_http2()
+                .build()
+        };
+
+        let http_client =
+            rusoto_core::request::HttpClient::from_builder(builder, connector);
+
+        let client = match (access_key_id, secret_access_key, token) {
+            (Some(access_key_id), Some(secret_access_key), Some(token)) => {
+                let credentials_provider = StaticProvider::new(
+                    access_key_id,
+                    secret_access_key,
+                    Some(token),
+                    None,
+                );
+                rusoto_s3::S3Client::new_with(http_client, credentials_provider, region)
+            }
+            (Some(access_key_id), Some(secret_access_key), None) => {
+                let credentials_provider =
+                    StaticProvider::new_minimal(access_key_id, secret_access_key);
+                rusoto_s3::S3Client::new_with(http_client, credentials_provider, region)
+            }
+            (None, Some(_), _) => return Err(Error::MissingAccessKey.into()),
+            (Some(_), None, _) => return Err(Error::MissingSecretAccessKey.into()),
+            _ if std::env::var_os("AWS_WEB_IDENTITY_TOKEN_FILE").is_some() => {
+                rusoto_s3::S3Client::new_with(
+                    http_client,
+                    WebIdentityProvider::from_k8s_env(),
+                    region,
+                )
+            }
+            _ => rusoto_s3::S3Client::new_with(
+                http_client,
+                InstanceMetadataProvider::new(),
+                region,
+            ),
+        };
 
-/// Create a new [`AmazonS3`] that always errors
-pub fn new_failing_s3() -> Result<AmazonS3> {
-    new_s3(
-        Some("foo"),
-        Some("bar"),
-        "us-east-1",
-        "bucket",
-        None as Option<&str>,
-        None as Option<&str>,
-        NonZeroUsize::new(16).unwrap(),
-        true,
-    )
+        Ok(AmazonS3 {
+            client_unrestricted: client,
+            connection_semaphore: Arc::new(Semaphore::new(max_connections.get())),
+            bucket_name,
+        })
+    }
 }
 
 /// S3 client bundled w/ a semaphore permit.
@@ -1057,7 +1161,7 @@ mod tests {
             get_nonexistent_object, list_uses_directories_correctly, list_with_delimiter,
             put_get_delete_list, rename_and_copy, stream_get,
         },
-        Error as ObjectStoreError, ObjectStore,
+        Error as ObjectStoreError,
     };
     use bytes::Bytes;
     use std::env;
@@ -1067,17 +1171,9 @@ mod tests {
 
     const NON_EXISTENT_NAME: &str = "nonexistentname";
 
-    #[derive(Debug)]
-    struct AwsConfig {
-        access_key_id: String,
-        secret_access_key: String,
-        region: String,
-        bucket: String,
-        endpoint: Option<String>,
-        token: Option<String>,
-    }
-
-    // Helper macro to skip tests if TEST_INTEGRATION and the AWS environment variables are not set.
+    // Helper macro to skip tests if TEST_INTEGRATION and the AWS
+    // environment variables are not set. Returns a configured
+    // AmazonS3Builder
     macro_rules! maybe_skip_integration {
         () => {{
             dotenv::dotenv().ok();
@@ -1116,18 +1212,38 @@ mod tests {
                 );
                 return;
             } else {
-                AwsConfig {
-                    access_key_id: env::var("AWS_ACCESS_KEY_ID")
-                        .expect("already checked AWS_ACCESS_KEY_ID"),
-                    secret_access_key: env::var("AWS_SECRET_ACCESS_KEY")
-                        .expect("already checked AWS_SECRET_ACCESS_KEY"),
-                    region: env::var("AWS_DEFAULT_REGION")
-                        .expect("already checked AWS_DEFAULT_REGION"),
-                    bucket: env::var("OBJECT_STORE_BUCKET")
-                        .expect("already checked OBJECT_STORE_BUCKET"),
-                    endpoint: env::var("AWS_ENDPOINT").ok(),
-                    token: env::var("AWS_SESSION_TOKEN").ok(),
-                }
+                let config = AmazonS3Builder::new()
+                    .with_access_key_id(
+                        env::var("AWS_ACCESS_KEY_ID")
+                            .expect("already checked AWS_ACCESS_KEY_ID"),
+                    )
+                    .with_secret_access_key(
+                        env::var("AWS_SECRET_ACCESS_KEY")
+                            .expect("already checked AWS_SECRET_ACCESS_KEY"),
+                    )
+                    .with_region(
+                        env::var("AWS_DEFAULT_REGION")
+                            .expect("already checked AWS_DEFAULT_REGION"),
+                    )
+                    .with_bucket_name(
+                        env::var("OBJECT_STORE_BUCKET")
+                            .expect("already checked OBJECT_STORE_BUCKET"),
+                    )
+                    .with_allow_http(true);
+
+                let config = if let Some(endpoint) = env::var("AWS_ENDPOINT").ok() {
+                    config.with_endpoint(endpoint)
+                } else {
+                    config
+                };
+
+                let config = if let Some(token) = env::var("AWS_SESSION_TOKEN").ok() {
+                    config.with_token(token)
+                } else {
+                    config
+                };
+
+                config
             }
         }};
     }
@@ -1148,24 +1264,10 @@ mod tests {
         r
     }
 
-    fn make_integration(config: AwsConfig) -> AmazonS3 {
-        new_s3(
-            Some(config.access_key_id),
-            Some(config.secret_access_key),
-            config.region,
-            config.bucket,
-            config.endpoint,
-            config.token,
-            NonZeroUsize::new(16).unwrap(),
-            true,
-        )
-        .expect("Valid S3 config")
-    }
-
     #[tokio::test]
     async fn s3_test() {
         let config = maybe_skip_integration!();
-        let integration = make_integration(config);
+        let integration = config.build().unwrap();
 
         check_credentials(put_get_delete_list(&integration).await).unwrap();
         check_credentials(list_uses_directories_correctly(&integration).await).unwrap();
@@ -1177,7 +1279,7 @@ mod tests {
     #[tokio::test]
     async fn s3_test_get_nonexistent_location() {
         let config = maybe_skip_integration!();
-        let integration = make_integration(config);
+        let integration = config.build().unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
 
@@ -1204,9 +1306,8 @@ mod tests {
 
     #[tokio::test]
     async fn s3_test_get_nonexistent_bucket() {
-        let mut config = maybe_skip_integration!();
-        config.bucket = NON_EXISTENT_NAME.into();
-        let integration = make_integration(config);
+        let config = maybe_skip_integration!().with_bucket_name(NON_EXISTENT_NAME);
+        let integration = config.build().unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
 
@@ -1220,9 +1321,9 @@ mod tests {
 
     #[tokio::test]
     async fn s3_test_put_nonexistent_bucket() {
-        let mut config = maybe_skip_integration!();
-        config.bucket = NON_EXISTENT_NAME.into();
-        let integration = make_integration(config);
+        let config = maybe_skip_integration!().with_bucket_name(NON_EXISTENT_NAME);
+
+        let integration = config.build().unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
         let data = Bytes::from("arbitrary data");
@@ -1244,7 +1345,7 @@ mod tests {
     #[tokio::test]
     async fn s3_test_delete_nonexistent_location() {
         let config = maybe_skip_integration!();
-        let integration = make_integration(config);
+        let integration = config.build().unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
 
@@ -1253,9 +1354,8 @@ mod tests {
 
     #[tokio::test]
     async fn s3_test_delete_nonexistent_bucket() {
-        let mut config = maybe_skip_integration!();
-        config.bucket = NON_EXISTENT_NAME.into();
-        let integration = make_integration(config);
+        let config = maybe_skip_integration!().with_bucket_name(NON_EXISTENT_NAME);
+        let integration = config.build().unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
 
diff --git a/object_store/src/azure.rs b/object_store/src/azure.rs
index 25f311a9a..dca52a356 100644
--- a/object_store/src/azure.rs
+++ b/object_store/src/azure.rs
@@ -185,6 +185,15 @@ enum Error {
         env_value: String,
         source: url::ParseError,
     },
+
+    #[snafu(display("Account must be specified"))]
+    MissingAccount {},
+
+    #[snafu(display("Access key must be specified"))]
+    MissingAccessKey {},
+
+    #[snafu(display("Container name must be specified"))]
+    MissingContainerName {},
 }
 
 impl From<Error> for super::Error {
@@ -570,73 +579,125 @@ fn url_from_env(env_name: &str, default_url: &str) -> Result<Url> {
     Ok(url)
 }
 
-/// Configure a connection to container with given name on Microsoft Azure
-/// Blob store.
+/// Configure a connection to Mirosoft Azure Blob Storage bucket using
+/// the specified credentials.
 ///
-/// The credentials `account` and `access_key` must provide access to the
-/// store.
-pub fn new_azure(
-    account: impl Into<String>,
-    access_key: impl Into<String>,
-    container_name: impl Into<String>,
+/// # Example
+/// ```
+/// # let ACCOUNT = "foo";
+/// # let BUCKET_NAME = "foo";
+/// # let ACCESS_KEY = "foo";
+/// let azure = object_store::azure::MicrosoftAzureBuilder::new()
+///  .with_account(ACCOUNT)
+///  .with_access_key(ACCESS_KEY)
+///  .with_container_name(BUCKET_NAME)
+///  .build();
+/// ```
+#[derive(Debug, Default)]
+pub struct MicrosoftAzureBuilder {
+    account: Option<String>,
+    access_key: Option<String>,
+    container_name: Option<String>,
     use_emulator: bool,
-) -> Result<MicrosoftAzure> {
-    let account = account.into();
-    let access_key = access_key.into();
-    let http_client: Arc<dyn HttpClient> = Arc::new(reqwest::Client::new());
-
-    let (is_emulator, storage_account_client) = if use_emulator {
-        check_if_emulator_works()?;
-        // Allow overriding defaults. Values taken from
-        // from https://docs.rs/azure_storage/0.2.0/src/azure_storage/core/clients/storage_account_client.rs.html#129-141
-        let http_client = azure_core::new_http_client();
-        let blob_storage_url =
-            url_from_env("AZURITE_BLOB_STORAGE_URL", "http://127.0.0.1:10000")?;
-        let queue_storage_url =
-            url_from_env("AZURITE_QUEUE_STORAGE_URL", "http://127.0.0.1:10001")?;
-        let table_storage_url =
-            url_from_env("AZURITE_TABLE_STORAGE_URL", "http://127.0.0.1:10002")?;
-        let filesystem_url =
-            url_from_env("AZURITE_TABLE_STORAGE_URL", "http://127.0.0.1:10004")?;
-
-        let storage_client = StorageAccountClient::new_emulator(
-            http_client,
-            &blob_storage_url,
-            &table_storage_url,
-            &queue_storage_url,
-            &filesystem_url,
-        );
-
-        (true, storage_client)
-    } else {
-        (
-            false,
-            StorageAccountClient::new_access_key(
-                Arc::clone(&http_client),
-                &account,
-                &access_key,
-            ),
-        )
-    };
+}
 
-    let storage_client = storage_account_client.as_storage_client();
-    let blob_base_url = storage_account_client
-        .blob_storage_url()
-        .as_ref()
-        // make url ending consistent between the emulator and remote storage account
-        .trim_end_matches('/')
-        .to_string();
+impl MicrosoftAzureBuilder {
+    /// Create a new [`MicrosoftAzureBuilder`] with default values.
+    pub fn new() -> Self {
+        Default::default()
+    }
 
-    let container_name = container_name.into();
+    /// Set the Azure Account (required)
+    pub fn with_account(mut self, account: impl Into<String>) -> Self {
+        self.account = Some(account.into());
+        self
+    }
 
-    let container_client = storage_client.as_container_client(&container_name);
+    /// Set the Azure Access Key (required)
+    pub fn with_access_key(mut self, access_key: impl Into<String>) -> Self {
+        self.access_key = Some(access_key.into());
+        self
+    }
+
+    /// Set the Azure Container Name (required)
+    pub fn with_container_name(mut self, container_name: impl Into<String>) -> Self {
+        self.container_name = Some(container_name.into());
+        self
+    }
 
-    Ok(MicrosoftAzure {
-        container_client,
-        container_name,
-        blob_base_url,
-        is_emulator,
-    })
+    /// Set if the Azure emulator should be used (defaults to false)
+    pub fn with_use_emulator(mut self, use_emulator: bool) -> Self {
+        self.use_emulator = use_emulator;
+        self
+    }
+
+    /// Configure a connection to container with given name on Microsoft Azure
+    /// Blob store.
+    pub fn build(self) -> Result<MicrosoftAzure> {
+        let Self {
+            account,
+            access_key,
+            container_name,
+            use_emulator,
+        } = self;
+
+        let account = account.ok_or(Error::MissingAccount {})?;
+        let access_key = access_key.ok_or(Error::MissingAccessKey {})?;
+        let container_name = container_name.ok_or(Error::MissingContainerName {})?;
+
+        let http_client: Arc<dyn HttpClient> = Arc::new(reqwest::Client::new());
+
+        let (is_emulator, storage_account_client) = if use_emulator {
+            check_if_emulator_works()?;
+            // Allow overriding defaults. Values taken from
+            // from https://docs.rs/azure_storage/0.2.0/src/azure_storage/core/clients/storage_account_client.rs.html#129-141
+            let http_client = azure_core::new_http_client();
+            let blob_storage_url =
+                url_from_env("AZURITE_BLOB_STORAGE_URL", "http://127.0.0.1:10000")?;
+            let queue_storage_url =
+                url_from_env("AZURITE_QUEUE_STORAGE_URL", "http://127.0.0.1:10001")?;
+            let table_storage_url =
+                url_from_env("AZURITE_TABLE_STORAGE_URL", "http://127.0.0.1:10002")?;
+            let filesystem_url =
+                url_from_env("AZURITE_TABLE_STORAGE_URL", "http://127.0.0.1:10004")?;
+
+            let storage_client = StorageAccountClient::new_emulator(
+                http_client,
+                &blob_storage_url,
+                &table_storage_url,
+                &queue_storage_url,
+                &filesystem_url,
+            );
+
+            (true, storage_client)
+        } else {
+            (
+                false,
+                StorageAccountClient::new_access_key(
+                    Arc::clone(&http_client),
+                    &account,
+                    &access_key,
+                ),
+            )
+        };
+
+        let storage_client = storage_account_client.as_storage_client();
+        let blob_base_url = storage_account_client
+            .blob_storage_url()
+            .as_ref()
+            // make url ending consistent between the emulator and remote storage account
+            .trim_end_matches('/')
+            .to_string();
+
+        let container_client = storage_client.as_container_client(&container_name);
+
+        Ok(MicrosoftAzure {
+            container_client,
+            container_name,
+            blob_base_url,
+            is_emulator,
+        })
+    }
 }
 
 // Relevant docs: https://azure.github.io/Storage/docs/application-and-user-data/basics/azure-blob-storage-upload-apis/
@@ -729,21 +790,13 @@ impl CloudMultiPartUploadImpl for AzureMultiPartUpload {
 
 #[cfg(test)]
 mod tests {
-    use crate::azure::new_azure;
+    use super::*;
     use crate::tests::{
         copy_if_not_exists, list_uses_directories_correctly, list_with_delimiter,
         put_get_delete_list, rename_and_copy,
     };
     use std::env;
 
-    #[derive(Debug)]
-    struct AzureConfig {
-        storage_account: String,
-        access_key: String,
-        bucket: String,
-        use_emulator: bool,
-    }
-
     // Helper macro to skip tests if TEST_INTEGRATION and the Azure environment
     // variables are not set.
     macro_rules! maybe_skip_integration {
@@ -785,28 +838,23 @@ mod tests {
                 );
                 return;
             } else {
-                AzureConfig {
-                    storage_account: env::var("AZURE_STORAGE_ACCOUNT")
-                        .unwrap_or_default(),
-                    access_key: env::var("AZURE_STORAGE_ACCESS_KEY").unwrap_or_default(),
-                    bucket: env::var("OBJECT_STORE_BUCKET")
-                        .expect("already checked OBJECT_STORE_BUCKET"),
-                    use_emulator,
-                }
+                MicrosoftAzureBuilder::new()
+                    .with_account(env::var("AZURE_STORAGE_ACCOUNT").unwrap_or_default())
+                    .with_access_key(
+                        env::var("AZURE_STORAGE_ACCESS_KEY").unwrap_or_default(),
+                    )
+                    .with_container_name(
+                        env::var("OBJECT_STORE_BUCKET")
+                            .expect("already checked OBJECT_STORE_BUCKET"),
+                    )
+                    .with_use_emulator(use_emulator)
             }
         }};
     }
 
     #[tokio::test]
     async fn azure_blob_test() {
-        let config = maybe_skip_integration!();
-        let integration = new_azure(
-            config.storage_account,
-            config.access_key,
-            config.bucket,
-            config.use_emulator,
-        )
-        .unwrap();
+        let integration = maybe_skip_integration!().build().unwrap();
 
         put_get_delete_list(&integration).await.unwrap();
         list_uses_directories_correctly(&integration).await.unwrap();
diff --git a/object_store/src/gcp.rs b/object_store/src/gcp.rs
index d740625bd..dea8769a7 100644
--- a/object_store/src/gcp.rs
+++ b/object_store/src/gcp.rs
@@ -98,6 +98,12 @@ enum Error {
 
     #[snafu(display("Error decoding object size: {}", source))]
     InvalidSize { source: std::num::ParseIntError },
+
+    #[snafu(display("Missing bucket name"))]
+    MissingBucketName {},
+
+    #[snafu(display("Missing service account path"))]
+    MissingServiceAccountPath,
 }
 
 impl From<Error> for super::Error {
@@ -779,55 +785,121 @@ fn reader_credentials_file(
     Ok(serde_json::from_reader(reader).context(DecodeCredentialsSnafu)?)
 }
 
-/// Configure a connection to Google Cloud Storage.
-pub fn new_gcs(
-    service_account_path: impl AsRef<std::path::Path>,
-    bucket_name: impl Into<String>,
-) -> Result<GoogleCloudStorage> {
-    new_gcs_with_client(service_account_path, bucket_name, Client::new())
+/// Configure a connection to Google Cloud Storage using the specified
+/// credentials.
+///
+/// # Example
+/// ```
+/// # let BUCKET_NAME = "foo";
+/// # let SERVICE_ACCOUNT_PATH = "/tmp/foo.json";
+/// let gcs = object_store::gcp::GoogleCloudStorageBuilder::new()
+///  .with_service_account_path(SERVICE_ACCOUNT_PATH)
+///  .with_bucket_name(BUCKET_NAME)
+///  .build();
+/// ```
+#[derive(Debug, Default)]
+pub struct GoogleCloudStorageBuilder {
+    bucket_name: Option<String>,
+    service_account_path: Option<String>,
+    client: Option<Client>,
 }
 
-/// Configure a connection to Google Cloud Storage with the specified HTTP client.
-pub fn new_gcs_with_client(
-    service_account_path: impl AsRef<std::path::Path>,
-    bucket_name: impl Into<String>,
-    client: Client,
-) -> Result<GoogleCloudStorage> {
-    let credentials = reader_credentials_file(service_account_path)?;
-
-    // TODO: https://cloud.google.com/storage/docs/authentication#oauth-scopes
-    let scope = "https://www.googleapis.com/auth/devstorage.full_control";
-    let audience = "https://www.googleapis.com/oauth2/v4/token".to_string();
-
-    let oauth_provider = (!credentials.disable_oauth)
-        .then(|| {
-            OAuthProvider::new(
-                credentials.client_email,
-                credentials.private_key,
-                scope.to_string(),
-                audience,
-            )
-        })
-        .transpose()?;
+impl GoogleCloudStorageBuilder {
+    /// Create a new [`GoogleCloudStorageBuilder`] with default values.
+    pub fn new() -> Self {
+        Default::default()
+    }
 
-    let bucket_name = bucket_name.into();
-    let encoded_bucket_name =
-        percent_encode(bucket_name.as_bytes(), NON_ALPHANUMERIC).to_string();
+    /// Set the bucket name (required)
+    pub fn with_bucket_name(mut self, bucket_name: impl Into<String>) -> Self {
+        self.bucket_name = Some(bucket_name.into());
+        self
+    }
 
-    // The cloud storage crate currently only supports authentication via
-    // environment variables. Set the environment variable explicitly so
-    // that we can optionally accept command line arguments instead.
-    Ok(GoogleCloudStorage {
-        client: Arc::new(GoogleCloudStorageClient {
-            client,
-            base_url: credentials.gcs_base_url,
-            oauth_provider,
-            token_cache: Default::default(),
+    /// Set the path to the service account file (required). Example
+    /// `"/tmp/gcs.json"`
+    ///
+    /// Example contents of `gcs.json`:
+    ///
+    /// ```json
+    /// {
+    ///    "gcs_base_url": "https://localhost:4443",
+    ///    "disable_oauth": true,
+    ///    "client_email": "",
+    ///    "private_key": ""
+    /// }
+    /// ```
+    pub fn with_service_account_path(
+        mut self,
+        service_account_path: impl Into<String>,
+    ) -> Self {
+        self.service_account_path = Some(service_account_path.into());
+        self
+    }
+
+    /// Use the specified http [`Client`] (defaults to [`Client::new`])
+    ///
+    /// This allows you to set custom client options such as allowing
+    /// non secure connections or custom headers.
+    ///
+    /// NOTE: Currently only available in `test`s to facilitate
+    /// testing, to avoid leaking details and preserve our ability to
+    /// make changes to the implementation.
+    #[cfg(test)]
+    pub fn with_client(mut self, client: Client) -> Self {
+        self.client = Some(client);
+        self
+    }
+
+    /// Configure a connection to Google Cloud Storage, returning a
+    /// new [`GoogleCloudStorage`] and consuming `self`
+    pub fn build(self) -> Result<GoogleCloudStorage> {
+        let Self {
             bucket_name,
-            bucket_name_encoded: encoded_bucket_name,
-            max_list_results: None,
-        }),
-    })
+            service_account_path,
+            client,
+        } = self;
+
+        let bucket_name = bucket_name.ok_or(Error::MissingBucketName {})?;
+        let service_account_path =
+            service_account_path.ok_or(Error::MissingServiceAccountPath)?;
+        let client = client.unwrap_or_else(Client::new);
+
+        let credentials = reader_credentials_file(service_account_path)?;
+
+        // TODO: https://cloud.google.com/storage/docs/authentication#oauth-scopes
+        let scope = "https://www.googleapis.com/auth/devstorage.full_control";
+        let audience = "https://www.googleapis.com/oauth2/v4/token".to_string();
+
+        let oauth_provider = (!credentials.disable_oauth)
+            .then(|| {
+                OAuthProvider::new(
+                    credentials.client_email,
+                    credentials.private_key,
+                    scope.to_string(),
+                    audience,
+                )
+            })
+            .transpose()?;
+
+        let encoded_bucket_name =
+            percent_encode(bucket_name.as_bytes(), NON_ALPHANUMERIC).to_string();
+
+        // The cloud storage crate currently only supports authentication via
+        // environment variables. Set the environment variable explicitly so
+        // that we can optionally accept command line arguments instead.
+        Ok(GoogleCloudStorage {
+            client: Arc::new(GoogleCloudStorageClient {
+                client,
+                base_url: credentials.gcs_base_url,
+                oauth_provider,
+                token_cache: Default::default(),
+                bucket_name,
+                bucket_name_encoded: encoded_bucket_name,
+                max_list_results: None,
+            }),
+        })
+    }
 }
 
 fn convert_object_meta(object: &Object) -> Result<ObjectMeta> {
@@ -860,24 +932,6 @@ mod test {
 
     const NON_EXISTENT_NAME: &str = "nonexistentname";
 
-    #[derive(Debug)]
-    struct GoogleCloudConfig {
-        bucket: String,
-        service_account: String,
-    }
-
-    impl GoogleCloudConfig {
-        fn build_test(self) -> Result<GoogleCloudStorage> {
-            // ignore HTTPS errors in tests so we can use fake-gcs server
-            let client = Client::builder()
-                .danger_accept_invalid_certs(true)
-                .build()
-                .expect("Error creating http client for testing");
-
-            new_gcs_with_client(self.service_account, self.bucket, client)
-        }
-    }
-
     // Helper macro to skip tests if TEST_INTEGRATION and the GCP environment variables are not set.
     macro_rules! maybe_skip_integration {
         () => {{
@@ -912,20 +966,29 @@ mod test {
                 );
                 return;
             } else {
-                GoogleCloudConfig {
-                    bucket: env::var("OBJECT_STORE_BUCKET")
-                        .expect("already checked OBJECT_STORE_BUCKET"),
-                    service_account: env::var("GOOGLE_SERVICE_ACCOUNT")
-                        .expect("already checked GOOGLE_SERVICE_ACCOUNT"),
-                }
+                GoogleCloudStorageBuilder::new()
+                    .with_bucket_name(
+                        env::var("OBJECT_STORE_BUCKET")
+                            .expect("already checked OBJECT_STORE_BUCKET")
+                    )
+                    .with_service_account_path(
+                        env::var("GOOGLE_SERVICE_ACCOUNT")
+                            .expect("already checked GOOGLE_SERVICE_ACCOUNT")
+                    )
+                    .with_client(
+                        // ignore HTTPS errors in tests so we can use fake-gcs server
+                        Client::builder()
+                            .danger_accept_invalid_certs(true)
+                            .build()
+                            .expect("Error creating http client for testing")
+                    )
             }
         }};
     }
 
     #[tokio::test]
     async fn gcs_test() {
-        let config = maybe_skip_integration!();
-        let integration = config.build_test().unwrap();
+        let integration = maybe_skip_integration!().build().unwrap();
 
         put_get_delete_list(&integration).await.unwrap();
         list_uses_directories_correctly(&integration).await.unwrap();
@@ -940,8 +1003,7 @@ mod test {
 
     #[tokio::test]
     async fn gcs_test_get_nonexistent_location() {
-        let config = maybe_skip_integration!();
-        let integration = config.build_test().unwrap();
+        let integration = maybe_skip_integration!().build().unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
 
@@ -956,9 +1018,10 @@ mod test {
 
     #[tokio::test]
     async fn gcs_test_get_nonexistent_bucket() {
-        let mut config = maybe_skip_integration!();
-        config.bucket = NON_EXISTENT_NAME.into();
-        let integration = config.build_test().unwrap();
+        let integration = maybe_skip_integration!()
+            .with_bucket_name(NON_EXISTENT_NAME)
+            .build()
+            .unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
 
@@ -975,8 +1038,7 @@ mod test {
 
     #[tokio::test]
     async fn gcs_test_delete_nonexistent_location() {
-        let config = maybe_skip_integration!();
-        let integration = config.build_test().unwrap();
+        let integration = maybe_skip_integration!().build().unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
 
@@ -990,9 +1052,10 @@ mod test {
 
     #[tokio::test]
     async fn gcs_test_delete_nonexistent_bucket() {
-        let mut config = maybe_skip_integration!();
-        config.bucket = NON_EXISTENT_NAME.into();
-        let integration = config.build_test().unwrap();
+        let integration = maybe_skip_integration!()
+            .with_bucket_name(NON_EXISTENT_NAME)
+            .build()
+            .unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
 
@@ -1006,9 +1069,10 @@ mod test {
 
     #[tokio::test]
     async fn gcs_test_put_nonexistent_bucket() {
-        let mut config = maybe_skip_integration!();
-        config.bucket = NON_EXISTENT_NAME.into();
-        let integration = config.build_test().unwrap();
+        let integration = maybe_skip_integration!()
+            .with_bucket_name(NON_EXISTENT_NAME)
+            .build()
+            .unwrap();
 
         let location = Path::from_iter([NON_EXISTENT_NAME]);
         let data = Bytes::from("arbitrary data");