You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2024/07/19 19:50:15 UTC
(iceberg-python) branch main updated: Standardize AWS credential names (#922)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new d0bfb4a6 Standardize AWS credential names (#922)
d0bfb4a6 is described below
commit d0bfb4a67bccd15c2d2e9baa481a5f33fe4c5220
Author: Honah J. <ho...@apache.org>
AuthorDate: Fri Jul 19 12:50:10 2024 -0700
Standardize AWS credential names (#922)
* first try
* glue + s3
* add dynamo
* change from aws. to client.
* add missing label
* change to PropertyUtil.get_first_property_value
* add unit tests for glue and dynamodb
* add unit tests for fileio
* add doc
* revise doc
* resolve review comments
* adopt kevinjqliu's suggestions
---
mkdocs/docs/configuration.md | 194 +++++++++++++++++++++++++++++------------
pyiceberg/catalog/__init__.py | 27 ++++++
pyiceberg/catalog/dynamodb.py | 35 ++++++--
pyiceberg/catalog/glue.py | 33 +++++--
pyiceberg/io/__init__.py | 4 +
pyiceberg/io/fsspec.py | 14 ++-
pyiceberg/io/pyarrow.py | 14 ++-
pyiceberg/table/__init__.py | 7 ++
tests/catalog/test_dynamodb.py | 62 ++++++++++++-
tests/catalog/test_glue.py | 56 +++++++++++-
tests/conftest.py | 15 ++++
tests/io/test_fsspec.py | 55 ++++++++++++
tests/io/test_pyarrow.py | 50 ++++++++++-
13 files changed, 487 insertions(+), 79 deletions(-)
diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md
index 76e1816c..985b8a77 100644
--- a/mkdocs/docs/configuration.md
+++ b/mkdocs/docs/configuration.md
@@ -22,35 +22,11 @@ hide:
- under the License.
-->
-# Catalogs
-
-PyIceberg currently has native support for REST, SQL, Hive, Glue and DynamoDB.
-
-There are three ways to pass in configuration:
-
-- Using the `~/.pyiceberg.yaml` configuration file
-- Through environment variables
-- By passing in credentials through the CLI or the Python API
-
-The configuration file is recommended since that's the easiest way to manage the credentials.
-
-Another option is through environment variables:
-
-```sh
-export PYICEBERG_CATALOG__DEFAULT__URI=thrift://localhost:9083
-export PYICEBERG_CATALOG__DEFAULT__S3__ACCESS_KEY_ID=username
-export PYICEBERG_CATALOG__DEFAULT__S3__SECRET_ACCESS_KEY=password
-```
-
-The environment variable picked up by Iceberg starts with `PYICEBERG_` and then follows the yaml structure below, where a double underscore `__` represents a nested field, and the underscore `_` is converted into a dash `-`.
-
-For example, `PYICEBERG_CATALOG__DEFAULT__S3__ACCESS_KEY_ID`, sets `s3.access-key-id` on the `default` catalog.
-
-# Tables
+## Tables
Iceberg tables support table properties to configure table behavior.
-## Write options
+### Write options
| Key | Options | Default | Description |
| --------------------------------- | --------------------------------- | ------- | ------------------------------------------------------------------------------------------- |
@@ -61,7 +37,7 @@ Iceberg tables support table properties to configure table behavior.
| `write.parquet.dict-size-bytes` | Size in bytes | 2MB | Set the dictionary page size limit per row group |
| `write.parquet.row-group-limit` | Number of rows | 122880 | The Parquet row group limit |
-## Table behavior options
+### Table behavior options
| Key | Options | Default | Description |
| ------------------------------------ | ------------------- | ------------- | ----------------------------------------------------------- |
@@ -76,7 +52,7 @@ Iceberg tables support table properties to configure table behavior.
<!-- prettier-ignore-end -->
-# FileIO
+## FileIO
Iceberg works with the concept of a FileIO which is a pluggable module for reading, writing, and deleting files. By default, PyIceberg will try to initialize the FileIO that's suitable for the scheme (`s3://`, `gs://`, etc.) and will use the first one that's installed.
@@ -101,8 +77,9 @@ For the FileIO there are several configuration options available:
| Key | Example | Description |
| -------------------- | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| s3.endpoint | https://10.0.19.25/ | Configure an alternative endpoint of the S3 service for the FileIO to access. This could be used to use S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. |
-| s3.access-key-id | admin | Configure the static secret access key used to access the FileIO. |
-| s3.secret-access-key | password | Configure the static session token used to access the FileIO. |
+| s3.access-key-id | admin | Configure the static access key id used to access the FileIO. |
+| s3.secret-access-key | password | Configure the static secret access key used to access the FileIO. |
+| s3.session-token | AQoDYXdzEJr... | Configure the static session token used to access the FileIO. |
| s3.signer | bearer | Configure the signature version of the FileIO. |
| s3.signer.uri | http://my.signer:8080/s3 | Configure the remote signing uri if it differs from the catalog uri. Remote signing is only implemented for `FsspecFileIO`. The final request is sent to `<s3.singer.uri>/v1/aws/s3/sign`. |
| s3.region | us-west-2 | Sets the region of the bucket |
@@ -160,7 +137,31 @@ For the FileIO there are several configuration options available:
<!-- markdown-link-check-enable-->
-## REST Catalog
+## Catalogs
+
+PyIceberg currently has native support for REST, SQL, Hive, Glue and DynamoDB.
+
+There are three ways to pass in configuration:
+
+- Using the `~/.pyiceberg.yaml` configuration file
+- Through environment variables
+- By passing in credentials through the CLI or the Python API
+
+The configuration file is recommended since that's the easiest way to manage the credentials.
+
+Another option is through environment variables:
+
+```sh
+export PYICEBERG_CATALOG__DEFAULT__URI=thrift://localhost:9083
+export PYICEBERG_CATALOG__DEFAULT__S3__ACCESS_KEY_ID=username
+export PYICEBERG_CATALOG__DEFAULT__S3__SECRET_ACCESS_KEY=password
+```
+
+The environment variable picked up by Iceberg starts with `PYICEBERG_` and then follows the yaml structure below, where a double underscore `__` represents a nested field, and the underscore `_` is converted into a dash `-`.
+
+For example, `PYICEBERG_CATALOG__DEFAULT__S3__ACCESS_KEY_ID`, sets `s3.access-key-id` on the `default` catalog.
+
+### REST Catalog
```yaml
catalog:
@@ -195,7 +196,7 @@ catalog:
<!-- markdown-link-check-enable-->
-### Headers in RESTCatalog
+#### Headers in RESTCatalog
To configure custom headers in RESTCatalog, include them in the catalog properties with the prefix `header.`. This
ensures that all HTTP requests to the REST service include the specified headers.
@@ -208,7 +209,7 @@ catalog:
header.content-type: application/vnd.api+json
```
-## SQL Catalog
+### SQL Catalog
The SQL catalog requires a database for its backend. PyIceberg supports PostgreSQL and SQLite through psycopg2. The database connection has to be configured using the `uri` property. See SQLAlchemy's [documentation for URL format](https://docs.sqlalchemy.org/en/20/core/engines.html#backend-specific-urls):
@@ -243,7 +244,7 @@ catalog:
| echo | true | false | SQLAlchemy engine [echo param](https://docs.sqlalchemy.org/en/20/core/engines.html#sqlalchemy.create_engine.params.echo) to log all statements to the default log handler |
| pool_pre_ping | true | false | SQLAlchemy engine [pool_pre_ping param](https://docs.sqlalchemy.org/en/20/core/engines.html#sqlalchemy.create_engine.params.pool_pre_ping) to test connections for liveness upon each checkout |
-## Hive Catalog
+### Hive Catalog
```yaml
catalog:
@@ -263,46 +264,71 @@ catalog:
hive.hive2-compatible: true
```
-## Glue Catalog
+### Glue Catalog
Your AWS credentials can be passed directly through the Python API.
Otherwise, please refer to
[How to configure AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) to set your AWS account credentials locally.
-If you did not set up a default AWS profile, you can configure the `profile_name`.
```yaml
catalog:
default:
type: glue
- aws_access_key_id: <ACCESS_KEY_ID>
- aws_secret_access_key: <SECRET_ACCESS_KEY>
- aws_session_token: <SESSION_TOKEN>
- region_name: <REGION_NAME>
+ glue.access-key-id: <ACCESS_KEY_ID>
+ glue.secret-access-key: <SECRET_ACCESS_KEY>
+ glue.session-token: <SESSION_TOKEN>
+ glue.region: <REGION_NAME>
+ s3.endpoint: http://localhost:9000
+ s3.access-key-id: admin
+ s3.secret-access-key: password
```
```yaml
catalog:
default:
type: glue
- profile_name: <PROFILE_NAME>
- region_name: <REGION_NAME>
+ glue.profile-name: <PROFILE_NAME>
+ glue.region: <REGION_NAME>
+ s3.endpoint: http://localhost:9000
+ s3.access-key-id: admin
+ s3.secret-access-key: password
```
+<!-- prettier-ignore-start -->
+
+!!! Note "Client-specific Properties"
+ `glue.*` properties are for Glue Catalog only. If you want to use the same credentials for both Glue Catalog and S3 FileIO, you can set the `client.*` properties. See the [Unified AWS Credentials](configuration.md#unified-aws-credentials) section for more details.
+
+<!-- prettier-ignore-end -->
+
<!-- markdown-link-check-disable -->
-| Key | Example | Description |
-| ----------------- | ------------------------------------ | ------------------------------------------------------------------------------- |
-| glue.id | 111111111111 | Configure the 12-digit ID of the Glue Catalog |
-| glue.skip-archive | true | Configure whether to skip the archival of older table versions. Default to true |
-| glue.endpoint | https://glue.us-east-1.amazonaws.com | Configure an alternative endpoint of the Glue service for GlueCatalog to access |
+| Key | Example | Description |
+| ---------------------- | ------------------------------------ | ------------------------------------------------------------------------------- |
+| glue.id | 111111111111 | Configure the 12-digit ID of the Glue Catalog |
+| glue.skip-archive | true | Configure whether to skip the archival of older table versions. Default to true |
+| glue.endpoint | https://glue.us-east-1.amazonaws.com | Configure an alternative endpoint of the Glue service for GlueCatalog to access |
+| glue.profile-name | default | Configure the static profile used to access the Glue Catalog |
+| glue.region | us-east-1 | Set the region of the Glue Catalog |
+| glue.access-key-id | admin | Configure the static access key id used to access the Glue Catalog |
+| glue.secret-access-key | password | Configure the static secret access key used to access the Glue Catalog |
+| glue.session-token | AQoDYXdzEJr... | Configure the static session token used to access the Glue Catalog |
<!-- markdown-link-check-enable-->
-## DynamoDB Catalog
+<!-- prettier-ignore-start -->
+
+!!! warning "Deprecated Properties"
+ `profile_name`, `region_name`, `botocore_session`, `aws_access_key_id`, `aws_secret_access_key`, `aws_session_token` are deprecated and will be removed in 0.8.0:
+
+<!-- prettier-ignore-end -->
+
+### DynamoDB Catalog
If you want to use AWS DynamoDB as the catalog, you can use the last two ways to configure the pyiceberg and refer
[How to configure AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html)
to set your AWS account credentials locally.
+If you want to use the same credentials for both Dynamodb Catalog and S3 FileIO, you can set the [`client.*` properties](configuration.md#unified-aws-credentials).
```yaml
catalog:
@@ -318,20 +344,78 @@ catalog:
default:
type: dynamodb
table-name: iceberg
- aws_access_key_id: <ACCESS_KEY_ID>
- aws_secret_access_key: <SECRET_ACCESS_KEY>
- aws_session_token: <SESSION_TOKEN>
- region_name: <REGION_NAME>
+ dynamodb.access-key-id: <ACCESS_KEY_ID>
+ dynamodb.secret-access-key: <SECRET_ACCESS_KEY>
+ dynamodb.session-token: <SESSION_TOKEN>
+ dynamodb.region: <REGION_NAME>
+ s3.endpoint: http://localhost:9000
+ s3.access-key-id: admin
+ s3.secret-access-key: password
```
-# Concurrency
+<!-- prettier-ignore-start -->
+
+!!! Note "Client-specific Properties"
+ `dynamodb.*` properties are for DynamoDB Catalog only. If you want to use the same credentials for both DynamoDB Catalog and S3 FileIO, you can set the `client.*` properties. See the [Unified AWS Credentials](configuration.md#unified-aws-credentials) section for more details.
+
+<!-- prettier-ignore-end -->
+
+<!-- markdown-link-check-disable -->
+
+| Key | Example | Description |
+| -------------------------- | -------------- | -------------------------------------------------------------------------- |
+| dynamodb.profile-name | default | Configure the static profile used to access the DynamoDB Catalog |
+| dynamodb.region | us-east-1 | Set the region of the DynamoDB Catalog |
+| dynamodb.access-key-id | admin | Configure the static access key id used to access the DynamoDB Catalog |
+| dynamodb.secret-access-key | password | Configure the static secret access key used to access the DynamoDB Catalog |
+| dynamodb.session-token | AQoDYXdzEJr... | Configure the static session token used to access the DynamoDB Catalog |
+
+<!-- markdown-link-check-enable-->
+
+<!-- prettier-ignore-start -->
+
+!!! warning "Deprecated Properties"
+ `profile_name`, `region_name`, `botocore_session`, `aws_access_key_id`, `aws_secret_access_key`, `aws_session_token` are deprecated and will be removed in 0.8.0:
+
+<!-- prettier-ignore-end -->
+
+## Unified AWS Credentials
+
+You can explicitly set the AWS credentials for both Glue/DynamoDB Catalog and S3 FileIO by configuring `client.*` properties. For example:
+
+```yaml
+catalog:
+ default:
+ type: glue
+ client.access-key-id: <ACCESS_KEY_ID>
+ client.secret-access-key: <SECRET_ACCESS_KEY>
+ client.region: <REGION_NAME>
+```
+
+configures the AWS credentials for both Glue Catalog and S3 FileIO.
+
+| Key | Example | Description |
+| ------------------------ | -------------- | ------------------------------------------------------------------------------------------------------ |
+| client.region | us-east-1 | Set the region of both the Glue/DynamoDB Catalog and the S3 FileIO |
+| client.access-key-id | admin | Configure the static access key id used to access both the Glue/DynamoDB Catalog and the S3 FileIO |
+| client.secret-access-key | password | Configure the static secret access key used to access both the Glue/DynamoDB Catalog and the S3 FileIO |
+| client.session-token | AQoDYXdzEJr... | Configure the static session token used to access both the Glue/DynamoDB Catalog and the S3 FileIO |
+
+<!-- prettier-ignore-start -->
+
+!!! Note "Properties Priority"
+ `client.*` properties will be overridden by service-specific properties if they are set. For example, if `client.region` is set to `us-west-1` and `s3.region` is set to `us-east-1`, the S3 FileIO will use `us-east-1` as the region.
+
+<!-- prettier-ignore-end -->
+
+## Concurrency
PyIceberg uses multiple threads to parallelize operations. The number of workers can be configured by supplying a `max-workers` entry in the configuration file, or by setting the `PYICEBERG_MAX_WORKERS` environment variable. The default value depends on the system hardware and Python version. See [the Python documentation](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor) for more details.
-# Backward Compatibility
+## Backward Compatibility
Previous versions of Java (`<1.4.0`) implementations incorrectly assume the optional attribute `current-snapshot-id` to be a required attribute in TableMetadata. This means that if `current-snapshot-id` is missing in the metadata file (e.g. on table creation), the application will throw an exception without being able to load the table. This assumption has been corrected in more recent Iceberg versions. However, it is possible to force PyIceberg to create a table with a metadata file tha [...]
-# Nanoseconds Support
+## Nanoseconds Support
PyIceberg currently only supports upto microsecond precision in its TimestampType. PyArrow timestamp types in 's' and 'ms' will be upcast automatically to 'us' precision timestamps on write. Timestamps in 'ns' precision can also be downcast automatically on write if desired. This can be configured by setting the `downcast-ns-timestamp-to-us-on-write` property as "True" in the configuration file, or by setting the `PYICEBERG_DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE` environment variable. Refe [...]
diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py
index ae978329..a84bde0d 100644
--- a/pyiceberg/catalog/__init__.py
+++ b/pyiceberg/catalog/__init__.py
@@ -66,6 +66,7 @@ from pyiceberg.typedef import (
RecursiveDict,
)
from pyiceberg.utils.config import Config, merge_config
+from pyiceberg.utils.deprecated import deprecated
if TYPE_CHECKING:
import pyarrow as pa
@@ -100,6 +101,21 @@ TABLE_METADATA_FILE_NAME_REGEX = re.compile(
re.X,
)
+DEPRECATED_PROFILE_NAME = "profile_name"
+DEPRECATED_REGION = "region_name"
+DEPRECATED_BOTOCORE_SESSION = "botocore_session"
+DEPRECATED_ACCESS_KEY_ID = "aws_access_key_id"
+DEPRECATED_SECRET_ACCESS_KEY = "aws_secret_access_key"
+DEPRECATED_SESSION_TOKEN = "aws_session_token"
+DEPRECATED_PROPERTY_NAMES = {
+ DEPRECATED_PROFILE_NAME,
+ DEPRECATED_REGION,
+ DEPRECATED_BOTOCORE_SESSION,
+ DEPRECATED_ACCESS_KEY_ID,
+ DEPRECATED_SECRET_ACCESS_KEY,
+ DEPRECATED_SESSION_TOKEN,
+}
+
class CatalogType(Enum):
REST = "rest"
@@ -692,6 +708,17 @@ class Catalog(ABC):
class MetastoreCatalog(Catalog, ABC):
+ def __init__(self, name: str, **properties: str):
+ super().__init__(name, **properties)
+
+ for property_name in DEPRECATED_PROPERTY_NAMES:
+ if self.properties.get(property_name):
+ deprecated(
+ deprecated_in="0.7.0",
+ removed_in="0.8.0",
+ help_message=f"The property {property_name} is deprecated. Please use properties that start with client., glue., and dynamo. instead",
+ )(lambda: None)()
+
def create_table_transaction(
self,
identifier: Union[str, Identifier],
diff --git a/pyiceberg/catalog/dynamodb.py b/pyiceberg/catalog/dynamodb.py
index bc5cbede..7cb5d985 100644
--- a/pyiceberg/catalog/dynamodb.py
+++ b/pyiceberg/catalog/dynamodb.py
@@ -29,6 +29,12 @@ from typing import (
import boto3
from pyiceberg.catalog import (
+ DEPRECATED_ACCESS_KEY_ID,
+ DEPRECATED_BOTOCORE_SESSION,
+ DEPRECATED_PROFILE_NAME,
+ DEPRECATED_REGION,
+ DEPRECATED_SECRET_ACCESS_KEY,
+ DEPRECATED_SESSION_TOKEN,
ICEBERG,
METADATA_LOCATION,
PREVIOUS_METADATA_LOCATION,
@@ -47,7 +53,7 @@ from pyiceberg.exceptions import (
NoSuchTableError,
TableAlreadyExistsError,
)
-from pyiceberg.io import load_file_io
+from pyiceberg.io import AWS_ACCESS_KEY_ID, AWS_REGION, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, load_file_io
from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec
from pyiceberg.schema import Schema
from pyiceberg.serializers import FromInputFile
@@ -78,17 +84,32 @@ PROPERTY_KEY_PREFIX = "p."
ACTIVE = "ACTIVE"
ITEM = "Item"
+DYNAMODB_PROFILE_NAME = "dynamodb.profile-name"
+DYNAMODB_REGION = "dynamodb.region"
+DYNAMODB_ACCESS_KEY_ID = "dynamodb.access-key-id"
+DYNAMODB_SECRET_ACCESS_KEY = "dynamodb.secret-access-key"
+DYNAMODB_SESSION_TOKEN = "dynamodb.session-token"
+
class DynamoDbCatalog(MetastoreCatalog):
def __init__(self, name: str, **properties: str):
super().__init__(name, **properties)
+
+ from pyiceberg.table import PropertyUtil
+
session = boto3.Session(
- profile_name=properties.get("profile_name"),
- region_name=properties.get("region_name"),
- botocore_session=properties.get("botocore_session"),
- aws_access_key_id=properties.get("aws_access_key_id"),
- aws_secret_access_key=properties.get("aws_secret_access_key"),
- aws_session_token=properties.get("aws_session_token"),
+ profile_name=PropertyUtil.get_first_property_value(properties, DYNAMODB_PROFILE_NAME, DEPRECATED_PROFILE_NAME),
+ region_name=PropertyUtil.get_first_property_value(properties, DYNAMODB_REGION, AWS_REGION, DEPRECATED_REGION),
+ botocore_session=properties.get(DEPRECATED_BOTOCORE_SESSION),
+ aws_access_key_id=PropertyUtil.get_first_property_value(
+ properties, DYNAMODB_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID, DEPRECATED_ACCESS_KEY_ID
+ ),
+ aws_secret_access_key=PropertyUtil.get_first_property_value(
+ properties, DYNAMODB_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, DEPRECATED_SECRET_ACCESS_KEY
+ ),
+ aws_session_token=PropertyUtil.get_first_property_value(
+ properties, DYNAMODB_SESSION_TOKEN, AWS_SESSION_TOKEN, DEPRECATED_SESSION_TOKEN
+ ),
)
self.dynamodb = session.client(DYNAMODB_CLIENT)
self.dynamodb_table_name = self.properties.get(DYNAMODB_TABLE_NAME, DYNAMODB_TABLE_NAME_DEFAULT)
diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py
index bc85b977..fa974a6f 100644
--- a/pyiceberg/catalog/glue.py
+++ b/pyiceberg/catalog/glue.py
@@ -39,6 +39,12 @@ from mypy_boto3_glue.type_defs import (
)
from pyiceberg.catalog import (
+ DEPRECATED_ACCESS_KEY_ID,
+ DEPRECATED_BOTOCORE_SESSION,
+ DEPRECATED_PROFILE_NAME,
+ DEPRECATED_REGION,
+ DEPRECATED_SECRET_ACCESS_KEY,
+ DEPRECATED_SESSION_TOKEN,
EXTERNAL_TABLE,
ICEBERG,
LOCATION,
@@ -58,6 +64,7 @@ from pyiceberg.exceptions import (
NoSuchTableError,
TableAlreadyExistsError,
)
+from pyiceberg.io import AWS_ACCESS_KEY_ID, AWS_REGION, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN
from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec
from pyiceberg.schema import Schema, SchemaVisitor, visit
from pyiceberg.serializers import FromInputFile
@@ -117,6 +124,12 @@ ICEBERG_FIELD_ID = "iceberg.field.id"
ICEBERG_FIELD_OPTIONAL = "iceberg.field.optional"
ICEBERG_FIELD_CURRENT = "iceberg.field.current"
+GLUE_PROFILE_NAME = "glue.profile-name"
+GLUE_REGION = "glue.region"
+GLUE_ACCESS_KEY_ID = "glue.access-key-id"
+GLUE_SECRET_ACCESS_KEY = "glue.secret-access-key"
+GLUE_SESSION_TOKEN = "glue.session-token"
+
def _construct_parameters(
metadata_location: str, glue_table: Optional[TableTypeDef] = None, prev_metadata_location: Optional[str] = None
@@ -285,13 +298,21 @@ class GlueCatalog(MetastoreCatalog):
def __init__(self, name: str, **properties: Any):
super().__init__(name, **properties)
+ from pyiceberg.table import PropertyUtil
+
session = boto3.Session(
- profile_name=properties.get("profile_name"),
- region_name=properties.get("region_name"),
- botocore_session=properties.get("botocore_session"),
- aws_access_key_id=properties.get("aws_access_key_id"),
- aws_secret_access_key=properties.get("aws_secret_access_key"),
- aws_session_token=properties.get("aws_session_token"),
+ profile_name=PropertyUtil.get_first_property_value(properties, GLUE_PROFILE_NAME, DEPRECATED_PROFILE_NAME),
+ region_name=PropertyUtil.get_first_property_value(properties, GLUE_REGION, AWS_REGION, DEPRECATED_REGION),
+ botocore_session=properties.get(DEPRECATED_BOTOCORE_SESSION),
+ aws_access_key_id=PropertyUtil.get_first_property_value(
+ properties, GLUE_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID, DEPRECATED_ACCESS_KEY_ID
+ ),
+ aws_secret_access_key=PropertyUtil.get_first_property_value(
+ properties, GLUE_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, DEPRECATED_SECRET_ACCESS_KEY
+ ),
+ aws_session_token=PropertyUtil.get_first_property_value(
+ properties, GLUE_SESSION_TOKEN, AWS_SESSION_TOKEN, DEPRECATED_SESSION_TOKEN
+ ),
)
self.glue: GlueClient = session.client("glue", endpoint_url=properties.get(GLUE_CATALOG_ENDPOINT))
diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py
index 36c3e625..d2008747 100644
--- a/pyiceberg/io/__init__.py
+++ b/pyiceberg/io/__init__.py
@@ -46,6 +46,10 @@ from pyiceberg.typedef import EMPTY_DICT, Properties
logger = logging.getLogger(__name__)
+AWS_REGION = "client.region"
+AWS_ACCESS_KEY_ID = "client.access-key-id"
+AWS_SECRET_ACCESS_KEY = "client.secret-access-key"
+AWS_SESSION_TOKEN = "client.session-token"
S3_ENDPOINT = "s3.endpoint"
S3_ACCESS_KEY_ID = "s3.access-key-id"
S3_SECRET_ACCESS_KEY = "s3.secret-access-key"
diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py
index bb76f043..c77a3024 100644
--- a/pyiceberg/io/fsspec.py
+++ b/pyiceberg/io/fsspec.py
@@ -46,6 +46,10 @@ from pyiceberg.io import (
ADLFS_CONNECTION_STRING,
ADLFS_SAS_TOKEN,
ADLFS_TENANT_ID,
+ AWS_ACCESS_KEY_ID,
+ AWS_REGION,
+ AWS_SECRET_ACCESS_KEY,
+ AWS_SESSION_TOKEN,
GCS_ACCESS,
GCS_CACHE_TIMEOUT,
GCS_CONSISTENCY,
@@ -114,12 +118,14 @@ def _file(_: Properties) -> LocalFileSystem:
def _s3(properties: Properties) -> AbstractFileSystem:
from s3fs import S3FileSystem
+ from pyiceberg.table import PropertyUtil
+
client_kwargs = {
"endpoint_url": properties.get(S3_ENDPOINT),
- "aws_access_key_id": properties.get(S3_ACCESS_KEY_ID),
- "aws_secret_access_key": properties.get(S3_SECRET_ACCESS_KEY),
- "aws_session_token": properties.get(S3_SESSION_TOKEN),
- "region_name": properties.get(S3_REGION),
+ "aws_access_key_id": PropertyUtil.get_first_property_value(properties, S3_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID),
+ "aws_secret_access_key": PropertyUtil.get_first_property_value(properties, S3_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY),
+ "aws_session_token": PropertyUtil.get_first_property_value(properties, S3_SESSION_TOKEN, AWS_SESSION_TOKEN),
+ "region_name": PropertyUtil.get_first_property_value(properties, S3_REGION, AWS_REGION),
}
config_kwargs = {}
register_events: Dict[str, Callable[[Properties], None]] = {}
diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py
index cd6736fb..2451bf7d 100644
--- a/pyiceberg/io/pyarrow.py
+++ b/pyiceberg/io/pyarrow.py
@@ -87,6 +87,10 @@ from pyiceberg.expressions.visitors import (
)
from pyiceberg.expressions.visitors import visit as boolean_expression_visit
from pyiceberg.io import (
+ AWS_ACCESS_KEY_ID,
+ AWS_REGION,
+ AWS_SECRET_ACCESS_KEY,
+ AWS_SESSION_TOKEN,
GCS_DEFAULT_LOCATION,
GCS_ENDPOINT,
GCS_TOKEN,
@@ -345,12 +349,14 @@ class PyArrowFileIO(FileIO):
if scheme in {"s3", "s3a", "s3n"}:
from pyarrow.fs import S3FileSystem
+ from pyiceberg.table import PropertyUtil
+
client_kwargs: Dict[str, Any] = {
"endpoint_override": self.properties.get(S3_ENDPOINT),
- "access_key": self.properties.get(S3_ACCESS_KEY_ID),
- "secret_key": self.properties.get(S3_SECRET_ACCESS_KEY),
- "session_token": self.properties.get(S3_SESSION_TOKEN),
- "region": self.properties.get(S3_REGION),
+ "access_key": PropertyUtil.get_first_property_value(self.properties, S3_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID),
+ "secret_key": PropertyUtil.get_first_property_value(self.properties, S3_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY),
+ "session_token": PropertyUtil.get_first_property_value(self.properties, S3_SESSION_TOKEN, AWS_SESSION_TOKEN),
+ "region": PropertyUtil.get_first_property_value(self.properties, S3_REGION, AWS_REGION),
}
if proxy_uri := self.properties.get(S3_PROXY_URI):
diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py
index 0b211e67..0cbe4630 100644
--- a/pyiceberg/table/__init__.py
+++ b/pyiceberg/table/__init__.py
@@ -253,6 +253,13 @@ class PropertyUtil:
return value.lower() == "true"
return default
+ @staticmethod
+ def get_first_property_value(properties: Properties, *property_names: str) -> Optional[Any]:
+ for property_name in property_names:
+ if property_value := properties.get(property_name):
+ return property_value
+ return None
+
class Transaction:
_table: Table
diff --git a/tests/catalog/test_dynamodb.py b/tests/catalog/test_dynamodb.py
index 7ad1301d..0289c5d0 100644
--- a/tests/catalog/test_dynamodb.py
+++ b/tests/catalog/test_dynamodb.py
@@ -42,7 +42,13 @@ from pyiceberg.exceptions import (
TableAlreadyExistsError,
)
from pyiceberg.schema import Schema
-from tests.conftest import BUCKET_NAME, TABLE_METADATA_LOCATION_REGEX
+from pyiceberg.typedef import Properties
+from tests.conftest import (
+ BUCKET_NAME,
+ DEPRECATED_AWS_SESSION_PROPERTIES,
+ TABLE_METADATA_LOCATION_REGEX,
+ UNIFIED_AWS_SESSION_PROPERTIES,
+)
@mock_aws
@@ -579,6 +585,60 @@ def test_passing_provided_profile() -> None:
assert test_catalog.dynamodb is mock_session().client()
+@mock_aws
+def test_passing_glue_session_properties() -> None:
+ session_properties: Properties = {
+ "dynamodb.access-key-id": "dynamodb.access-key-id",
+ "dynamodb.secret-access-key": "dynamodb.secret-access-key",
+ "dynamodb.profile-name": "dynamodb.profile-name",
+ "dynamodb.region": "dynamodb.region",
+ "dynamodb.session-token": "dynamodb.session-token",
+ **UNIFIED_AWS_SESSION_PROPERTIES,
+ **DEPRECATED_AWS_SESSION_PROPERTIES,
+ }
+
+ with mock.patch("boto3.Session") as mock_session:
+ mock_client = mock.Mock()
+ mock_session.return_value.client.return_value = mock_client
+ mock_client.describe_table.return_value = {"Table": {"TableStatus": "ACTIVE"}}
+ test_catalog = DynamoDbCatalog("dynamodb", **session_properties)
+
+ mock_session.assert_called_with(
+ aws_access_key_id="dynamodb.access-key-id",
+ aws_secret_access_key="dynamodb.secret-access-key",
+ aws_session_token="dynamodb.session-token",
+ region_name="dynamodb.region",
+ profile_name="dynamodb.profile-name",
+ botocore_session=None,
+ )
+ assert test_catalog.dynamodb is mock_session().client()
+
+
+@mock_aws
+def test_passing_unified_session_properties_to_dynamodb() -> None:
+ session_properties: Properties = {
+ "dynamodb.profile-name": "dynamodb.profile-name",
+ **UNIFIED_AWS_SESSION_PROPERTIES,
+ **DEPRECATED_AWS_SESSION_PROPERTIES,
+ }
+
+ with mock.patch("boto3.Session") as mock_session:
+ mock_client = mock.Mock()
+ mock_session.return_value.client.return_value = mock_client
+ mock_client.describe_table.return_value = {"Table": {"TableStatus": "ACTIVE"}}
+ test_catalog = DynamoDbCatalog("dynamodb", **session_properties)
+
+ mock_session.assert_called_with(
+ aws_access_key_id="client.access-key-id",
+ aws_secret_access_key="client.secret-access-key",
+ aws_session_token="client.session-token",
+ region_name="client.region",
+ profile_name="dynamodb.profile-name",
+ botocore_session=None,
+ )
+ assert test_catalog.dynamodb is mock_session().client()
+
+
@mock_aws
def test_table_exists(
_bucket_initialize: None, moto_endpoint_url: str, table_schema_nested: Schema, database_name: str, table_name: str
diff --git a/tests/catalog/test_glue.py b/tests/catalog/test_glue.py
index c4afa50c..10f399b2 100644
--- a/tests/catalog/test_glue.py
+++ b/tests/catalog/test_glue.py
@@ -36,8 +36,14 @@ from pyiceberg.io.pyarrow import schema_to_pyarrow
from pyiceberg.partitioning import PartitionField, PartitionSpec
from pyiceberg.schema import Schema
from pyiceberg.transforms import IdentityTransform
+from pyiceberg.typedef import Properties
from pyiceberg.types import IntegerType
-from tests.conftest import BUCKET_NAME, TABLE_METADATA_LOCATION_REGEX
+from tests.conftest import (
+ BUCKET_NAME,
+ DEPRECATED_AWS_SESSION_PROPERTIES,
+ TABLE_METADATA_LOCATION_REGEX,
+ UNIFIED_AWS_SESSION_PROPERTIES,
+)
@mock_aws
@@ -638,6 +644,54 @@ def test_passing_profile_name() -> None:
assert test_catalog.glue is mock_session().client()
+@mock_aws
+def test_passing_glue_session_properties() -> None:
+ session_properties: Properties = {
+ "glue.access-key-id": "glue.access-key-id",
+ "glue.secret-access-key": "glue.secret-access-key",
+ "glue.profile-name": "glue.profile-name",
+ "glue.region": "glue.region",
+ "glue.session-token": "glue.session-token",
+ **UNIFIED_AWS_SESSION_PROPERTIES,
+ **DEPRECATED_AWS_SESSION_PROPERTIES,
+ }
+
+ with mock.patch("boto3.Session") as mock_session:
+ test_catalog = GlueCatalog("glue", **session_properties)
+
+ mock_session.assert_called_with(
+ aws_access_key_id="glue.access-key-id",
+ aws_secret_access_key="glue.secret-access-key",
+ aws_session_token="glue.session-token",
+ region_name="glue.region",
+ profile_name="glue.profile-name",
+ botocore_session=None,
+ )
+ assert test_catalog.glue is mock_session().client()
+
+
+@mock_aws
+def test_passing_unified_session_properties_to_glue() -> None:
+ session_properties: Properties = {
+ "glue.profile-name": "glue.profile-name",
+ **UNIFIED_AWS_SESSION_PROPERTIES,
+ **DEPRECATED_AWS_SESSION_PROPERTIES,
+ }
+
+ with mock.patch("boto3.Session") as mock_session:
+ test_catalog = GlueCatalog("glue", **session_properties)
+
+ mock_session.assert_called_with(
+ aws_access_key_id="client.access-key-id",
+ aws_secret_access_key="client.secret-access-key",
+ aws_session_token="client.session-token",
+ region_name="client.region",
+ profile_name="glue.profile-name",
+ botocore_session=None,
+ )
+ assert test_catalog.glue is mock_session().client()
+
+
@mock_aws
def test_commit_table_update_schema(
_glue: boto3.client,
diff --git a/tests/conftest.py b/tests/conftest.py
index 7f9a2bcf..f65f4ed5 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -2029,6 +2029,21 @@ TABLE_METADATA_LOCATION_REGEX = re.compile(
re.X,
)
+DEPRECATED_AWS_SESSION_PROPERTIES = {
+ "aws_access_key_id": "aws_access_key_id",
+ "aws_secret_access_key": "aws_secret_access_key",
+ "aws_session_token": "aws_session_token",
+ "region_name": "region_name",
+ "profile_name": "profile_name",
+}
+
+UNIFIED_AWS_SESSION_PROPERTIES = {
+ "client.access-key-id": "client.access-key-id",
+ "client.secret-access-key": "client.secret-access-key",
+ "client.region": "client.region",
+ "client.session-token": "client.session-token",
+}
+
@pytest.fixture(name="_bucket_initialize")
def fixture_s3_bucket(_s3) -> None: # type: ignore
diff --git a/tests/io/test_fsspec.py b/tests/io/test_fsspec.py
index 013fc5d8..3dd79e18 100644
--- a/tests/io/test_fsspec.py
+++ b/tests/io/test_fsspec.py
@@ -19,6 +19,7 @@ import os
import pickle
import tempfile
import uuid
+from unittest import mock
import pytest
from botocore.awsrequest import AWSRequest
@@ -29,6 +30,8 @@ from pyiceberg.exceptions import SignError
from pyiceberg.io import fsspec
from pyiceberg.io.fsspec import FsspecFileIO, s3v4_rest_signer
from pyiceberg.io.pyarrow import PyArrowFileIO
+from pyiceberg.typedef import Properties
+from tests.conftest import UNIFIED_AWS_SESSION_PROPERTIES
def test_fsspec_infer_local_fs_from_path(fsspec_fileio: FsspecFileIO) -> None:
@@ -235,6 +238,58 @@ def test_fsspec_pickle_round_trip_s3(fsspec_fileio: FsspecFileIO) -> None:
_test_fsspec_pickle_round_trip(fsspec_fileio, "s3://warehouse/foo.txt")
+def test_fsspec_s3_session_properties() -> None:
+ session_properties: Properties = {
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ "s3.region": "us-east-1",
+ "s3.session-token": "s3.session-token",
+ **UNIFIED_AWS_SESSION_PROPERTIES,
+ }
+
+ with mock.patch("s3fs.S3FileSystem") as mock_s3fs:
+ s3_fileio = FsspecFileIO(properties=session_properties)
+ filename = str(uuid.uuid4())
+
+ s3_fileio.new_input(location=f"s3://warehouse/{filename}")
+
+ mock_s3fs.assert_called_with(
+ client_kwargs={
+ "endpoint_url": "http://localhost:9000",
+ "aws_access_key_id": "admin",
+ "aws_secret_access_key": "password",
+ "region_name": "us-east-1",
+ "aws_session_token": "s3.session-token",
+ },
+ config_kwargs={},
+ )
+
+
+def test_fsspec_unified_session_properties() -> None:
+ session_properties: Properties = {
+ "s3.endpoint": "http://localhost:9000",
+ **UNIFIED_AWS_SESSION_PROPERTIES,
+ }
+
+ with mock.patch("s3fs.S3FileSystem") as mock_s3fs:
+ s3_fileio = FsspecFileIO(properties=session_properties)
+ filename = str(uuid.uuid4())
+
+ s3_fileio.new_input(location=f"s3://warehouse/{filename}")
+
+ mock_s3fs.assert_called_with(
+ client_kwargs={
+ "endpoint_url": "http://localhost:9000",
+ "aws_access_key_id": "client.access-key-id",
+ "aws_secret_access_key": "client.secret-access-key",
+ "region_name": "client.region",
+ "aws_session_token": "client.session-token",
+ },
+ config_kwargs={},
+ )
+
+
@pytest.mark.adlfs
def test_fsspec_new_input_file_adlfs(adlfs_fsspec_fileio: FsspecFileIO) -> None:
"""Test creating a new input file from an fsspec file-io"""
diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py
index d61a50bb..82b35341 100644
--- a/tests/io/test_pyarrow.py
+++ b/tests/io/test_pyarrow.py
@@ -18,6 +18,7 @@
import os
import tempfile
+import uuid
from datetime import date
from typing import Any, List, Optional
from unittest.mock import MagicMock, patch
@@ -77,7 +78,7 @@ from pyiceberg.schema import Schema, make_compatible_name, visit
from pyiceberg.table import FileScanTask, TableProperties
from pyiceberg.table.metadata import TableMetadataV2
from pyiceberg.transforms import IdentityTransform
-from pyiceberg.typedef import UTF8, Record
+from pyiceberg.typedef import UTF8, Properties, Record
from pyiceberg.types import (
BinaryType,
BooleanType,
@@ -98,6 +99,7 @@ from pyiceberg.types import (
TimestamptzType,
TimeType,
)
+from tests.conftest import UNIFIED_AWS_SESSION_PROPERTIES
def test_pyarrow_infer_local_fs_from_path() -> None:
@@ -348,6 +350,52 @@ def test_deleting_hdfs_file_not_found() -> None:
assert "Cannot delete file, does not exist:" in str(exc_info.value)
+def test_pyarrow_s3_session_properties() -> None:
+ session_properties: Properties = {
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ "s3.region": "us-east-1",
+ "s3.session-token": "s3.session-token",
+ **UNIFIED_AWS_SESSION_PROPERTIES,
+ }
+
+ with patch("pyarrow.fs.S3FileSystem") as mock_s3fs:
+ s3_fileio = PyArrowFileIO(properties=session_properties)
+ filename = str(uuid.uuid4())
+
+ s3_fileio.new_input(location=f"s3://warehouse/{filename}")
+
+ mock_s3fs.assert_called_with(
+ endpoint_override="http://localhost:9000",
+ access_key="admin",
+ secret_key="password",
+ region="us-east-1",
+ session_token="s3.session-token",
+ )
+
+
+def test_pyarrow_unified_session_properties() -> None:
+ session_properties: Properties = {
+ "s3.endpoint": "http://localhost:9000",
+ **UNIFIED_AWS_SESSION_PROPERTIES,
+ }
+
+ with patch("pyarrow.fs.S3FileSystem") as mock_s3fs:
+ s3_fileio = PyArrowFileIO(properties=session_properties)
+ filename = str(uuid.uuid4())
+
+ s3_fileio.new_input(location=f"s3://warehouse/{filename}")
+
+ mock_s3fs.assert_called_with(
+ endpoint_override="http://localhost:9000",
+ access_key="client.access-key-id",
+ secret_key="client.secret-access-key",
+ region="client.region",
+ session_token="client.session-token",
+ )
+
+
def test_schema_to_pyarrow_schema_include_field_ids(table_schema_nested: Schema) -> None:
actual = schema_to_pyarrow(table_schema_nested)
expected = """foo: large_string