You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by gi...@apache.org on 2026/10/01 20:24:52 UTC
(iceberg-python) 01/01: fix(io): pass S3 CA cert to PyArrow (#4019)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-4019-2ab74f855cd36260d9b925024f723416c73d7ca2
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
commit 329df2fac16eb963de5b5e74c0a1819dd91e393c
Author: Stefan Wang <1f...@gmail.com>
AuthorDate: Thu Oct 1 20:15:32 2026 +0000
fix(io): pass S3 CA cert to PyArrow (#4019)
* fix(io): pass S3 CA cert to PyArrow
Signed-off-by: 1fanwang <1f...@gmail.com>
* fix(io): use PyArrow version alias
Signed-off-by: 1fanwang <1f...@gmail.com>
---------
Signed-off-by: 1fanwang <1f...@gmail.com>
---
pyiceberg/io/__init__.py | 1 +
pyiceberg/io/pyarrow.py | 15 +++++++++++++++
tests/io/test_pyarrow.py | 20 +++++++++++++++++++-
3 files changed, 35 insertions(+), 1 deletion(-)
diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py
index c44e105e6..9cae219f7 100644
--- a/pyiceberg/io/__init__.py
+++ b/pyiceberg/io/__init__.py
@@ -69,6 +69,7 @@ S3_SECRET_ACCESS_KEY = "s3.secret-access-key"
S3_SESSION_TOKEN = "s3.session-token"
S3_REGION = "s3.region"
S3_RESOLVE_REGION = "s3.resolve-region"
+S3_SSL_CA_CERT = "s3.ssl.ca-cert"
S3_PROXY_URI = "s3.proxy-uri"
S3_CONNECT_TIMEOUT = "s3.connect-timeout"
S3_REQUEST_TIMEOUT = "s3.request-timeout"
diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py
index 1179f763a..2dcb8a579 100644
--- a/pyiceberg/io/pyarrow.py
+++ b/pyiceberg/io/pyarrow.py
@@ -117,6 +117,7 @@ from pyiceberg.io import (
S3_ROLE_SESSION_NAME,
S3_SECRET_ACCESS_KEY,
S3_SESSION_TOKEN,
+ S3_SSL_CA_CERT,
FileIO,
InputFile,
InputStream,
@@ -454,6 +455,16 @@ class PyArrowFileIO(FileIO):
else:
raise ValueError(f"Unrecognized filesystem type in URI: {scheme}")
+ def _set_tls_ca_file_path(self, client_kwargs: dict[str, Any]) -> None:
+ if tls_ca_file_path := self.properties.get(S3_SSL_CA_CERT):
+ min_pyarrow_version_supporting_tls_ca_file_path = (21, 0)
+ if tuple(map(int, pa.__version__.split(".")[:2])) < min_pyarrow_version_supporting_tls_ca_file_path:
+ raise ImportError(
+ "pyarrow version >= 21.0.0 required for S3FileSystem tls_ca_file_path support, "
+ f"but found version {pa.__version__}."
+ )
+ client_kwargs["tls_ca_file_path"] = tls_ca_file_path
+
def _initialize_oss_fs(self) -> FileSystem:
from pyarrow.fs import S3FileSystem
@@ -484,6 +495,8 @@ class PyArrowFileIO(FileIO):
if s3_anonymous := self.properties.get(S3_ANONYMOUS):
client_kwargs["anonymous"] = strtobool(s3_anonymous)
+ self._set_tls_ca_file_path(client_kwargs)
+
return S3FileSystem(**client_kwargs)
def _initialize_s3_fs(self, netloc: str | None) -> FileSystem:
@@ -538,6 +551,8 @@ class PyArrowFileIO(FileIO):
if s3_anonymous := self.properties.get(S3_ANONYMOUS):
client_kwargs["anonymous"] = strtobool(s3_anonymous)
+ self._set_tls_ca_file_path(client_kwargs)
+
return S3FileSystem(**client_kwargs)
def _initialize_azure_fs(self) -> FileSystem:
diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py
index 386ea076d..4d5d4431c 100644
--- a/tests/io/test_pyarrow.py
+++ b/tests/io/test_pyarrow.py
@@ -63,7 +63,7 @@ from pyiceberg.expressions import (
Or,
)
from pyiceberg.expressions.literals import literal
-from pyiceberg.io import S3_RETRY_STRATEGY_IMPL, InputStream, OutputStream, load_file_io
+from pyiceberg.io import S3_RETRY_STRATEGY_IMPL, S3_SSL_CA_CERT, InputStream, OutputStream, load_file_io
from pyiceberg.io.pyarrow import (
ICEBERG_SCHEMA,
PYARROW_PARQUET_FIELD_ID_KEY,
@@ -382,6 +382,7 @@ def test_pyarrow_s3_session_properties() -> None:
"s3.secret-access-key": "password",
"s3.region": "us-east-1",
"s3.session-token": "s3.session-token",
+ S3_SSL_CA_CERT: "/path/to/ca.pem",
**UNIFIED_AWS_SESSION_PROPERTIES,
}
@@ -399,9 +400,26 @@ def test_pyarrow_s3_session_properties() -> None:
secret_key="password",
region="us-east-1",
session_token="s3.session-token",
+ tls_ca_file_path="/path/to/ca.pem",
)
+def test_pyarrow_s3_ssl_ca_cert_requires_supported_pyarrow_version() -> None:
+ session_properties: Properties = {
+ S3_SSL_CA_CERT: "/path/to/ca.pem",
+ }
+
+ with (
+ patch("pyiceberg.io.pyarrow.pa.__version__", "20.0.0"),
+ patch("pyarrow.fs.S3FileSystem"),
+ patch("pyarrow.fs.resolve_s3_region") as mock_s3_region_resolver,
+ ):
+ mock_s3_region_resolver.side_effect = OSError("S3 bucket is not found")
+
+ with pytest.raises(ImportError, match="pyarrow version >= 21.0.0 required"):
+ PyArrowFileIO(properties=session_properties).new_input(location=f"s3://warehouse/{uuid.uuid4()}")
+
+
def test_pyarrow_s3_session_properties_with_anonymous() -> None:
session_properties: Properties = {
"s3.anonymous": "true",