You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by gi...@apache.org on 2026/10/22 23:21:49 UTC
(iceberg-python) 01/01: fix: map write.parquet.page-row-limit to pyarrow's max_rows_per_page (#4127)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-4127-d37149c8b5d1b737da0eb9fdcb585bc3ef3bac69
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
commit e3fbd9440ad0bb25c45848797ff9ec5ad7758624
Author: Yuang Gao <91...@users.noreply.github.com>
AuthorDate: Thu Oct 22 18:55:40 2026 +0000
fix: map write.parquet.page-row-limit to pyarrow's max_rows_per_page (#4127)
---
pyiceberg/io/pyarrow.py | 13 +++++++++----
tests/integration/test_writes/test_writes.py | 1 +
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py
index 7fd0c85b7..874646b71 100644
--- a/pyiceberg/io/pyarrow.py
+++ b/pyiceberg/io/pyarrow.py
@@ -2986,7 +2986,7 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]:
if compression_codec == ICEBERG_UNCOMPRESSED_CODEC:
compression_codec = PYARROW_UNCOMPRESSED_CODEC
- return {
+ kwargs: dict[str, Any] = {
"compression": compression_codec,
"compression_level": compression_level,
"data_page_size": property_as_int(
@@ -2999,12 +2999,17 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]:
property_name=TableProperties.PARQUET_DICT_SIZE_BYTES,
default=TableProperties.PARQUET_DICT_SIZE_BYTES_DEFAULT,
),
- "write_batch_size": property_as_int(
+ }
+ # pyarrow added max_rows_per_page, the per-page row limit, in 23.0.0.
+ if tuple(map(int, pa.__version__.split(".")[:2])) >= (23, 0):
+ kwargs["max_rows_per_page"] = property_as_int(
properties=table_properties,
property_name=TableProperties.PARQUET_PAGE_ROW_LIMIT,
default=TableProperties.PARQUET_PAGE_ROW_LIMIT_DEFAULT,
- ),
- }
+ )
+ elif TableProperties.PARQUET_PAGE_ROW_LIMIT in table_properties:
+ warnings.warn(f"{TableProperties.PARQUET_PAGE_ROW_LIMIT} requires pyarrow >= 23.0.0", stacklevel=2)
+ return kwargs
def _dataframe_to_data_files(
diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py
index 30fdd76ab..fe720440f 100644
--- a/tests/integration/test_writes/test_writes.py
+++ b/tests/integration/test_writes/test_writes.py
@@ -669,6 +669,7 @@ def test_write_parquet_compression_properties(
[
({"write.parquet.page-size-bytes": "42"}, {"data_page_size": 42}),
({"write.parquet.dict-size-bytes": "42"}, {"dictionary_pagesize_limit": 42}),
+ ({"write.parquet.page-row-limit": "42"}, {"max_rows_per_page": 42, "write_batch_size": None}),
],
)
def test_write_parquet_other_properties(