You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by gi...@apache.org on 2026/09/11 16:46:12 UTC
(iceberg-python) branch main updated: Infra: Replace MinIO with RustFS (#3928)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new d5474fdf0 Infra: Replace MinIO with RustFS (#3928)
d5474fdf0 is described below
commit d5474fdf0eb393d9979498fd88421e78a0554746
Author: Alex Stephen <13...@users.noreply.github.com>
AuthorDate: Fri Sep 11 16:32:39 2026 +0000
Infra: Replace MinIO with RustFS (#3928)
* Infra: Replace MinIO with RustFS in the dev and integration stacks
MinIO Community Edition stopped publishing images in October 2025 and the
repository was archived in April 2026, so `minio/minio:latest` resolves to a
frozen build that will not receive further security fixes (#2664).
I'm choosing RustFS since that's what Polaris uses.
* Infra: Rename the object store service to `object-store`
Feedback from @kevinjqliu
---
Makefile | 2 +-
dev/docker-compose-integration.yml | 54 ++++++++++++++++++++++----------------
dev/docker-compose.yml | 46 ++++++++++++++++++--------------
dev/hive/core-site.xml | 2 +-
dev/{run-minio.sh => run-s3.sh} | 11 +++-----
dev/spark/spark-defaults.conf | 6 ++---
mkdocs/docs/contributing.md | 4 +--
7 files changed, 68 insertions(+), 57 deletions(-)
diff --git a/Makefile b/Makefile
index 9a73b240a..7bbc05e9d 100644
--- a/Makefile
+++ b/Makefile
@@ -129,7 +129,7 @@ test-integration-rebuild: ## Rebuild integration Docker services from scratch
docker compose -f dev/docker-compose-integration.yml build --no-cache
test-s3: ## Run tests marked with @pytest.mark.s3
- sh ./dev/run-minio.sh
+ sh ./dev/run-s3.sh
$(TEST_RUNNER) pytest tests/ -m s3 $(PYTEST_ARGS)
test-adls: ## Run tests marked with @pytest.mark.adls
diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml
index 32cdccbb1..3eab22d9f 100644
--- a/dev/docker-compose-integration.yml
+++ b/dev/docker-compose-integration.yml
@@ -28,7 +28,7 @@ services:
depends_on:
- rest
- hive
- - minio
+ - object-store
ports:
- 15002:15002 # Spark Connect
- 4040:4040 # Spark UI
@@ -39,7 +39,7 @@ services:
links:
- rest:rest
- hive:hive
- - minio:minio
+ - object-store:object-store
healthcheck:
test: ["CMD", "sh", "-c", "netstat -an | grep 15002 | grep LISTEN"]
interval: 30s
@@ -59,41 +59,51 @@ services:
- AWS_REGION=us-east-1
- CATALOG_WAREHOUSE=s3://warehouse/
- CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
- - CATALOG_S3_ENDPOINT=http://minio:9000
+ - CATALOG_S3_ENDPOINT=http://object-store:9000
- CATALOG_JDBC_STRICT__MODE=true
- minio:
- image: minio/minio
- container_name: pyiceberg-minio
+ object-store:
+ image: rustfs/rustfs:1.0.0-rc.5
+ container_name: pyiceberg-object-store
networks:
iceberg_net:
aliases:
- - warehouse.minio
+ - warehouse.object-store
ports:
- 9001:9001
- 9000:9000
environment:
- - MINIO_ROOT_USER=admin
- - MINIO_ROOT_PASSWORD=password
- - MINIO_DOMAIN=minio
- command: ["server", "/data", "--console-address", ":9001"]
- mc:
- image: minio/mc
- container_name: pyiceberg-mc
+ - RUSTFS_ACCESS_KEY=admin
+ - RUSTFS_SECRET_KEY=password
+ - RUSTFS_VOLUMES=/data
+ - RUSTFS_ADDRESS=:9000
+ - RUSTFS_CONSOLE_ENABLE=true
+ - RUSTFS_CONSOLE_ADDRESS=:9001
+ # Resolves virtual-hosted-style `warehouse.object-store` to the warehouse
+ # bucket. The port is required until rustfs/rustfs#7051 ships.
+ - RUSTFS_SERVER_DOMAINS=object-store:9000
+ healthcheck:
+ test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health"]
+ interval: 5s
+ timeout: 5s
+ retries: 10
+ start_period: 5s
+ aws-cli:
+ image: amazon/aws-cli:2.36.40
+ container_name: pyiceberg-aws-cli
networks:
iceberg_net:
depends_on:
- - minio
+ object-store:
+ condition: service_healthy
environment:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=password
- AWS_REGION=us-east-1
- entrypoint: >
- /bin/sh -c "
- until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
- /usr/bin/mc mb minio/warehouse;
- /usr/bin/mc policy set public minio/warehouse;
- tail -f /dev/null
- "
+ - AWS_ENDPOINT_URL=http://object-store:9000
+ entrypoint: /bin/sh
+ command:
+ - -c
+ - "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse"
hive:
image: pyiceberg-hive:latest
build:
diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
index 609ac8d51..8ef82731e 100644
--- a/dev/docker-compose.yml
+++ b/dev/docker-compose.yml
@@ -16,31 +16,37 @@
# under the License.
services:
- minio:
- image: minio/minio
- container_name: pyiceberg-minio
- environment:
- - MINIO_ROOT_USER=admin
- - MINIO_ROOT_PASSWORD=password
- - MINIO_DOMAIN=minio
+ object-store:
+ image: rustfs/rustfs:1.0.0-rc.5
+ container_name: pyiceberg-object-store
ports:
- 9001:9001
- 9000:9000
- command: ["server", "/data", "--console-address", ":9001"]
- mc:
+ environment:
+ - RUSTFS_ACCESS_KEY=admin
+ - RUSTFS_SECRET_KEY=password
+ - RUSTFS_VOLUMES=/data
+ - RUSTFS_ADDRESS=:9000
+ - RUSTFS_CONSOLE_ENABLE=true
+ - RUSTFS_CONSOLE_ADDRESS=:9001
+ healthcheck:
+ test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health"]
+ interval: 5s
+ timeout: 5s
+ retries: 10
+ start_period: 5s
+ aws-cli:
+ image: amazon/aws-cli:2.36.40
+ container_name: pyiceberg-aws-cli
depends_on:
- - minio
- image: minio/mc
- container_name: pyiceberg-mc
+ object-store:
+ condition: service_healthy
environment:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=password
- AWS_REGION=us-east-1
- entrypoint: >
- /bin/sh -c "
- until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
- /usr/bin/mc rm -r --force minio/warehouse;
- /usr/bin/mc mb minio/warehouse;
- /usr/bin/mc policy set public minio/warehouse;
- exit 0;
- "
+ - AWS_ENDPOINT_URL=http://object-store:9000
+ entrypoint: /bin/sh
+ command:
+ - -c
+ - "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse"
diff --git a/dev/hive/core-site.xml b/dev/hive/core-site.xml
index f5a9473b5..d17656f01 100644
--- a/dev/hive/core-site.xml
+++ b/dev/hive/core-site.xml
@@ -32,7 +32,7 @@
</property>
<property>
<name>fs.s3a.endpoint</name>
- <value>http://minio:9000</value>
+ <value>http://object-store:9000</value>
</property>
<property>
<name>fs.s3a.access.key</name>
diff --git a/dev/run-minio.sh b/dev/run-s3.sh
similarity index 72%
rename from dev/run-minio.sh
rename to dev/run-s3.sh
index 3e4a55c19..359abd5b1 100755
--- a/dev/run-minio.sh
+++ b/dev/run-s3.sh
@@ -20,14 +20,9 @@
set -ex
-if [ $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]; then
- echo "Minio backend running"
+if [ $(docker ps -q --filter "name=pyiceberg-object-store" --filter "health=healthy" ) ]; then
+ echo "S3 backend running"
else
docker compose -f dev/docker-compose.yml kill
- docker compose -f dev/docker-compose.yml up -d
- while [ -z $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]
- do
- echo "Waiting for Minio"
- sleep 1
- done
+ docker compose -f dev/docker-compose.yml up -d --wait
fi
diff --git a/dev/spark/spark-defaults.conf b/dev/spark/spark-defaults.conf
index 847e73370..774155d20 100644
--- a/dev/spark/spark-defaults.conf
+++ b/dev/spark/spark-defaults.conf
@@ -23,7 +23,7 @@ spark.sql.catalog.rest.type rest
spark.sql.catalog.rest.uri http://rest:8181
spark.sql.catalog.rest.io-impl org.apache.iceberg.aws.s3.S3FileIO
spark.sql.catalog.rest.warehouse s3://warehouse/rest/
-spark.sql.catalog.rest.s3.endpoint http://minio:9000
+spark.sql.catalog.rest.s3.endpoint http://object-store:9000
spark.sql.catalog.rest.cache-enabled false
# Configure Iceberg Hive catalog
@@ -32,13 +32,13 @@ spark.sql.catalog.hive.type hive
spark.sql.catalog.hive.uri thrift://hive:9083
spark.sql.catalog.hive.io-impl org.apache.iceberg.aws.s3.S3FileIO
spark.sql.catalog.hive.warehouse s3://warehouse/hive/
-spark.sql.catalog.hive.s3.endpoint http://minio:9000
+spark.sql.catalog.hive.s3.endpoint http://object-store:9000
# Configure Spark's default session catalog (spark_catalog) to use Iceberg backed by the Hive Metastore
spark.sql.catalog.spark_catalog org.apache.iceberg.spark.SparkSessionCatalog
spark.sql.catalog.spark_catalog.type hive
spark.sql.catalog.spark_catalog.uri thrift://hive:9083
-spark.hadoop.fs.s3a.endpoint http://minio:9000
+spark.hadoop.fs.s3a.endpoint http://object-store:9000
spark.sql.catalogImplementation hive
spark.sql.warehouse.dir s3a://warehouse/hive/
diff --git a/mkdocs/docs/contributing.md b/mkdocs/docs/contributing.md
index 0155907a6..025109346 100644
--- a/mkdocs/docs/contributing.md
+++ b/mkdocs/docs/contributing.md
@@ -155,7 +155,7 @@ For Python, `pytest` is used a testing framework in combination with `coverage`
make test
```
-By default, S3 and ADLS tests are ignored because that require minio and azurite to be running.
+By default, S3 and ADLS tests are ignored because that require RustFS and azurite to be running.
To run the S3 suite:
```bash
@@ -257,7 +257,7 @@ This command spins up the full integration test infrastructure via Docker Compos
- **Spark** (with Spark Connect)
- **Iceberg REST Catalog** (using the [`apache/iceberg-rest-fixture`](https://hub.docker.com/r/apache/iceberg-rest-fixture) image)
- **Hive Metastore**
-- **S3-compatible object storage** (Minio)
+- **S3-compatible object storage** (RustFS)
**Spark Example Notebook** (`notebooks/spark_integration_example.ipynb`) is based on the [Spark Getting Started](https://iceberg.apache.org/docs/nightly/spark-getting-started/) guide. This notebook demonstrates how to work with PyIceberg alongside Spark, leveraging the Docker-based testing setup for a complete local development environment.