You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2022/09/19 06:38:24 UTC
[iceberg] branch master updated: Python: Add docker-compose for s3 tests (#5750)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 371ce7e5ef Python: Add docker-compose for s3 tests (#5750)
371ce7e5ef is described below
commit 371ce7e5efd598531e3fb0bafa08a895799a3263
Author: Fokko Driesprong <fo...@apache.org>
AuthorDate: Mon Sep 19 08:38:19 2022 +0200
Python: Add docker-compose for s3 tests (#5750)
This PR adds a docker-compose.yml with the right configuration to
run the s3 tests
---
.github/workflows/python-ci.yml | 2 +-
python/CONTRIBUTING.md | 8 +++++-
python/Makefile | 7 ++++++
python/dev/docker-compose.yml | 46 +++++++++++++++++++++++++++++++++++
python/{Makefile => dev/run-minio.sh} | 26 +++++++++++---------
5 files changed, 76 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
index e1530a7bbd..a6113ad344 100644
--- a/.github/workflows/python-ci.yml
+++ b/.github/workflows/python-ci.yml
@@ -54,4 +54,4 @@ jobs:
run: make lint
- name: Tests
working-directory: ./python
- run: make test
+ run: make test-s3
diff --git a/python/CONTRIBUTING.md b/python/CONTRIBUTING.md
index 3a258189f7..91ae8215b4 100644
--- a/python/CONTRIBUTING.md
+++ b/python/CONTRIBUTING.md
@@ -37,7 +37,7 @@ If you want to install the library on the host, you can simply run `pip3 install
To set up IDEA with Poetry ([also on Loom](https://www.loom.com/share/6d36464d45f244729d91003e7f671fd2)):
- Open up the Python project in IntelliJ
-- Make sure that you're on a lastest master (that includes Poetry)
+- Make sure that you're on latest master (that includes Poetry)
- Go to File -> Project Structure (⌘;)
- Go to Platform Settings -> SDKs
- Click the + sign -> Add Python SDK
@@ -72,6 +72,12 @@ For Python, we use pytest in combination with coverage to maintain 90% code cove
make test
```
+By default we ignore the s3 tests that require minio to be running. To run this suite, we can run:
+
+```bash
+make test-s3
+```
+
To pass additional arguments to pytest, you can use `PYTEST_ARGS`.
*Run pytest in verbose mode*
diff --git a/python/Makefile b/python/Makefile
index 0a67cc7817..14a03c3bdc 100644
--- a/python/Makefile
+++ b/python/Makefile
@@ -27,3 +27,10 @@ test:
poetry run coverage report -m --fail-under=90
poetry run coverage html
poetry run coverage xml
+
+test-s3:
+ sh ./dev/run-minio.sh
+ poetry run coverage run --source=pyiceberg/ -m pytest tests/ ${PYTEST_ARGS}
+ poetry run coverage report -m --fail-under=90
+ poetry run coverage html
+ poetry run coverage xml
diff --git a/python/dev/docker-compose.yml b/python/dev/docker-compose.yml
new file mode 100644
index 0000000000..f7fbd6deaa
--- /dev/null
+++ b/python/dev/docker-compose.yml
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+version: "3"
+
+services:
+ minio:
+ image: minio/minio
+ container_name: pyiceberg-minio
+ environment:
+ - MINIO_ROOT_USER=admin
+ - MINIO_ROOT_PASSWORD=password
+ ports:
+ - 9001:9001
+ - 9000:9000
+ command: [ "server", "/data", "--console-address", ":9001" ]
+ mc:
+ depends_on:
+ - minio
+ image: minio/mc
+ container_name: pyiceberg-mc
+ environment:
+ - AWS_ACCESS_KEY_ID=admin
+ - AWS_SECRET_ACCESS_KEY=password
+ - AWS_REGION=us-east-1
+ entrypoint: >
+ /bin/sh -c "
+ until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
+ /usr/bin/mc rm -r --force minio/warehouse;
+ /usr/bin/mc mb minio/warehouse;
+ /usr/bin/mc policy set public minio/warehouse;
+ exit 0;
+ "
diff --git a/python/Makefile b/python/dev/run-minio.sh
old mode 100644
new mode 100755
similarity index 65%
copy from python/Makefile
copy to python/dev/run-minio.sh
index 0a67cc7817..0e40be4e9d
--- a/python/Makefile
+++ b/python/dev/run-minio.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,16 +16,18 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
-install:
- pip install poetry
- poetry install -E pyarrow -E hive -E s3fs
-
-lint:
- poetry run pre-commit run --all-files
+set -ex
-test:
- poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m "not s3" ${PYTEST_ARGS}
- poetry run coverage report -m --fail-under=90
- poetry run coverage html
- poetry run coverage xml
+if [[ $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]]; then
+ echo "Minio backend running"
+else
+ docker-compose -f dev/docker-compose.yml kill
+ docker-compose -f dev/docker-compose.yml up -d
+ while [[ -z $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]]
+ do
+ echo "Waiting for Minio"
+ sleep 1
+ done
+fi