You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/10/30 15:44:39 UTC
(incubator-opendal) branch main updated: feat(test): Enable new test workflow planner for python binding (#3397)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new c0e9640ed feat(test): Enable new test workflow planner for python binding (#3397)
c0e9640ed is described below
commit c0e9640ed4a6083dc03c815bfb2e882097b0c81b
Author: Nadeshiko Manju <me...@manjusaka.me>
AuthorDate: Mon Oct 30 23:44:35 2023 +0800
feat(test): Enable new test workflow planner for python binding (#3397)
* feat(test): Enable new test workflow planner for python binding
Signed-off-by: Manjusaka <me...@manjusaka.me>
* Update code
Signed-off-by: Manjusaka <me...@manjusaka.me>
* Update code
Signed-off-by: Manjusaka <me...@manjusaka.me>
* Update code
Signed-off-by: Manjusaka <me...@manjusaka.me>
* Update code
Signed-off-by: Manjusaka <me...@manjusaka.me>
* update code
Signed-off-by: Manjusaka <me...@manjusaka.me>
* Update code
Signed-off-by: Manjusaka <me...@manjusaka.me>
* update code
Signed-off-by: Manjusaka <me...@manjusaka.me>
---------
Signed-off-by: Manjusaka <me...@manjusaka.me>
---
.../actions/behavior_test_binding_java/action.yaml | 2 +-
.../action.yaml | 16 +-
.github/scripts/behavior_test/plan.py | 51 ++++-
.github/services/mysql/mysql/action.yml | 8 +-
.github/workflows/behavior_test.yml | 12 ++
.github/workflows/behavior_test_binding_python.yml | 71 +++++++
.github/workflows/bindings_python.yml | 19 --
.github/workflows/service_test_memory.yml | 61 ------
.github/workflows/service_test_s3.yml | 42 +---
bindings/python/Cargo.toml | 100 +++++++++
bindings/python/tests/conftest.py | 60 ++++++
.../tests/{conftest.py => test_capability.py} | 14 +-
bindings/python/tests/test_read.py | 97 +++++++++
bindings/python/tests/test_services.py | 224 ---------------------
bindings/python/tests/test_write.py | 115 +++++++++++
15 files changed, 530 insertions(+), 362 deletions(-)
diff --git a/.github/actions/behavior_test_binding_java/action.yaml b/.github/actions/behavior_test_binding_java/action.yaml
index cb3724a81..d9252b1c3 100644
--- a/.github/actions/behavior_test_binding_java/action.yaml
+++ b/.github/actions/behavior_test_binding_java/action.yaml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-name: Test Core
+name: Test Binding Java
description: 'Test Core with given setup and service'
inputs:
setup:
diff --git a/.github/actions/behavior_test_binding_java/action.yaml b/.github/actions/behavior_test_binding_python/action.yaml
similarity index 79%
copy from .github/actions/behavior_test_binding_java/action.yaml
copy to .github/actions/behavior_test_binding_python/action.yaml
index cb3724a81..72437b08f 100644
--- a/.github/actions/behavior_test_binding_java/action.yaml
+++ b/.github/actions/behavior_test_binding_python/action.yaml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-name: Test Core
+name: Test Binding Python
description: 'Test Core with given setup and service'
inputs:
setup:
@@ -31,19 +31,21 @@ runs:
- name: Setup
shell: bash
run: |
- mkdir -p ./dynamic_test_binding_java &&
- cat <<EOF >./dynamic_test_binding_java/action.yml
+ mkdir -p ./dynamic_test_binding_python &&
+ cat <<EOF >./dynamic_test_binding_python/action.yml
runs:
using: composite
steps:
- name: Setup Test
uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }}
- - name: Run Test Binding Java
+ - name: Run Test Binding Python
shell: bash
- working-directory: bindings/java
- run: ./mvnw test -Dtest="behavior.*Test" -Dcargo-build.features=${{ inputs.feature }}
+ working-directory: bindings/python
+ run: |
+ source venv/bin/activate
+ pytest -v tests
env:
OPENDAL_TEST: ${{ inputs.service }}
EOF
- name: Run
- uses: ./dynamic_test_binding_java
+ uses: ./dynamic_test_binding_python
diff --git a/.github/scripts/behavior_test/plan.py b/.github/scripts/behavior_test/plan.py
index 69397f67f..3e0e5bdd6 100755
--- a/.github/scripts/behavior_test/plan.py
+++ b/.github/scripts/behavior_test/plan.py
@@ -22,6 +22,7 @@ import os
import re
from pathlib import Path
from dataclasses import dataclass, field
+from typing import Any
# The path for current script.
SCRIPT_PATH = Path(__file__).parent.absolute()
@@ -31,7 +32,7 @@ GITHUB_DIR = SCRIPT_PATH.parent.parent
PROJECT_DIR = GITHUB_DIR.parent
-def provided_cases():
+def provided_cases() -> list[dict[str, str]]:
root_dir = f"{GITHUB_DIR}/services"
cases = [
@@ -74,6 +75,8 @@ class Hint:
core: bool = field(default=False, init=False)
# Is binding java affected?
binding_java: bool = field(default=False, init=False)
+ # Is binding python affected?
+ binding_python: bool = field(default=False, init=False)
# Should we run all services test?
all_service: bool = field(default=False, init=False)
@@ -81,7 +84,7 @@ class Hint:
services: set = field(default_factory=set, init=False)
-def calculate_hint(changed_files):
+def calculate_hint(changed_files: list[str]) -> Hint:
hint = Hint()
# Remove all files that ends with `.md`
@@ -95,12 +98,14 @@ def calculate_hint(changed_files):
if p == ".github/workflows/behavior_test.yml":
hint.core = True
hint.binding_java = True
+ hint.binding_python = True
hint.all_service = True
if p == ".github/workflows/behavior_test_core.yml":
hint.core = True
hint.all_service = True
if p == ".github/workflows/behavior_test_binding_java.yml":
hint.binding_java = True
+ hint.binding_python = True
hint.all_service = True
# core affected
@@ -113,11 +118,13 @@ def calculate_hint(changed_files):
):
hint.core = True
hint.binding_java = True
+ hint.binding_python = True
hint.all_service = True
# binding java affected.
if p.startswith("bindings/java/"):
hint.binding_java = True
+ hint.binding_python = True
hint.all_service = True
# core service affected
@@ -125,6 +132,7 @@ def calculate_hint(changed_files):
if match:
hint.core = True
hint.binding_java = True
+ hint.binding_python = True
hint.services.add(match.group(1))
# core test affected
@@ -132,6 +140,7 @@ def calculate_hint(changed_files):
if match:
hint.core = True
hint.binding_java = True
+ hint.binding_python = True
hint.services.add(match.group(1))
return hint
@@ -154,7 +163,9 @@ def unique_cases(cases):
return list(ucases.values())
-def generate_core_cases(cases, hint):
+def generate_core_cases(
+ cases: list[dict[str, str]], hint: Hint
+) -> list[dict[str, str]]:
# Always run all tests if it is a push event.
if os.getenv("GITHUB_IS_PUSH") == "true":
return cases
@@ -172,7 +183,9 @@ def generate_core_cases(cases, hint):
return cases
-def generate_binding_java_cases(cases, hint):
+def generate_binding_java_cases(
+ cases: list[dict[str, str]], hint: Hint
+) -> list[dict[str, str]]:
cases = unique_cases(cases)
# Always run all tests if it is a push event.
@@ -192,20 +205,43 @@ def generate_binding_java_cases(cases, hint):
return cases
-def plan(changed_files):
+def generate_binding_python_cases(
+ cases: list[dict[str, str]], hint: Hint
+) -> list[dict[str, str]]:
+ cases = unique_cases(cases)
+ if os.getenv("GITHUB_IS_PUSH") == "true":
+ return cases
+
+ # Return empty if core is False
+ if not hint.binding_python:
+ return []
+
+ # Return all services if all_service is True
+ if hint.all_service:
+ return cases
+
+ # Filter all cases that not shown un in changed files
+ cases = [v for v in cases if v["service"] in hint.services]
+ return cases
+
+
+def plan(changed_files: list[str]) -> dict[str, Any]:
cases = provided_cases()
hint = calculate_hint(changed_files)
core_cases = generate_core_cases(cases, hint)
binding_java_cases = generate_binding_java_cases(cases, hint)
+ binding_python_cases = generate_binding_python_cases(cases, hint)
jobs = {
"components": {
"core": False,
"binding_java": False,
+ "binding_python": False,
},
"core": [],
"binding_java": [],
+ "binding_python": [],
}
if len(core_cases) > 0:
@@ -228,6 +264,11 @@ def plan(changed_files):
jobs["binding_java"].append(
{"os": "ubuntu-latest", "cases": binding_java_cases}
)
+ if len(binding_python_cases) > 0:
+ jobs["components"]["binding_python"] = True
+ jobs["binding_python"].append(
+ {"os": "ubuntu-latest", "cases": binding_python_cases}
+ )
return jobs
diff --git a/.github/services/mysql/mysql/action.yml b/.github/services/mysql/mysql/action.yml
index 2f2b332e5..a41e7d2be 100644
--- a/.github/services/mysql/mysql/action.yml
+++ b/.github/services/mysql/mysql/action.yml
@@ -24,7 +24,13 @@ runs:
- name: Setup MySQL Server
shell: bash
working-directory: fixtures/mysql
- run: docker-compose -f docker-compose.yml up -d
+ run: |
+ apt update && apt install -y mysql-client
+ docker-compose -f docker-compose.yml up -d
+ while ! mysql -h localhost -P 3306 --protocol=tcp -u root -p'password' -e "SELECT 1"; do
+ echo "Waiting for MySQL..."
+ sleep 1
+ done
- name: Setup
shell: bash
run: |
diff --git a/.github/workflows/behavior_test.yml b/.github/workflows/behavior_test.yml
index 364eba63b..861a7f981 100644
--- a/.github/workflows/behavior_test.yml
+++ b/.github/workflows/behavior_test.yml
@@ -102,3 +102,15 @@ jobs:
with:
os: ${{ matrix.os }}
cases: ${{ toJson(matrix.cases) }}
+ test_binding_python:
+ name: binding_python / ${{ matrix.os }}
+ needs: [plan]
+ if: fromJson(needs.plan.outputs.plan).components.binding_python
+ secrets: inherit
+ strategy:
+ matrix:
+ include: ${{ fromJson(needs.plan.outputs.plan).binding_python }}
+ uses: ./.github/workflows/behavior_test_binding_python.yml
+ with:
+ os: ${{ matrix.os }}
+ cases: ${{ toJson(matrix.cases) }}
diff --git a/.github/workflows/behavior_test_binding_python.yml b/.github/workflows/behavior_test_binding_python.yml
new file mode 100644
index 000000000..ed6b099d2
--- /dev/null
+++ b/.github/workflows/behavior_test_binding_python.yml
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Behavior Test Binding Python
+
+on:
+ workflow_call:
+ inputs:
+ os:
+ required: true
+ type: string
+ cases:
+ required: true
+ type: string
+
+jobs:
+ test:
+ name: ${{ matrix.cases.service }} / ${{ matrix.cases.setup }}
+ runs-on: ${{ inputs.os }}
+ strategy:
+ matrix:
+ cases: ${{ fromJson(inputs.cases) }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup
+ with:
+ need-nextest: true
+ need-protoc: true
+ need-rocksdb: true
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+
+ # TODO: 1password is only supported on linux
+ #
+ # Waiting for https://github.com/1Password/load-secrets-action/issues/46
+ - name: Setup service account token for 1password
+ if: runner.os == 'Linux'
+ shell: bash
+ run: echo "OP_SERVICE_ACCOUNT_TOKEN=${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" >> $GITHUB_ENV
+ - uses: actions/setup-python@v4
+ with:
+ python-version: '3.11'
+ - name: Build with maturin
+ shell: bash
+ working-directory: "bindings/python"
+ run: |
+ python -m venv venv
+ source venv/bin/activate
+ pip install maturin[patchelf]
+ maturin develop -E test -F ${{ matrix.cases.feature }}
+
+ - name: Test Core
+ uses: ./.github/actions/behavior_test_binding_python
+ with:
+ setup: ${{ matrix.cases.setup }}
+ service: ${{ matrix.cases.service }}
+ feature: ${{ matrix.cases.feature }}
diff --git a/.github/workflows/bindings_python.yml b/.github/workflows/bindings_python.yml
index 56bfbea68..d3ba0bcd2 100644
--- a/.github/workflows/bindings_python.yml
+++ b/.github/workflows/bindings_python.yml
@@ -39,25 +39,6 @@ permissions:
contents: read
jobs:
- test:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-python@v4
- with:
- python-version: '3.11'
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- - name: Build with maturin
- working-directory: "bindings/python"
- run: |
- python -m pip install -e .[test]
- - name: Run pytest
- working-directory: "bindings/python"
- env:
- OPENDAL_MEMORY_TEST: on
- run: |
- pytest -vk TestMemory
sdist:
runs-on: ubuntu-latest
diff --git a/.github/workflows/service_test_memory.yml b/.github/workflows/service_test_memory.yml
deleted file mode 100644
index 3277a4a67..000000000
--- a/.github/workflows/service_test_memory.yml
+++ /dev/null
@@ -1,61 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Service Test Memory
-
-on:
- push:
- branches:
- - main
- pull_request:
- branches:
- - main
- paths:
- - "core/src/**"
- - "core/tests/**"
- - "!core/src/docs/**"
- - "!core/src/services/**"
- - "core/src/services/memory/**"
- - ".github/workflows/service_test_memory.yml"
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
- cancel-in-progress: true
-
-jobs:
- python:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-python@v4
- with:
- python-version: "3.11"
-
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
-
- - name: Build with maturin
- working-directory: "bindings/python"
- run: |
- python -m pip install -e .[test]
-
- - name: Test
- shell: bash
- working-directory: bindings/python
- run: pytest -vk TestMemory
- env:
- OPENDAL_MEMORY_TEST: on
diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml
index ed2f5a387..1a255585a 100644
--- a/.github/workflows/service_test_s3.yml
+++ b/.github/workflows/service_test_s3.yml
@@ -69,44 +69,4 @@ jobs:
OPENDAL_S3_ROOT: CI/
OPENDAL_S3_BUCKET: opendal-testing
OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing
- OPENDAL_S3_REGION: ap-northeast-1
-
- python:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-python@v4
- with:
- python-version: "3.11"
-
- - name: Setup MinIO Server
- shell: bash
- working-directory: fixtures/s3
- run: docker-compose -f docker-compose-minio.yml up -d
-
- - name: Setup test bucket
- env:
- AWS_ACCESS_KEY_ID: "minioadmin"
- AWS_SECRET_ACCESS_KEY: "minioadmin"
- AWS_EC2_METADATA_DISABLED: "true"
- run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
-
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
-
- - name: Build with maturin
- working-directory: "bindings/python"
- run: |
- python -m pip install -e .[test]
-
- - name: Test
- shell: bash
- working-directory: bindings/python
- run: pytest -vk TestS3
- env:
- OPENDAL_S3_TEST: on
- OPENDAL_S3_BUCKET: test
- OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000"
- OPENDAL_S3_ACCESS_KEY_ID: minioadmin
- OPENDAL_S3_SECRET_ACCESS_KEY: minioadmin
- OPENDAL_S3_REGION: us-east-1
+ OPENDAL_S3_REGION: ap-northeast-1
\ No newline at end of file
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
index 4bf502e8e..a2ef50b5b 100644
--- a/bindings/python/Cargo.toml
+++ b/bindings/python/Cargo.toml
@@ -27,6 +27,106 @@ repository.workspace = true
rust-version.workspace = true
version.workspace = true
+
+[features]
+# Enable all opendal default feature by default.
+default = [
+ "services-azblob",
+ "services-azdls",
+ "services-cos",
+ "services-fs",
+ "services-gcs",
+ "services-ghac",
+ "services-http",
+ "services-ipmfs",
+ "services-memory",
+ "services-obs",
+ "services-oss",
+ "services-s3",
+ "services-webdav",
+ "services-webhdfs",
+]
+
+services-all = [
+ "default",
+
+ "services-cacache",
+ "services-dashmap",
+ "services-dropbox",
+ "services-etcd",
+ # FIXME this requires a preinstalled fdb library
+ # "services-foundationdb",
+ "services-ftp",
+ "services-gdrive",
+ # FIXME how to support HDFS services in bindings?
+ # "services-hdfs",
+ "services-ipfs",
+ "services-memcached",
+ "services-mini-moka",
+ "services-moka",
+ "services-onedrive",
+ "services-persy",
+ "services-postgresql",
+ "services-mysql",
+ "services-redb",
+ "services-redis",
+ "services-redis-rustls",
+ "services-rocksdb",
+ "services-sled",
+ "services-supabase",
+ "services-tikv",
+ "services-vercel-artifacts",
+ "services-wasabi",
+ "services-mongodb",
+ "services-sqlite",
+]
+
+# Default services provided by opendal.
+services-azblob = [ "opendal/services-azblob" ]
+services-azdls = [ "opendal/services-azdls" ]
+services-cos = [ "opendal/services-cos" ]
+services-fs = [ "opendal/services-fs" ]
+services-gcs = [ "opendal/services-gcs" ]
+services-ghac = [ "opendal/services-ghac" ]
+services-http = [ "opendal/services-http" ]
+services-ipmfs = [ "opendal/services-ipmfs" ]
+services-memory = [ "opendal/services-memory" ]
+services-obs = [ "opendal/services-obs" ]
+services-oss = [ "opendal/services-oss" ]
+services-s3 = [ "opendal/services-s3" ]
+services-webdav = [ "opendal/services-webdav" ]
+services-webhdfs = [ "opendal/services-webhdfs" ]
+
+# Optional services provided by opendal.
+services-cacache = ["opendal/services-cacache"]
+services-dashmap = ["opendal/services-dashmap"]
+services-dropbox = ["opendal/services-dropbox"]
+services-etcd = ["opendal/services-etcd"]
+services-foundationdb = ["opendal/services-foundationdb"]
+services-ftp = ["opendal/services-ftp"]
+services-gdrive = ["opendal/services-gdrive"]
+services-hdfs = ["opendal/services-hdfs"]
+services-ipfs = ["opendal/services-ipfs"]
+services-memcached = ["opendal/services-memcached"]
+services-mini-moka = ["opendal/services-mini-moka"]
+services-moka = ["opendal/services-moka"]
+services-onedrive = ["opendal/services-onedrive"]
+services-persy = ["opendal/services-persy"]
+services-postgresql = ["opendal/services-postgresql"]
+services-redb = ["opendal/services-redb"]
+services-redis = ["opendal/services-redis"]
+services-redis-rustls = ["opendal/services-redis-rustls"]
+services-rocksdb = ["opendal/services-rocksdb"]
+services-sftp = ["opendal/services-sftp"]
+services-sled = ["opendal/services-sled"]
+services-supabase = ["opendal/services-supabase"]
+services-tikv = ["opendal/services-tikv"]
+services-vercel-artifacts = ["opendal/services-vercel-artifacts"]
+services-wasabi = ["opendal/services-wasabi"]
+services-mysql = ["opendal/services-mysql"]
+services-mongodb = ["opendal/services-mongodb"]
+services-sqlite = ["opendal/services-sqlite"]
+
[lib]
crate-type = ["cdylib"]
doc = false
diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py
index 9db7dbe92..6c1a518da 100644
--- a/bindings/python/tests/conftest.py
+++ b/bindings/python/tests/conftest.py
@@ -15,9 +15,69 @@
# specific language governing permissions and limitations
# under the License.
+import os
+
from dotenv import load_dotenv
import pytest
+import opendal
load_dotenv()
pytest_plugins = ("pytest_asyncio",)
+
+
+def pytest_configure(config):
+ # register an additional marker
+ config.addinivalue_line(
+ "markers",
+ "need_capability(*capability): mark test to run only on named capability",
+ )
+
+
+@pytest.fixture()
+def service_name():
+ service_name = os.environ.get("OPENDAL_TEST")
+ if service_name is None:
+ pytest.skip("OPENDAL_TEST not set")
+ return service_name
+
+
+@pytest.fixture()
+def setup_config(service_name):
+ # Read arguments from envs.
+ prefix = f"opendal_{service_name}_"
+ config = {}
+ for key in os.environ.keys():
+ if key.lower().startswith(prefix):
+ config[key[len(prefix) :].lower()] = os.environ.get(key)
+
+ return config
+
+
+@pytest.fixture()
+def operator(service_name, setup_config):
+ return opendal.Operator(service_name, **setup_config)
+
+
+@pytest.fixture()
+def async_operator(service_name, setup_config):
+ return opendal.AsyncOperator(service_name, **setup_config)
+
+
+@pytest.fixture(autouse=True)
+def check_capability(request, operator, async_operator):
+ if request.node.get_closest_marker("need_capability"):
+ if request.node.get_closest_marker("need_capability").args:
+ if not all(
+ [
+ getattr(operator.capability(), x)
+ for x in request.node.get_closest_marker("need_capability").args
+ ]
+ + [
+ getattr(async_operator.capability(), x)
+ for x in request.node.get_closest_marker("need_capability").args
+ ]
+ ):
+ pytest.skip(
+ f"skip because {request.node.get_closest_marker('need_capability').args} not supported"
+ )
diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/test_capability.py
similarity index 71%
copy from bindings/python/tests/conftest.py
copy to bindings/python/tests/test_capability.py
index 9db7dbe92..da2e9c14f 100644
--- a/bindings/python/tests/conftest.py
+++ b/bindings/python/tests/test_capability.py
@@ -15,9 +15,17 @@
# specific language governing permissions and limitations
# under the License.
-from dotenv import load_dotenv
import pytest
-load_dotenv()
-pytest_plugins = ("pytest_asyncio",)
+def test_capability(service_name, operator):
+ cap = operator.capability()
+ assert cap is not None
+ assert cap.read is not None
+
+
+def test_capability_exception(service_name, operator):
+ cap = operator.capability()
+ assert cap is not None
+ with pytest.raises(AttributeError) as e_info:
+ cap.read_demo
diff --git a/bindings/python/tests/test_read.py b/bindings/python/tests/test_read.py
new file mode 100644
index 000000000..7c89ac9fd
--- /dev/null
+++ b/bindings/python/tests/test_read.py
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from uuid import uuid4
+from random import randint
+
+import pytest
+
+
+@pytest.mark.need_capability("read", "write", "delete")
+def test_sync_read(service_name, operator, async_operator):
+ size = randint(1, 1024)
+ filename = f"random_file_{str(uuid4())}"
+ content = os.urandom(size)
+ operator.write(filename, content)
+
+ read_content = operator.read(filename)
+ assert read_content is not None
+ assert read_content == content
+
+ operator.delete(filename)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read", "write", "delete")
+async def test_async_read(service_name, operator, async_operator):
+ size = randint(1, 1024)
+ filename = f"random_file_{str(uuid4())}"
+ content = os.urandom(size)
+ await async_operator.write(filename, content)
+
+ read_content = await async_operator.read(filename)
+ assert read_content is not None
+ assert read_content == content
+
+ await async_operator.delete(filename)
+
+
+@pytest.mark.need_capability("read", "write", "delete", "stat")
+def test_sync_read_stat(service_name, operator, async_operator):
+ size = randint(1, 1024)
+ filename = f"random_file_{str(uuid4())}"
+ content = os.urandom(size)
+ operator.write(filename, content)
+
+ metadata = operator.stat(filename)
+ assert metadata is not None
+ assert metadata.content_length == len(content)
+ assert metadata.mode.is_file()
+
+ operator.delete(filename)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read", "write", "delete", "stat")
+async def test_async_read_stat(service_name, operator, async_operator):
+ size = randint(1, 1024)
+ filename = f"random_file_{str(uuid4())}"
+ content = os.urandom(size)
+ await async_operator.write(filename, content)
+
+ metadata = await async_operator.stat(filename)
+ assert metadata is not None
+ assert metadata.content_length == len(content)
+ assert metadata.mode.is_file()
+
+ await async_operator.delete(filename)
+
+ operator.delete(filename)
+
+
+@pytest.mark.need_capability("read")
+def test_sync_read_not_exists(service_name, operator, async_operator):
+ with pytest.raises(FileNotFoundError):
+ operator.read(str(uuid4()))
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read")
+async def test_async_read_not_exists(service_name, operator, async_operator):
+ with pytest.raises(FileNotFoundError):
+ await async_operator.read(str(uuid4()))
diff --git a/bindings/python/tests/test_services.py b/bindings/python/tests/test_services.py
deleted file mode 100644
index 1d23cefc2..000000000
--- a/bindings/python/tests/test_services.py
+++ /dev/null
@@ -1,224 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-from abc import ABC
-from uuid import uuid4
-from random import randint
-
-import opendal
-import pytest
-
-
-class AbstractTestSuite(ABC):
- service_name = ""
-
- def setup_method(self):
- # Read arguments from envs.
- prefix = f"opendal_{self.service_name}_"
- self.config = {}
- for key in os.environ.keys():
- if key.lower().startswith(prefix):
- self.config[key[len(prefix) :].lower()] = os.environ.get(key)
-
- # Check if current test be enabled.
- test_flag = self.config.get("test", "")
- if test_flag != "on" and test_flag != "true":
- raise ValueError(f"Service {self.service_name} test is not enabled.")
-
- self.operator = opendal.Operator(self.service_name, **self.config)
- self.async_operator = opendal.AsyncOperator(self.service_name, **self.config)
-
- def test_sync_read(self):
- size = randint(1, 1024)
- filename = f"random_file_{str(uuid4())}"
- content = os.urandom(size)
- self.operator.write(filename, content)
-
- read_content = self.operator.read(filename)
- assert read_content is not None
- assert read_content == content
-
- self.operator.delete(filename)
-
- @pytest.mark.asyncio
- async def test_async_read(self):
- size = randint(1, 1024)
- filename = f"random_file_{str(uuid4())}"
- content = os.urandom(size)
- await self.async_operator.write(filename, content)
-
- read_content = await self.async_operator.read(filename)
- assert read_content is not None
- assert read_content == content
-
- await self.async_operator.delete(filename)
-
- def test_sync_read_stat(self):
- size = randint(1, 1024)
- filename = f"random_file_{str(uuid4())}"
- content = os.urandom(size)
- self.operator.write(filename, content)
-
- metadata = self.operator.stat(filename)
- assert metadata is not None
- assert metadata.content_length == len(content)
- assert metadata.mode.is_file()
-
- self.operator.delete(filename)
-
- @pytest.mark.asyncio
- async def test_async_read_stat(self):
- size = randint(1, 1024)
- filename = f"random_file_{str(uuid4())}"
- content = os.urandom(size)
- await self.async_operator.write(filename, content)
-
- metadata = await self.async_operator.stat(filename)
- assert metadata is not None
- assert metadata.content_length == len(content)
- assert metadata.mode.is_file()
-
- await self.async_operator.delete(filename)
-
- self.operator.delete(filename)
-
- def test_sync_read_not_exists(self):
- with pytest.raises(FileNotFoundError):
- self.operator.read(str(uuid4()))
-
- @pytest.mark.asyncio
- async def test_async_read_not_exists(self):
- with pytest.raises(FileNotFoundError):
- await self.async_operator.read(str(uuid4()))
-
- def test_sync_write(self):
- size = randint(1, 1024)
- filename = f"test_file_{str(uuid4())}.txt"
- content = os.urandom(size)
- size = len(content)
- self.operator.write(filename, content)
- metadata = self.operator.stat(filename)
- assert metadata is not None
- assert metadata.mode.is_file()
- assert metadata.content_length == size
-
- self.operator.delete(filename)
-
- @pytest.mark.asyncio
- async def test_async_write(self):
- size = randint(1, 1024)
- filename = f"test_file_{str(uuid4())}.txt"
- content = os.urandom(size)
- size = len(content)
- await self.async_operator.write(filename, content)
- metadata = await self.async_operator.stat(filename)
- assert metadata is not None
- assert metadata.mode.is_file()
- assert metadata.content_length == size
-
- await self.async_operator.delete(filename)
-
- def test_sync_write_with_non_ascii_name(self):
- size = randint(1, 1024)
- filename = f"βπ±δΈζ_{str(uuid4())}.test"
- content = os.urandom(size)
- size = len(content)
- self.operator.write(filename, content)
- metadata = self.operator.stat(filename)
- assert metadata is not None
- assert metadata.mode.is_file()
- assert metadata.content_length == size
-
- self.operator.delete(filename)
-
- @pytest.mark.asyncio
- async def test_async_write_with_non_ascii_name(self):
- size = randint(1, 1024)
- filename = f"βπ±δΈζ_{str(uuid4())}.test"
- content = os.urandom(size)
- size = len(content)
- await self.async_operator.write(filename, content)
- metadata = await self.async_operator.stat(filename)
- assert metadata is not None
- assert metadata.mode.is_file()
- assert metadata.content_length == size
-
- await self.async_operator.delete(filename)
-
- def test_sync_create_dir(self):
- path = f"test_dir_{str(uuid4())}/"
- self.operator.create_dir(path)
- metadata = self.operator.stat(path)
- assert metadata is not None
- assert metadata.mode.is_dir()
-
- self.operator.delete(path)
-
- @pytest.mark.asyncio
- async def test_async_create_dir(self):
- path = f"test_dir_{str(uuid4())}/"
- await self.async_operator.create_dir(path)
- metadata = await self.async_operator.stat(path)
- assert metadata is not None
- assert metadata.mode.is_dir()
-
- await self.async_operator.delete(path)
-
- def test_sync_delete(self):
- size = randint(1, 1024)
- filename = f"test_file_{str(uuid4())}.txt"
- content = os.urandom(size)
- size = len(content)
- self.operator.write(filename, content)
- self.operator.delete(filename)
- with pytest.raises(FileNotFoundError):
- self.operator.stat(filename)
-
- @pytest.mark.asyncio
- async def test_async_delete(self):
- size = randint(1, 1024)
- filename = f"test_file_{str(uuid4())}.txt"
- content = os.urandom(size)
- size = len(content)
- await self.async_operator.write(filename, content)
- await self.async_operator.delete(filename)
- with pytest.raises(FileNotFoundError):
- await self.operator.stat(filename)
-
- def test_capability(self):
- cap = self.operator.capability()
- assert cap is not None
- assert cap.read is not None
-
- def test_capability_exception(self):
- cap = self.operator.capability()
- assert cap is not None
- with pytest.raises(AttributeError) as e_info:
- cap.read_demo
-
-
-class TestS3(AbstractTestSuite):
- service_name = "s3"
-
-
-class TestFS(AbstractTestSuite):
- service_name = "fs"
-
-
-class TestMemory(AbstractTestSuite):
- service_name = "memory"
diff --git a/bindings/python/tests/test_write.py b/bindings/python/tests/test_write.py
new file mode 100644
index 000000000..12e173ed9
--- /dev/null
+++ b/bindings/python/tests/test_write.py
@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from uuid import uuid4
+from random import randint
+
+import pytest
+
+
+@pytest.mark.need_capability("write", "delete", "stat")
+def test_sync_write(service_name, operator, async_operator):
+ size = randint(1, 1024)
+ filename = f"test_file_{str(uuid4())}.txt"
+ content = os.urandom(size)
+ size = len(content)
+ operator.write(filename, content)
+ metadata = operator.stat(filename)
+ assert metadata is not None
+ assert metadata.mode.is_file()
+ assert metadata.content_length == size
+
+ operator.delete(filename)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("write", "delete", "stat")
+async def test_async_write(service_name, operator, async_operator):
+ size = randint(1, 1024)
+ filename = f"test_file_{str(uuid4())}.txt"
+ content = os.urandom(size)
+ size = len(content)
+ await async_operator.write(filename, content)
+ metadata = await async_operator.stat(filename)
+ assert metadata is not None
+ assert metadata.mode.is_file()
+ assert metadata.content_length == size
+
+ await async_operator.delete(filename)
+
+
+@pytest.mark.need_capability("write", "delete", "stat")
+def test_sync_write_with_non_ascii_name(service_name, operator, async_operator):
+ size = randint(1, 1024)
+ filename = f"βπ±δΈζ_{str(uuid4())}.test"
+ content = os.urandom(size)
+ size = len(content)
+ operator.write(filename, content)
+ metadata = operator.stat(filename)
+ assert metadata is not None
+ assert metadata.mode.is_file()
+ assert metadata.content_length == size
+
+ operator.delete(filename)
+
+@pytest.mark.need_capability("create_dir", "stat")
+def test_sync_create_dir(service_name, operator, async_operator):
+ path = f"test_dir_{str(uuid4())}/"
+ operator.create_dir(path)
+ metadata = operator.stat(path)
+ assert metadata is not None
+ assert metadata.mode.is_dir()
+
+ operator.delete(path)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("create_dir", "stat")
+async def test_async_create_dir(service_name, operator, async_operator):
+ path = f"test_dir_{str(uuid4())}/"
+ await async_operator.create_dir(path)
+ metadata = await async_operator.stat(path)
+ assert metadata is not None
+ assert metadata.mode.is_dir()
+
+ await async_operator.delete(path)
+
+
+@pytest.mark.need_capability("delete", "stat")
+def test_sync_delete(service_name, operator, async_operator):
+ size = randint(1, 1024)
+ filename = f"test_file_{str(uuid4())}.txt"
+ content = os.urandom(size)
+ size = len(content)
+ operator.write(filename, content)
+ operator.delete(filename)
+ with pytest.raises(FileNotFoundError):
+ operator.stat(filename)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("delete", "stat")
+async def test_async_delete(service_name, operator, async_operator):
+ size = randint(1, 1024)
+ filename = f"test_file_{str(uuid4())}.txt"
+ content = os.urandom(size)
+ size = len(content)
+ await async_operator.write(filename, content)
+ await async_operator.delete(filename)
+ with pytest.raises(FileNotFoundError):
+ await operator.stat(filename)