You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by ma...@apache.org on 2023/11/01 11:50:38 UTC
(incubator-opendal) branch manjusaka/python-support-copy updated (aebdda656 -> ccf073a8b)
This is an automated email from the ASF dual-hosted git repository.
manjusaka pushed a change to branch manjusaka/python-support-copy
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
omit aebdda656 Update comment
omit d69ccb564 Update code
omit 96cb858bb sort Python import by isort
omit f47d4a821 feat(binding/python): Support Copy operation for Python binding
add 1d089426d refactor(services/wasabi)!: Remove native support for wasabi services (#3455)
add 0721a6e59 refactor(ci): Polish the test planner code (#3457)
new ae97d4f78 feat(binding/python): Support Copy operation for Python binding
new f2cf9dfd6 sort Python import by isort
new 359fb9ad0 Update code
new ccf073a8b Update comment
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (aebdda656)
\
N -- N -- N refs/heads/manjusaka/python-support-copy (ccf073a8b)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/scripts/behavior_test/plan.py | 61 +-
.github/scripts/behavior_test/test_plan.py | 1 +
.github/workflows/service_test_wasabi.yml | 68 ---
core/Cargo.toml | 9 +-
core/src/docs/upgrade.md | 4 +
core/src/services/mod.rs | 5 -
core/src/services/wasabi/backend.rs | 951 -----------------------------
core/src/services/wasabi/core.rs | 911 ---------------------------
core/src/services/wasabi/docs.md | 198 ------
core/src/services/wasabi/error.rs | 131 ----
core/src/services/wasabi/mod.rs | 24 -
core/src/services/wasabi/pager.rs | 231 -------
core/src/services/wasabi/writer.rs | 65 --
core/src/types/operator/builder.rs | 2 -
core/src/types/scheme.rs | 6 -
website/docs/services/wasabi.mdx | 70 ---
16 files changed, 18 insertions(+), 2719 deletions(-)
delete mode 100644 .github/workflows/service_test_wasabi.yml
delete mode 100644 core/src/services/wasabi/backend.rs
delete mode 100644 core/src/services/wasabi/core.rs
delete mode 100644 core/src/services/wasabi/docs.md
delete mode 100644 core/src/services/wasabi/error.rs
delete mode 100644 core/src/services/wasabi/mod.rs
delete mode 100644 core/src/services/wasabi/pager.rs
delete mode 100644 core/src/services/wasabi/writer.rs
delete mode 100644 website/docs/services/wasabi.mdx
(incubator-opendal) 03/04: Update code
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
manjusaka pushed a commit to branch manjusaka/python-support-copy
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 359fb9ad09562aecf258959d0cb38ec68cc05e7b
Author: Manjusaka <me...@manjusaka.me>
AuthorDate: Wed Nov 1 19:27:53 2023 +0800
Update code
Signed-off-by: Manjusaka <me...@manjusaka.me>
---
bindings/python/src/asyncio.rs | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/bindings/python/src/asyncio.rs b/bindings/python/src/asyncio.rs
index 9f0cf6a18..a97285851 100644
--- a/bindings/python/src/asyncio.rs
+++ b/bindings/python/src/asyncio.rs
@@ -140,8 +140,13 @@ impl AsyncOperator {
}
/// Copy from `src` to `dst`.
-
- pub fn copy<'p>(&'p self, py: Python<'p>, source: String, target: String) -> PyResult<&'p PyAny> {
+
+ pub fn copy<'p>(
+ &'p self,
+ py: Python<'p>,
+ source: String,
+ target: String,
+ ) -> PyResult<&'p PyAny> {
let this = self.0.clone();
future_into_py(py, async move {
this.copy(&source, &target).await.map_err(format_pyerr)
(incubator-opendal) 02/04: sort Python import by isort
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
manjusaka pushed a commit to branch manjusaka/python-support-copy
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit f2cf9dfd6949b0ff2a62e7257c0382694f83b862
Author: Manjusaka <me...@manjusaka.me>
AuthorDate: Wed Nov 1 19:23:15 2023 +0800
sort Python import by isort
Signed-off-by: Manjusaka <me...@manjusaka.me>
---
bindings/python/benchmark/async_opendal_benchmark.py | 6 ++++--
bindings/python/python/opendal/__init__.py | 1 -
bindings/python/tests/conftest.py | 4 ++--
bindings/python/tests/test_async_copy.py | 2 +-
bindings/python/tests/test_read.py | 2 +-
bindings/python/tests/test_sync_copy.py | 2 +-
bindings/python/tests/test_write.py | 2 +-
7 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/bindings/python/benchmark/async_opendal_benchmark.py b/bindings/python/benchmark/async_opendal_benchmark.py
index 368eaadbc..6b88b9433 100644
--- a/bindings/python/benchmark/async_opendal_benchmark.py
+++ b/bindings/python/benchmark/async_opendal_benchmark.py
@@ -15,11 +15,13 @@
# specific language governing permissions and limitations
# under the License.
-from pydantic import BaseSettings
import asyncio
-import opendal
import timeit
+from pydantic import BaseSettings
+
+import opendal
+
class Config(BaseSettings):
aws_endpoint: str
diff --git a/bindings/python/python/opendal/__init__.py b/bindings/python/python/opendal/__init__.py
index 35bc589a5..54f3a10ce 100644
--- a/bindings/python/python/opendal/__init__.py
+++ b/bindings/python/python/opendal/__init__.py
@@ -17,6 +17,5 @@
from ._opendal import *
-
__doc__ = _opendal.__doc__
__all__ = _opendal.__all__
diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py
index 6c1a518da..e4a88caa7 100644
--- a/bindings/python/tests/conftest.py
+++ b/bindings/python/tests/conftest.py
@@ -17,10 +17,10 @@
import os
-from dotenv import load_dotenv
import pytest
-import opendal
+from dotenv import load_dotenv
+import opendal
load_dotenv()
pytest_plugins = ("pytest_asyncio",)
diff --git a/bindings/python/tests/test_async_copy.py b/bindings/python/tests/test_async_copy.py
index dadff6401..b5fea80d1 100644
--- a/bindings/python/tests/test_async_copy.py
+++ b/bindings/python/tests/test_async_copy.py
@@ -16,8 +16,8 @@
# under the License.
import os
-from uuid import uuid4
from random import randint
+from uuid import uuid4
import pytest
diff --git a/bindings/python/tests/test_read.py b/bindings/python/tests/test_read.py
index 7c89ac9fd..9c318943c 100644
--- a/bindings/python/tests/test_read.py
+++ b/bindings/python/tests/test_read.py
@@ -16,8 +16,8 @@
# under the License.
import os
-from uuid import uuid4
from random import randint
+from uuid import uuid4
import pytest
diff --git a/bindings/python/tests/test_sync_copy.py b/bindings/python/tests/test_sync_copy.py
index 0a29da976..3db7bb6f1 100644
--- a/bindings/python/tests/test_sync_copy.py
+++ b/bindings/python/tests/test_sync_copy.py
@@ -16,8 +16,8 @@
# under the License.
import os
-from uuid import uuid4
from random import randint
+from uuid import uuid4
import pytest
diff --git a/bindings/python/tests/test_write.py b/bindings/python/tests/test_write.py
index 68395b3fb..987f0ce3c 100644
--- a/bindings/python/tests/test_write.py
+++ b/bindings/python/tests/test_write.py
@@ -16,8 +16,8 @@
# under the License.
import os
-from uuid import uuid4
from random import randint
+from uuid import uuid4
import pytest
(incubator-opendal) 01/04: feat(binding/python): Support Copy operation for Python binding
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
manjusaka pushed a commit to branch manjusaka/python-support-copy
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit ae97d4f784df4897dc1ff69195c54269b378a4ba
Author: Manjusaka <me...@manjusaka.me>
AuthorDate: Wed Nov 1 19:21:49 2023 +0800
feat(binding/python): Support Copy operation for Python binding
Signed-off-by: Manjusaka <me...@manjusaka.me>
---
bindings/python/python/opendal/__init__.pyi | 2 +
bindings/python/src/asyncio.rs | 9 +++
bindings/python/src/lib.rs | 5 ++
bindings/python/tests/test_async_copy.py | 114 ++++++++++++++++++++++++++++
bindings/python/tests/test_sync_copy.py | 107 ++++++++++++++++++++++++++
5 files changed, 237 insertions(+)
diff --git a/bindings/python/python/opendal/__init__.pyi b/bindings/python/python/opendal/__init__.pyi
index 55164c816..df8134e3e 100644
--- a/bindings/python/python/opendal/__init__.pyi
+++ b/bindings/python/python/opendal/__init__.pyi
@@ -39,6 +39,7 @@ class Operator:
def list(self, path: str) -> Iterable[Entry]: ...
def scan(self, path: str) -> Iterable[Entry]: ...
def capability(self) -> Capability: ...
+ def copy(self, source: str, target: str): ...
class AsyncOperator:
def __init__(self, scheme: str, **kwargs): ...
@@ -65,6 +66,7 @@ class AsyncOperator:
self, path: str, expire_second: int
) -> PresignedRequest: ...
def capability(self) -> Capability: ...
+ async def copy(self, source: str, target: str): ...
class Reader:
def read(self, size: Optional[int] = None) -> memoryview: ...
diff --git a/bindings/python/src/asyncio.rs b/bindings/python/src/asyncio.rs
index 1cedac75e..9f0cf6a18 100644
--- a/bindings/python/src/asyncio.rs
+++ b/bindings/python/src/asyncio.rs
@@ -139,6 +139,15 @@ impl AsyncOperator {
})
}
+ /// Copy from `src` to `dst`.
+
+ pub fn copy<'p>(&'p self, py: Python<'p>, source: String, target: String) -> PyResult<&'p PyAny> {
+ let this = self.0.clone();
+ future_into_py(py, async move {
+ this.copy(&source, &target).await.map_err(format_pyerr)
+ })
+ }
+
/// Create a dir at given path.
///
/// # Notes
diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs
index 1ee0884f9..4106ec492 100644
--- a/bindings/python/src/lib.rs
+++ b/bindings/python/src/lib.rs
@@ -188,6 +188,11 @@ impl Operator {
self.0.stat(path).map_err(format_pyerr).map(Metadata)
}
+ /// Copy src to dst.
+ pub fn copy(&self, source: &str, target: &str) -> PyResult<()> {
+ self.0.copy(source, target).map_err(format_pyerr)
+ }
+
/// Create a dir at given path.
///
/// # Notes
diff --git a/bindings/python/tests/test_async_copy.py b/bindings/python/tests/test_async_copy.py
new file mode 100644
index 000000000..dadff6401
--- /dev/null
+++ b/bindings/python/tests/test_async_copy.py
@@ -0,0 +1,114 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from uuid import uuid4
+from random import randint
+
+import pytest
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read", "write", "copy")
+async def test_async_copy(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}"
+ content = os.urandom(1024)
+ await async_operator.write(source_path, content)
+ target_path = f"random_file_{str(uuid4())}"
+ await async_operator.copy(source_path, target_path)
+ read_content = await async_operator.read(target_path)
+ assert read_content is not None
+ assert read_content == content
+ await async_operator.delete(source_path)
+ await async_operator.delete(target_path)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read", "write", "copy")
+async def test_async_copy_non_exist(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}"
+ target_path = f"random_file_{str(uuid4())}"
+ with pytest.raises(Exception) as e_info:
+ await async_operator.copy(source_path, target_path)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read", "write", "copy")
+async def test_async_copy_source_directory(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}/"
+ await async_operator.create_dir(source_path)
+ target_path = f"random_file_{str(uuid4())}"
+ with pytest.raises(Exception) as e_info:
+ await async_operator.copy(source_path, target_path)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read", "write", "copy")
+async def test_async_copy_target_directory(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}"
+ content = os.urandom(1024)
+ await async_operator.write(source_path, content)
+ target_path = f"random_file_{str(uuid4())}/"
+ await async_operator.create_dir(target_path)
+ with pytest.raises(Exception) as e_info:
+ await async_operator.copy(source_path, target_path)
+ await async_operator.delete(source_path)
+ await async_operator.delete(target_path)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read", "write", "copy")
+async def test_async_copy_self(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}"
+ content = os.urandom(1024)
+ await async_operator.write(source_path, content)
+ with pytest.raises(Exception) as e_info:
+ await async_operator.copy(source_path, source_path)
+ await async_operator.delete(source_path)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read", "write", "copy")
+async def test_async_copy_nested(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}"
+ target_path = f"random_file_{str(uuid4())}/{str(uuid4())}/{str(uuid4())}"
+ content = os.urandom(1024)
+ await async_operator.write(source_path, content)
+ await async_operator.copy(source_path, target_path)
+ target_content = await async_operator.read(target_path)
+ assert target_content is not None
+ assert target_content == content
+ await async_operator.delete(source_path)
+ await async_operator.delete(target_path)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read", "write", "copy")
+async def test_async_copy_overwrite(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}"
+ target_path = f"random_file_{str(uuid4())}"
+ source_content = os.urandom(1024)
+ target_content = os.urandom(1024)
+ assert source_content != target_content
+ await async_operator.write(source_path, source_content)
+ await async_operator.write(target_path, target_content)
+ await async_operator.copy(source_path, target_path)
+ target_content = await async_operator.read(target_path)
+ assert target_content is not None
+ assert target_content == source_content
+ await async_operator.delete(source_path)
+ await async_operator.delete(target_path)
diff --git a/bindings/python/tests/test_sync_copy.py b/bindings/python/tests/test_sync_copy.py
new file mode 100644
index 000000000..0a29da976
--- /dev/null
+++ b/bindings/python/tests/test_sync_copy.py
@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from uuid import uuid4
+from random import randint
+
+import pytest
+
+
+@pytest.mark.need_capability("read", "write", "copy")
+def test_sync_copy(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}"
+ content = os.urandom(1024)
+ operator.write(source_path, content)
+ target_path = f"random_file_{str(uuid4())}"
+ operator.copy(source_path, target_path)
+ read_content = operator.read(target_path)
+ assert read_content is not None
+ assert read_content == content
+ operator.delete(source_path)
+ operator.delete(target_path)
+
+
+@pytest.mark.need_capability("read", "write", "copy")
+def test_sync_copy_non_exist(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}"
+ target_path = f"random_file_{str(uuid4())}"
+ with pytest.raises(Exception) as e_info:
+ operator.copy(source_path, target_path)
+
+
+@pytest.mark.need_capability("read", "write", "copy")
+def test_sync_copy_source_directory(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}/"
+ operator.create_dir(source_path)
+ target_path = f"random_file_{str(uuid4())}"
+ with pytest.raises(Exception) as e_info:
+ operator.copy(source_path, target_path)
+
+
+@pytest.mark.need_capability("read", "write", "copy")
+def test_sync_copy_target_directory(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}"
+ content = os.urandom(1024)
+ operator.write(source_path, content)
+ target_path = f"random_file_{str(uuid4())}/"
+ operator.create_dir(target_path)
+ with pytest.raises(Exception) as e_info:
+ operator.copy(source_path, target_path)
+ operator.delete(source_path)
+ operator.delete(target_path)
+
+
+@pytest.mark.need_capability("read", "write", "copy")
+def test_sync_copy_self(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}"
+ content = os.urandom(1024)
+ operator.write(source_path, content)
+ with pytest.raises(Exception) as e_info:
+ operator.copy(source_path, source_path)
+ operator.delete(source_path)
+
+
+@pytest.mark.need_capability("read", "write", "copy")
+def test_sync_copy_nested(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}"
+ target_path = f"random_file_{str(uuid4())}/{str(uuid4())}/{str(uuid4())}"
+ content = os.urandom(1024)
+ operator.write(source_path, content)
+ operator.copy(source_path, target_path)
+ target_content = operator.read(target_path)
+ assert target_content is not None
+ assert target_content == content
+ operator.delete(source_path)
+ operator.delete(target_path)
+
+
+@pytest.mark.need_capability("read", "write", "copy")
+def test_sync_copy_overwrite(service_name, operator, async_operator):
+ source_path = f"random_file_{str(uuid4())}"
+ target_path = f"random_file_{str(uuid4())}"
+ source_content = os.urandom(1024)
+ target_content = os.urandom(1024)
+ assert source_content != target_content
+ operator.write(source_path, source_content)
+ operator.write(target_path, target_content)
+ operator.copy(source_path, target_path)
+ target_content = operator.read(target_path)
+ assert target_content is not None
+ assert target_content == source_content
+ operator.delete(source_path)
+ operator.delete(target_path)
(incubator-opendal) 04/04: Update comment
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
manjusaka pushed a commit to branch manjusaka/python-support-copy
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit ccf073a8b5c146ca9faac545771b7edfa580670c
Author: Manjusaka <me...@manjusaka.me>
AuthorDate: Wed Nov 1 19:33:35 2023 +0800
Update comment
Signed-off-by: Manjusaka <me...@manjusaka.me>
---
bindings/python/src/asyncio.rs | 3 +--
bindings/python/src/lib.rs | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/bindings/python/src/asyncio.rs b/bindings/python/src/asyncio.rs
index a97285851..2ebd99dd2 100644
--- a/bindings/python/src/asyncio.rs
+++ b/bindings/python/src/asyncio.rs
@@ -139,8 +139,7 @@ impl AsyncOperator {
})
}
- /// Copy from `src` to `dst`.
-
+ /// Copy source to target.``
pub fn copy<'p>(
&'p self,
py: Python<'p>,
diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs
index 4106ec492..8271c483a 100644
--- a/bindings/python/src/lib.rs
+++ b/bindings/python/src/lib.rs
@@ -188,7 +188,7 @@ impl Operator {
self.0.stat(path).map_err(format_pyerr).map(Metadata)
}
- /// Copy src to dst.
+ /// Copy source to target.
pub fn copy(&self, source: &str, target: &str) -> PyResult<()> {
self.0.copy(source, target).map_err(format_pyerr)
}