You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/05/24 10:02:54 UTC
[incubator-opendal] branch main updated: refactor(core): Implement RFC-2299 for stat_with (#2303)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 5dc2736a refactor(core): Implement RFC-2299 for stat_with (#2303)
5dc2736a is described below
commit 5dc2736a6eb10a543e1dfecdf74b136434269a32
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed May 24 18:02:44 2023 +0800
refactor(core): Implement RFC-2299 for stat_with (#2303)
* refactor(core): Implement RFC-2299 for stat_with
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Add docs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---------
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/http/backend.rs | 4 +-
core/src/types/mod.rs | 1 +
core/src/types/operator/mod.rs | 2 +
core/src/types/operator/operator.rs | 24 +++--
core/src/types/operator/operator_futures.rs | 137 ++++++++++++++++++++++++++++
core/tests/behavior/read_only.rs | 27 ++----
core/tests/behavior/write.rs | 30 +++---
7 files changed, 182 insertions(+), 43 deletions(-)
diff --git a/core/src/services/http/backend.rs b/core/src/services/http/backend.rs
index 8be0f16a..e719c42b 100644
--- a/core/src/services/http/backend.rs
+++ b/core/src/services/http/backend.rs
@@ -558,9 +558,7 @@ mod tests {
builder.endpoint(&mock_server.uri());
builder.root("/");
let op = Operator::new(builder)?.finish();
- let bs = op
- .stat_with("hello", OpStat::new().with_if_none_match("*"))
- .await?;
+ let bs = op.stat_with("hello").if_none_match("*").await?;
assert_eq!(bs.mode(), EntryMode::FILE);
assert_eq!(bs.content_length(), 128);
diff --git a/core/src/types/mod.rs b/core/src/types/mod.rs
index 00bed4bf..ce9b51f2 100644
--- a/core/src/types/mod.rs
+++ b/core/src/types/mod.rs
@@ -41,6 +41,7 @@ pub use list::BlockingLister;
pub use list::Lister;
mod operator;
+pub use operator::operator_futures;
pub use operator::BlockingOperator;
pub use operator::Operator;
pub use operator::OperatorBuilder;
diff --git a/core/src/types/operator/mod.rs b/core/src/types/operator/mod.rs
index a01475a9..6689a679 100644
--- a/core/src/types/operator/mod.rs
+++ b/core/src/types/operator/mod.rs
@@ -29,3 +29,5 @@ pub use builder::OperatorBuilder;
mod metadata;
pub use metadata::OperatorInfo;
+
+pub mod operator_futures;
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index da0cf1c7..d3949b60 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -28,6 +28,7 @@ use futures::TryStreamExt;
use tokio::io::ReadBuf;
use super::BlockingOperator;
+use crate::operator_futures::*;
use crate::ops::*;
use crate::raw::*;
use crate::*;
@@ -190,7 +191,7 @@ impl Operator {
/// # }
/// ```
pub async fn stat(&self, path: &str) -> Result<Metadata> {
- self.stat_with(path, OpStat::new()).await
+ self.stat_with(path).await
}
/// Get current path's metadata **without cache** directly with extra options.
@@ -217,7 +218,7 @@ impl Operator {
/// #
/// # #[tokio::main]
/// # async fn test(op: Operator) -> Result<()> {
- /// if let Err(e) = op.stat_with("test", OpStat::new()).await {
+ /// if let Err(e) = op.stat_with("test").if_match("<etag>").await {
/// if e.kind() == ErrorKind::NotFound {
/// println!("file not exist")
/// }
@@ -225,13 +226,24 @@ impl Operator {
/// # Ok(())
/// # }
/// ```
- pub async fn stat_with(&self, path: &str, args: OpStat) -> Result<Metadata> {
+ pub fn stat_with(&self, path: &str) -> FutureStat {
let path = normalize_path(path);
- let rp = self.inner().stat(&path, args).await?;
- let meta = rp.into_metadata();
+ let fut = FutureStat(OperatorFuture::new(
+ self.inner().clone(),
+ path,
+ OpStat::default(),
+ |inner, path, args| {
+ let fut = async move {
+ let rp = inner.stat(&path, args).await?;
+ Ok(rp.into_metadata())
+ };
+
+ Box::pin(fut)
+ },
+ ));
- Ok(meta)
+ fut
}
/// Get current metadata with cache.
diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs
new file mode 100644
index 00000000..a95b1efd
--- /dev/null
+++ b/core/src/types/operator/operator_futures.rs
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Futures provides the futures generated by [`Operator`]
+//!
+//! By using futures, users can add more options for operation.
+
+use std::mem;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use futures::future::BoxFuture;
+use futures::Future;
+use futures::FutureExt;
+
+use crate::ops::*;
+use crate::raw::*;
+use crate::*;
+
+/// OperatorFuture is the future generated by [`Operator`].
+///
+/// The future will consume all the input to generate a future.
+pub(crate) enum OperatorFuture<T, F> {
+ /// Idle state, waiting for the future to be polled
+ Idle(
+ /// The accessor to the underlying object storage
+ FusedAccessor,
+ /// The path of string
+ String,
+ /// The input args
+ T,
+ /// The function which will move all the args and return a static future
+ fn(FusedAccessor, String, T) -> BoxFuture<'static, Result<F>>,
+ ),
+ /// Polling state, waiting for the future to be ready
+ Poll(BoxFuture<'static, Result<F>>),
+ /// Empty state, the future has been polled and completed or
+ /// something is broken during state switch.
+ Empty,
+}
+
+impl<T, F> OperatorFuture<T, F> {
+ pub fn new(
+ inner: FusedAccessor,
+ path: String,
+ args: T,
+ f: fn(FusedAccessor, String, T) -> BoxFuture<'static, Result<F>>,
+ ) -> Self {
+ OperatorFuture::Idle(inner, path, args, f)
+ }
+
+ fn map_args(self, f: impl FnOnce(T) -> T) -> Self {
+ match self {
+ OperatorFuture::Idle(inner, path, args, func) => {
+ OperatorFuture::Idle(inner, path, f(args), func)
+ }
+ _ => unreachable!("future has been polled and should not be changed again"),
+ }
+ }
+}
+
+impl<T, F> Future for OperatorFuture<T, F>
+where
+ T: Unpin,
+ F: Unpin,
+{
+ type Output = Result<F>;
+
+ /// We will move the self state out by replace a `Empty` into.
+ ///
+ /// - If the future is `Idle`, we will move all args out to build
+ /// a new future, and update self state to `Poll`.
+ /// - If the future is `Poll`, we will poll the inner future
+ /// - If future is `Ready`, we will return it directly with
+ /// self state is `Empty`
+ /// - If future is `Pending`, we will set self state to `Poll`
+ /// and wait for next poll
+ ///
+ /// In general, `Empty` state should not be polled.
+ fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+ *self = match mem::replace(self.as_mut().get_mut(), OperatorFuture::Empty) {
+ OperatorFuture::Idle(inner, path, args, f) => {
+ OperatorFuture::Poll(f(inner, path, args))
+ }
+ OperatorFuture::Poll(mut fut) => match fut.as_mut().poll(cx) {
+ Poll::Pending => OperatorFuture::Poll(fut),
+ Poll::Ready(v) => return Poll::Ready(v),
+ },
+ OperatorFuture::Empty => {
+ panic!("future polled after completion");
+ }
+ };
+ cx.waker().wake_by_ref();
+ Poll::Pending
+ }
+}
+
+/// Future that generated by [`Operator::stat_with`].
+///
+/// Users can add more options by public functions provided by this struct.
+pub struct FutureStat(pub(crate) OperatorFuture<OpStat, Metadata>);
+
+impl FutureStat {
+ /// Set the If-Match for this operation.
+ pub fn if_match(mut self, v: &str) -> Self {
+ self.0 = self.0.map_args(|args| args.with_if_match(v));
+ self
+ }
+
+ /// Set the If-None-Match for this operation.
+ pub fn if_none_match(mut self, v: &str) -> Self {
+ self.0 = self.0.map_args(|args| args.with_if_none_match(v));
+ self
+ }
+}
+
+impl Future for FutureStat {
+ type Output = Result<Metadata>;
+
+ fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+ self.0.poll_unpin(cx)
+ }
+}
diff --git a/core/tests/behavior/read_only.rs b/core/tests/behavior/read_only.rs
index 9c1f60c6..da99322c 100644
--- a/core/tests/behavior/read_only.rs
+++ b/core/tests/behavior/read_only.rs
@@ -18,7 +18,6 @@
use anyhow::Result;
use futures::AsyncReadExt;
use opendal::ops::OpRead;
-use opendal::ops::OpStat;
use opendal::EntryMode;
use opendal::ErrorKind;
use opendal::Operator;
@@ -139,17 +138,14 @@ pub async fn test_stat_with_if_match(op: Operator) -> Result<()> {
assert_eq!(meta.mode(), EntryMode::FILE);
assert_eq!(meta.content_length(), 262144);
- let mut op_stat = OpStat::default();
- op_stat = op_stat.with_if_match("invalid_etag");
-
- let res = op.stat_with(path, op_stat).await;
+ let res = op.stat_with(path).if_match("invalid_etag").await;
assert!(res.is_err());
assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
- let mut op_stat = OpStat::default();
- op_stat = op_stat.with_if_match(meta.etag().expect("etag must exist"));
-
- let result = op.stat_with(path, op_stat).await;
+ let result = op
+ .stat_with(path)
+ .if_match(meta.etag().expect("etag must exist"))
+ .await;
assert!(result.is_ok());
Ok(())
@@ -167,17 +163,14 @@ pub async fn test_stat_with_if_none_match(op: Operator) -> Result<()> {
assert_eq!(meta.mode(), EntryMode::FILE);
assert_eq!(meta.content_length(), 262144);
- let mut op_stat = OpStat::default();
- op_stat = op_stat.with_if_none_match(meta.etag().expect("etag must exist"));
-
- let res = op.stat_with(path, op_stat).await;
+ let res = op
+ .stat_with(path)
+ .if_none_match(meta.etag().expect("etag must exist"))
+ .await;
assert!(res.is_err());
assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
- let mut op_stat = OpStat::default();
- op_stat = op_stat.with_if_none_match("invalid_etag");
-
- let res = op.stat_with(path, op_stat).await?;
+ let res = op.stat_with(path).if_none_match("invalid_etag").await?;
assert_eq!(res.mode(), meta.mode());
assert_eq!(res.content_length(), meta.content_length());
diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs
index 0da8a1d2..74b2366c 100644
--- a/core/tests/behavior/write.rs
+++ b/core/tests/behavior/write.rs
@@ -26,7 +26,6 @@ use http::StatusCode;
use log::debug;
use log::warn;
use opendal::ops::OpRead;
-use opendal::ops::OpStat;
use opendal::ops::OpWrite;
use opendal::EntryMode;
use opendal::ErrorKind;
@@ -373,17 +372,14 @@ pub async fn test_stat_with_if_match(op: Operator) -> Result<()> {
assert_eq!(meta.mode(), EntryMode::FILE);
assert_eq!(meta.content_length(), size as u64);
- let mut op_stat = OpStat::default();
- op_stat = op_stat.with_if_match("\"invalid_etag\"");
-
- let res = op.stat_with(&path, op_stat).await;
+ let res = op.stat_with(&path).if_match("\"invalid_etag\"").await;
assert!(res.is_err());
assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
- let mut op_stat = OpStat::default();
- op_stat = op_stat.with_if_match(meta.etag().expect("etag must exist"));
-
- let result = op.stat_with(&path, op_stat).await;
+ let result = op
+ .stat_with(&path)
+ .if_match(meta.etag().expect("etag must exist"))
+ .await;
assert!(result.is_ok());
op.delete(&path).await.expect("delete must succeed");
@@ -408,17 +404,17 @@ pub async fn test_stat_with_if_none_match(op: Operator) -> Result<()> {
assert_eq!(meta.mode(), EntryMode::FILE);
assert_eq!(meta.content_length(), size as u64);
- let mut op_stat = OpStat::default();
- op_stat = op_stat.with_if_none_match(meta.etag().expect("etag must exist"));
-
- let res = op.stat_with(&path, op_stat).await;
+ let res = op
+ .stat_with(&path)
+ .if_none_match(meta.etag().expect("etag must exist"))
+ .await;
assert!(res.is_err());
assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
- let mut op_stat = OpStat::default();
- op_stat = op_stat.with_if_none_match("\"invalid_etag\"");
-
- let res = op.stat_with(&path, op_stat).await?;
+ let res = op
+ .stat_with(&path)
+ .if_none_match("\"invalid_etag\"")
+ .await?;
assert_eq!(res.mode(), meta.mode());
assert_eq!(res.content_length(), meta.content_length());