You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/05/24 10:02:54 UTC

[incubator-opendal] branch main updated: refactor(core): Implement RFC-2299 for stat_with (#2303)

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new 5dc2736a refactor(core): Implement RFC-2299 for stat_with (#2303)
5dc2736a is described below

commit 5dc2736a6eb10a543e1dfecdf74b136434269a32
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed May 24 18:02:44 2023 +0800

    refactor(core): Implement RFC-2299 for stat_with (#2303)
    
    * refactor(core): Implement RFC-2299 for stat_with
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
    
    * Add docs
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
    
    ---------
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 core/src/services/http/backend.rs           |   4 +-
 core/src/types/mod.rs                       |   1 +
 core/src/types/operator/mod.rs              |   2 +
 core/src/types/operator/operator.rs         |  24 +++--
 core/src/types/operator/operator_futures.rs | 137 ++++++++++++++++++++++++++++
 core/tests/behavior/read_only.rs            |  27 ++----
 core/tests/behavior/write.rs                |  30 +++---
 7 files changed, 182 insertions(+), 43 deletions(-)

diff --git a/core/src/services/http/backend.rs b/core/src/services/http/backend.rs
index 8be0f16a..e719c42b 100644
--- a/core/src/services/http/backend.rs
+++ b/core/src/services/http/backend.rs
@@ -558,9 +558,7 @@ mod tests {
         builder.endpoint(&mock_server.uri());
         builder.root("/");
         let op = Operator::new(builder)?.finish();
-        let bs = op
-            .stat_with("hello", OpStat::new().with_if_none_match("*"))
-            .await?;
+        let bs = op.stat_with("hello").if_none_match("*").await?;
 
         assert_eq!(bs.mode(), EntryMode::FILE);
         assert_eq!(bs.content_length(), 128);
diff --git a/core/src/types/mod.rs b/core/src/types/mod.rs
index 00bed4bf..ce9b51f2 100644
--- a/core/src/types/mod.rs
+++ b/core/src/types/mod.rs
@@ -41,6 +41,7 @@ pub use list::BlockingLister;
 pub use list::Lister;
 
 mod operator;
+pub use operator::operator_futures;
 pub use operator::BlockingOperator;
 pub use operator::Operator;
 pub use operator::OperatorBuilder;
diff --git a/core/src/types/operator/mod.rs b/core/src/types/operator/mod.rs
index a01475a9..6689a679 100644
--- a/core/src/types/operator/mod.rs
+++ b/core/src/types/operator/mod.rs
@@ -29,3 +29,5 @@ pub use builder::OperatorBuilder;
 
 mod metadata;
 pub use metadata::OperatorInfo;
+
+pub mod operator_futures;
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index da0cf1c7..d3949b60 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -28,6 +28,7 @@ use futures::TryStreamExt;
 use tokio::io::ReadBuf;
 
 use super::BlockingOperator;
+use crate::operator_futures::*;
 use crate::ops::*;
 use crate::raw::*;
 use crate::*;
@@ -190,7 +191,7 @@ impl Operator {
     /// # }
     /// ```
     pub async fn stat(&self, path: &str) -> Result<Metadata> {
-        self.stat_with(path, OpStat::new()).await
+        self.stat_with(path).await
     }
 
     /// Get current path's metadata **without cache** directly with extra options.
@@ -217,7 +218,7 @@ impl Operator {
     /// #
     /// # #[tokio::main]
     /// # async fn test(op: Operator) -> Result<()> {
-    /// if let Err(e) = op.stat_with("test", OpStat::new()).await {
+    /// if let Err(e) = op.stat_with("test").if_match("<etag>").await {
     ///     if e.kind() == ErrorKind::NotFound {
     ///         println!("file not exist")
     ///     }
@@ -225,13 +226,24 @@ impl Operator {
     /// # Ok(())
     /// # }
     /// ```
-    pub async fn stat_with(&self, path: &str, args: OpStat) -> Result<Metadata> {
+    pub fn stat_with(&self, path: &str) -> FutureStat {
         let path = normalize_path(path);
 
-        let rp = self.inner().stat(&path, args).await?;
-        let meta = rp.into_metadata();
+        let fut = FutureStat(OperatorFuture::new(
+            self.inner().clone(),
+            path,
+            OpStat::default(),
+            |inner, path, args| {
+                let fut = async move {
+                    let rp = inner.stat(&path, args).await?;
+                    Ok(rp.into_metadata())
+                };
+
+                Box::pin(fut)
+            },
+        ));
 
-        Ok(meta)
+        fut
     }
 
     /// Get current metadata with cache.
diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs
new file mode 100644
index 00000000..a95b1efd
--- /dev/null
+++ b/core/src/types/operator/operator_futures.rs
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Futures provides the futures generated by [`Operator`]
+//!
+//! By using futures, users can add more options for operation.
+
+use std::mem;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use futures::future::BoxFuture;
+use futures::Future;
+use futures::FutureExt;
+
+use crate::ops::*;
+use crate::raw::*;
+use crate::*;
+
+/// OperatorFuture is the future generated by [`Operator`].
+///
+/// The future will consume all the input to generate a future.
+pub(crate) enum OperatorFuture<T, F> {
+    /// Idle state, waiting for the future to be polled
+    Idle(
+        /// The accessor to the underlying object storage
+        FusedAccessor,
+        /// The path of string
+        String,
+        /// The input args
+        T,
+        /// The function which will move all the args and return a static future
+        fn(FusedAccessor, String, T) -> BoxFuture<'static, Result<F>>,
+    ),
+    /// Polling state, waiting for the future to be ready
+    Poll(BoxFuture<'static, Result<F>>),
+    /// Empty state, the future has been polled and completed or
+    /// something is broken during state switch.
+    Empty,
+}
+
+impl<T, F> OperatorFuture<T, F> {
+    pub fn new(
+        inner: FusedAccessor,
+        path: String,
+        args: T,
+        f: fn(FusedAccessor, String, T) -> BoxFuture<'static, Result<F>>,
+    ) -> Self {
+        OperatorFuture::Idle(inner, path, args, f)
+    }
+
+    fn map_args(self, f: impl FnOnce(T) -> T) -> Self {
+        match self {
+            OperatorFuture::Idle(inner, path, args, func) => {
+                OperatorFuture::Idle(inner, path, f(args), func)
+            }
+            _ => unreachable!("future has been polled and should not be changed again"),
+        }
+    }
+}
+
+impl<T, F> Future for OperatorFuture<T, F>
+where
+    T: Unpin,
+    F: Unpin,
+{
+    type Output = Result<F>;
+
+    /// We will move the self state out by replace a `Empty` into.
+    ///
+    /// - If the future is `Idle`, we will move all args out to build
+    ///   a new future, and update self state to `Poll`.
+    /// - If the future is `Poll`, we will poll the inner future
+    ///   - If future is `Ready`, we will return it directly with
+    ///     self state is `Empty`
+    ///   - If future is `Pending`, we will set self state to `Poll`
+    ///     and wait for next poll
+    ///
+    /// In general, `Empty` state should not be polled.
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        *self = match mem::replace(self.as_mut().get_mut(), OperatorFuture::Empty) {
+            OperatorFuture::Idle(inner, path, args, f) => {
+                OperatorFuture::Poll(f(inner, path, args))
+            }
+            OperatorFuture::Poll(mut fut) => match fut.as_mut().poll(cx) {
+                Poll::Pending => OperatorFuture::Poll(fut),
+                Poll::Ready(v) => return Poll::Ready(v),
+            },
+            OperatorFuture::Empty => {
+                panic!("future polled after completion");
+            }
+        };
+        cx.waker().wake_by_ref();
+        Poll::Pending
+    }
+}
+
+/// Future that generated by [`Operator::stat_with`].
+///
+/// Users can add more options by public functions provided by this struct.
+pub struct FutureStat(pub(crate) OperatorFuture<OpStat, Metadata>);
+
+impl FutureStat {
+    /// Set the If-Match for this operation.
+    pub fn if_match(mut self, v: &str) -> Self {
+        self.0 = self.0.map_args(|args| args.with_if_match(v));
+        self
+    }
+
+    /// Set the If-None-Match for this operation.
+    pub fn if_none_match(mut self, v: &str) -> Self {
+        self.0 = self.0.map_args(|args| args.with_if_none_match(v));
+        self
+    }
+}
+
+impl Future for FutureStat {
+    type Output = Result<Metadata>;
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        self.0.poll_unpin(cx)
+    }
+}
diff --git a/core/tests/behavior/read_only.rs b/core/tests/behavior/read_only.rs
index 9c1f60c6..da99322c 100644
--- a/core/tests/behavior/read_only.rs
+++ b/core/tests/behavior/read_only.rs
@@ -18,7 +18,6 @@
 use anyhow::Result;
 use futures::AsyncReadExt;
 use opendal::ops::OpRead;
-use opendal::ops::OpStat;
 use opendal::EntryMode;
 use opendal::ErrorKind;
 use opendal::Operator;
@@ -139,17 +138,14 @@ pub async fn test_stat_with_if_match(op: Operator) -> Result<()> {
     assert_eq!(meta.mode(), EntryMode::FILE);
     assert_eq!(meta.content_length(), 262144);
 
-    let mut op_stat = OpStat::default();
-    op_stat = op_stat.with_if_match("invalid_etag");
-
-    let res = op.stat_with(path, op_stat).await;
+    let res = op.stat_with(path).if_match("invalid_etag").await;
     assert!(res.is_err());
     assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
 
-    let mut op_stat = OpStat::default();
-    op_stat = op_stat.with_if_match(meta.etag().expect("etag must exist"));
-
-    let result = op.stat_with(path, op_stat).await;
+    let result = op
+        .stat_with(path)
+        .if_match(meta.etag().expect("etag must exist"))
+        .await;
     assert!(result.is_ok());
 
     Ok(())
@@ -167,17 +163,14 @@ pub async fn test_stat_with_if_none_match(op: Operator) -> Result<()> {
     assert_eq!(meta.mode(), EntryMode::FILE);
     assert_eq!(meta.content_length(), 262144);
 
-    let mut op_stat = OpStat::default();
-    op_stat = op_stat.with_if_none_match(meta.etag().expect("etag must exist"));
-
-    let res = op.stat_with(path, op_stat).await;
+    let res = op
+        .stat_with(path)
+        .if_none_match(meta.etag().expect("etag must exist"))
+        .await;
     assert!(res.is_err());
     assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
 
-    let mut op_stat = OpStat::default();
-    op_stat = op_stat.with_if_none_match("invalid_etag");
-
-    let res = op.stat_with(path, op_stat).await?;
+    let res = op.stat_with(path).if_none_match("invalid_etag").await?;
     assert_eq!(res.mode(), meta.mode());
     assert_eq!(res.content_length(), meta.content_length());
 
diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs
index 0da8a1d2..74b2366c 100644
--- a/core/tests/behavior/write.rs
+++ b/core/tests/behavior/write.rs
@@ -26,7 +26,6 @@ use http::StatusCode;
 use log::debug;
 use log::warn;
 use opendal::ops::OpRead;
-use opendal::ops::OpStat;
 use opendal::ops::OpWrite;
 use opendal::EntryMode;
 use opendal::ErrorKind;
@@ -373,17 +372,14 @@ pub async fn test_stat_with_if_match(op: Operator) -> Result<()> {
     assert_eq!(meta.mode(), EntryMode::FILE);
     assert_eq!(meta.content_length(), size as u64);
 
-    let mut op_stat = OpStat::default();
-    op_stat = op_stat.with_if_match("\"invalid_etag\"");
-
-    let res = op.stat_with(&path, op_stat).await;
+    let res = op.stat_with(&path).if_match("\"invalid_etag\"").await;
     assert!(res.is_err());
     assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
 
-    let mut op_stat = OpStat::default();
-    op_stat = op_stat.with_if_match(meta.etag().expect("etag must exist"));
-
-    let result = op.stat_with(&path, op_stat).await;
+    let result = op
+        .stat_with(&path)
+        .if_match(meta.etag().expect("etag must exist"))
+        .await;
     assert!(result.is_ok());
 
     op.delete(&path).await.expect("delete must succeed");
@@ -408,17 +404,17 @@ pub async fn test_stat_with_if_none_match(op: Operator) -> Result<()> {
     assert_eq!(meta.mode(), EntryMode::FILE);
     assert_eq!(meta.content_length(), size as u64);
 
-    let mut op_stat = OpStat::default();
-    op_stat = op_stat.with_if_none_match(meta.etag().expect("etag must exist"));
-
-    let res = op.stat_with(&path, op_stat).await;
+    let res = op
+        .stat_with(&path)
+        .if_none_match(meta.etag().expect("etag must exist"))
+        .await;
     assert!(res.is_err());
     assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
 
-    let mut op_stat = OpStat::default();
-    op_stat = op_stat.with_if_none_match("\"invalid_etag\"");
-
-    let res = op.stat_with(&path, op_stat).await?;
+    let res = op
+        .stat_with(&path)
+        .if_none_match("\"invalid_etag\"")
+        .await?;
     assert_eq!(res.mode(), meta.mode());
     assert_eq!(res.content_length(), meta.content_length());