You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/06/01 09:46:39 UTC

[incubator-opendal] 01/01: feat: Implement Timeout Layer

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch timeout-layer'
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git

commit b17c8388079988aca40c25fd8ed34997e09885ac
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Thu Jun 1 17:46:24 2023 +0800

    feat: Implement Timeout Layer
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 core/src/layers/mod.rs       |   3 +
 core/src/layers/timeout.rs   | 402 +++++++++++++++++++++++++++++++++++++++++++
 core/tests/behavior/utils.rs |   2 +
 3 files changed, 407 insertions(+)

diff --git a/core/src/layers/mod.rs b/core/src/layers/mod.rs
index 9fde82096..dcf424d14 100644
--- a/core/src/layers/mod.rs
+++ b/core/src/layers/mod.rs
@@ -26,6 +26,9 @@ pub use immutable_index::ImmutableIndexLayer;
 mod logging;
 pub use logging::LoggingLayer;
 
+mod timeout;
+pub use timeout::TimeoutLayer;
+
 #[cfg(feature = "layers-chaos")]
 mod chaos;
 #[cfg(feature = "layers-chaos")]
diff --git a/core/src/layers/timeout.rs b/core/src/layers/timeout.rs
new file mode 100644
index 000000000..c44862e72
--- /dev/null
+++ b/core/src/layers/timeout.rs
@@ -0,0 +1,402 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::oio::{AppendOperation, PageOperation, ReadOperation, WriteOperation};
+use crate::raw::*;
+use crate::*;
+use async_trait::async_trait;
+use bytes::Bytes;
+use std::io::SeekFrom;
+use std::task::{Context, Poll};
+use std::time::{Duration, Instant};
+
+/// Add timeout for every operations.
+///
+/// # Notes
+///
+/// - For io operatiosn like `read`, `write` and `append`, we will set a timeout
+///   for each single IO operation.
+/// - For other operations like `stat`, `delete`, the timeout is for the whole
+///   operation.
+///
+/// Besides, we will also set a slow speed for each IO operation. If the IO
+/// operation's speed is lower than the slow speed, we will return a timeout error
+/// instead of kept waiting for it.
+///
+/// For examples, if we set timeout to 60 seconds and speed to 1MiB/s, then:
+///
+/// - If `stat` didn't return in 60 seconds, we will return a timeout error.
+/// - If `Reader::read` didn't return in 60 seconds, we will return a timeout error.
+/// - For `Writer::write(vec![1024*1024*1024])`
+///   - didn't return in 60s, it's ok, we will keep waiting.
+///   - didn't return in 1024s (1GiB/1MiB), we will return a timeout error.
+///
+/// # Default
+///
+/// - timeout: 60 seconds
+/// - speed: 1024 bytes per second, aka, 1KiB/s.
+///
+/// # Examples
+///
+/// ```
+/// use anyhow::Result;
+/// use opendal::layers::TimeoutLayer;
+/// use opendal::services;
+/// use opendal::Operator;
+/// use opendal::Scheme;
+///
+/// let _ = Operator::new(services::Memory::default())
+///     .expect("must init")
+///     .layer(TimeoutLayer::default())
+///     .finish();
+/// ```
+#[derive(Clone)]
+pub struct TimeoutLayer {
+    timeout: Duration,
+    speed: u64,
+}
+
+impl Default for TimeoutLayer {
+    fn default() -> Self {
+        Self {
+            timeout: Duration::from_secs(60),
+            speed: 1024,
+        }
+    }
+}
+
+impl TimeoutLayer {
+    /// Create a new `TimeoutLayer` with default settings.
+    ///
+    /// - timeout: 60 seconds
+    /// - speed: 1024 bytes per second, aka, 1KiB/s.
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Set timeout for TimeoutLayer with given value.
+    pub fn with_timeout(mut self, timeout: Duration) -> Self {
+        self.timeout = timeout;
+        self
+    }
+
+    /// Set speed for TimeoutLayer with given value.
+    ///
+    /// # Panics
+    ///
+    /// This function will panic if speed is 0.
+    pub fn with_speed(mut self, speed: u64) -> Self {
+        assert_ne!(speed, 0, "TimeoutLayer speed must not be 0");
+
+        self.speed = speed;
+        self
+    }
+}
+
+impl<A: Accessor> Layer<A> for TimeoutLayer {
+    type LayeredAccessor = TimeoutAccessor<A>;
+
+    fn layer(&self, inner: A) -> Self::LayeredAccessor {
+        TimeoutAccessor {
+            inner,
+
+            timeout: self.timeout,
+            speed: self.speed,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct TimeoutAccessor<A: Accessor> {
+    inner: A,
+
+    timeout: Duration,
+    speed: u64,
+}
+
+#[async_trait]
+impl<A: Accessor> LayeredAccessor for TimeoutAccessor<A> {
+    type Inner = A;
+    type Reader = TimeoutWrapper<A::Reader>;
+    type BlockingReader = A::BlockingReader;
+    type Writer = TimeoutWrapper<A::Writer>;
+    type BlockingWriter = A::BlockingWriter;
+    type Appender = TimeoutWrapper<A::Appender>;
+    type Pager = TimeoutWrapper<A::Pager>;
+    type BlockingPager = A::BlockingPager;
+
+    fn inner(&self) -> &Self::Inner {
+        &self.inner
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        tokio::time::timeout(self.timeout, self.inner.read(path, args))
+            .await
+            .map_err(|_| {
+                Error::new(ErrorKind::Unexpected, "operation timeout")
+                    .with_operation(Operation::Read)
+                    .with_context("timeout", self.timeout.as_secs_f64().to_string())
+                    .set_temporary()
+            })?
+            .map(|(rp, r)| (rp, TimeoutWrapper::new(r, self.timeout, self.speed)))
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        tokio::time::timeout(self.timeout, self.inner.write(path, args))
+            .await
+            .map_err(|_| {
+                Error::new(ErrorKind::Unexpected, "operation timeout")
+                    .with_operation(Operation::Write)
+                    .with_context("timeout", self.timeout.as_secs_f64().to_string())
+                    .set_temporary()
+            })?
+            .map(|(rp, r)| (rp, TimeoutWrapper::new(r, self.timeout, self.speed)))
+    }
+
+    async fn append(&self, path: &str, args: OpAppend) -> Result<(RpAppend, Self::Appender)> {
+        tokio::time::timeout(self.timeout, self.inner.append(path, args))
+            .await
+            .map_err(|_| {
+                Error::new(ErrorKind::Unexpected, "operation timeout")
+                    .with_operation(Operation::Append)
+                    .with_context("timeout", self.timeout.as_secs_f64().to_string())
+                    .set_temporary()
+            })?
+            .map(|(rp, r)| (rp, TimeoutWrapper::new(r, self.timeout, self.speed)))
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {
+        tokio::time::timeout(self.timeout, self.inner.list(path, args))
+            .await
+            .map_err(|_| {
+                Error::new(ErrorKind::Unexpected, "operation timeout")
+                    .with_operation(Operation::List)
+                    .with_context("timeout", self.timeout.as_secs_f64().to_string())
+                    .set_temporary()
+            })?
+            .map(|(rp, r)| (rp, TimeoutWrapper::new(r, self.timeout, self.speed)))
+    }
+
+    fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
+        self.inner.blocking_read(path, args)
+    }
+
+    fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
+        self.inner.blocking_write(path, args)
+    }
+
+    fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingPager)> {
+        self.inner.blocking_list(path, args)
+    }
+}
+
+pub struct TimeoutWrapper<R> {
+    inner: R,
+
+    timeout: Duration,
+    speed: u64,
+
+    start: Option<Instant>,
+}
+
+impl<R> TimeoutWrapper<R> {
+    fn new(inner: R, timeout: Duration, speed: u64) -> Self {
+        Self {
+            inner,
+            timeout,
+            speed,
+            start: None,
+        }
+    }
+
+    fn io_timeout(&self, size: u64) -> Duration {
+        let timeout = Duration::from_millis(size * 1000 / self.speed + 1);
+
+        timeout.max(self.timeout)
+    }
+}
+
+impl<R: oio::Read> oio::Read for TimeoutWrapper<R> {
+    fn poll_read(&mut self, cx: &mut Context<'_>, buf: &mut [u8]) -> Poll<Result<usize>> {
+        match self.start {
+            Some(start) => {
+                if start.elapsed() > self.timeout {
+                    // Clean up the start time before return ready.
+                    self.start = None;
+
+                    return Poll::Ready(Err(Error::new(
+                        ErrorKind::Unexpected,
+                        "operation timeout",
+                    )
+                    .with_operation(ReadOperation::Read)
+                    .with_context("timeout", self.timeout.as_secs_f64().to_string())
+                    .set_temporary()));
+                }
+            }
+            None => {
+                self.start = Some(Instant::now());
+            }
+        }
+
+        match self.inner.poll_read(cx, buf) {
+            Poll::Pending => Poll::Pending,
+            Poll::Ready(v) => {
+                self.start = None;
+                Poll::Ready(v)
+            }
+        }
+    }
+
+    fn poll_seek(&mut self, cx: &mut Context<'_>, pos: SeekFrom) -> Poll<Result<u64>> {
+        match self.start {
+            Some(start) => {
+                if start.elapsed() > self.timeout {
+                    // Clean up the start time before return ready.
+                    self.start = None;
+
+                    return Poll::Ready(Err(Error::new(
+                        ErrorKind::Unexpected,
+                        "operation timeout",
+                    )
+                    .with_operation(ReadOperation::Seek)
+                    .with_context("timeout", self.timeout.as_secs_f64().to_string())
+                    .set_temporary()));
+                }
+            }
+            None => {
+                self.start = Some(Instant::now());
+            }
+        }
+
+        match self.inner.poll_seek(cx, pos) {
+            Poll::Pending => Poll::Pending,
+            Poll::Ready(v) => {
+                self.start = None;
+                Poll::Ready(v)
+            }
+        }
+    }
+
+    fn poll_next(&mut self, cx: &mut Context<'_>) -> Poll<Option<Result<Bytes>>> {
+        match self.start {
+            Some(start) => {
+                if start.elapsed() > self.timeout {
+                    // Clean up the start time before return ready.
+                    self.start = None;
+
+                    return Poll::Ready(Some(Err(Error::new(
+                        ErrorKind::Unexpected,
+                        "operation timeout",
+                    )
+                    .with_operation(ReadOperation::Next)
+                    .with_context("timeout", self.timeout.as_secs_f64().to_string())
+                    .set_temporary())));
+                }
+            }
+            None => {
+                self.start = Some(Instant::now());
+            }
+        }
+
+        match self.inner.poll_next(cx) {
+            Poll::Pending => Poll::Pending,
+            Poll::Ready(v) => {
+                self.start = None;
+                Poll::Ready(v)
+            }
+        }
+    }
+}
+
+#[async_trait]
+impl<R: oio::Write> oio::Write for TimeoutWrapper<R> {
+    async fn write(&mut self, bs: Bytes) -> Result<()> {
+        let timeout = self.io_timeout(bs.len() as u64);
+
+        tokio::time::timeout(timeout, self.inner.write(bs))
+            .await
+            .map_err(|_| {
+                Error::new(ErrorKind::Unexpected, "operation timeout")
+                    .with_operation(WriteOperation::Write)
+                    .with_context("timeout", timeout.as_secs_f64().to_string())
+                    .set_temporary()
+            })?
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        tokio::time::timeout(self.timeout, self.inner.abort())
+            .await
+            .map_err(|_| {
+                Error::new(ErrorKind::Unexpected, "operation timeout")
+                    .with_operation(WriteOperation::Abort)
+                    .with_context("timeout", self.timeout.as_secs_f64().to_string())
+                    .set_temporary()
+            })?
+    }
+
+    async fn close(&mut self) -> Result<()> {
+        tokio::time::timeout(self.timeout, self.inner.close())
+            .await
+            .map_err(|_| {
+                Error::new(ErrorKind::Unexpected, "operation timeout")
+                    .with_operation(WriteOperation::Close)
+                    .with_context("timeout", self.timeout.as_secs_f64().to_string())
+                    .set_temporary()
+            })?
+    }
+}
+
+#[async_trait]
+impl<A: oio::Append> oio::Append for TimeoutWrapper<A> {
+    async fn append(&mut self, bs: Bytes) -> Result<()> {
+        let timeout = self.io_timeout(bs.len() as u64);
+
+        tokio::time::timeout(timeout, self.inner.append(bs))
+            .await
+            .map_err(|_| {
+                Error::new(ErrorKind::Unexpected, "operation timeout")
+                    .with_operation(AppendOperation::Append)
+                    .with_context("timeout", timeout.as_secs_f64().to_string())
+                    .set_temporary()
+            })?
+    }
+
+    async fn close(&mut self) -> Result<()> {
+        tokio::time::timeout(self.timeout, self.inner.close())
+            .await
+            .map_err(|_| {
+                Error::new(ErrorKind::Unexpected, "operation timeout")
+                    .with_operation(AppendOperation::Close)
+                    .with_context("timeout", self.timeout.as_secs_f64().to_string())
+                    .set_temporary()
+            })?
+    }
+}
+
+#[async_trait]
+impl<R: oio::Page> oio::Page for TimeoutWrapper<R> {
+    async fn next(&mut self) -> Result<Option<Vec<oio::Entry>>> {
+        tokio::time::timeout(self.timeout, self.inner.next())
+            .await
+            .map_err(|_| {
+                Error::new(ErrorKind::Unexpected, "operation timeout")
+                    .with_operation(PageOperation::Next)
+                    .with_context("timeout", self.timeout.as_secs_f64().to_string())
+                    .set_temporary()
+            })?
+    }
+}
diff --git a/core/tests/behavior/utils.rs b/core/tests/behavior/utils.rs
index 8f30bbdc5..9c2543df9 100644
--- a/core/tests/behavior/utils.rs
+++ b/core/tests/behavior/utils.rs
@@ -28,6 +28,7 @@ use log::debug;
 use log::warn;
 use opendal::layers::LoggingLayer;
 use opendal::layers::RetryLayer;
+use opendal::layers::TimeoutLayer;
 use opendal::*;
 use rand::distributions::uniform::SampleRange;
 use rand::prelude::*;
@@ -84,6 +85,7 @@ pub fn init_service<B: Builder>() -> Option<Operator> {
 
     let op = op
         .layer(LoggingLayer::default())
+        .layer(TimeoutLayer::new())
         .layer(RetryLayer::new())
         .finish();