You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/06/01 09:46:39 UTC
[incubator-opendal] 01/01: feat: Implement Timeout Layer
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch timeout-layer'
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit b17c8388079988aca40c25fd8ed34997e09885ac
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Thu Jun 1 17:46:24 2023 +0800
feat: Implement Timeout Layer
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/layers/mod.rs | 3 +
core/src/layers/timeout.rs | 402 +++++++++++++++++++++++++++++++++++++++++++
core/tests/behavior/utils.rs | 2 +
3 files changed, 407 insertions(+)
diff --git a/core/src/layers/mod.rs b/core/src/layers/mod.rs
index 9fde82096..dcf424d14 100644
--- a/core/src/layers/mod.rs
+++ b/core/src/layers/mod.rs
@@ -26,6 +26,9 @@ pub use immutable_index::ImmutableIndexLayer;
mod logging;
pub use logging::LoggingLayer;
+mod timeout;
+pub use timeout::TimeoutLayer;
+
#[cfg(feature = "layers-chaos")]
mod chaos;
#[cfg(feature = "layers-chaos")]
diff --git a/core/src/layers/timeout.rs b/core/src/layers/timeout.rs
new file mode 100644
index 000000000..c44862e72
--- /dev/null
+++ b/core/src/layers/timeout.rs
@@ -0,0 +1,402 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::oio::{AppendOperation, PageOperation, ReadOperation, WriteOperation};
+use crate::raw::*;
+use crate::*;
+use async_trait::async_trait;
+use bytes::Bytes;
+use std::io::SeekFrom;
+use std::task::{Context, Poll};
+use std::time::{Duration, Instant};
+
+/// Add timeout for every operations.
+///
+/// # Notes
+///
+/// - For io operatiosn like `read`, `write` and `append`, we will set a timeout
+/// for each single IO operation.
+/// - For other operations like `stat`, `delete`, the timeout is for the whole
+/// operation.
+///
+/// Besides, we will also set a slow speed for each IO operation. If the IO
+/// operation's speed is lower than the slow speed, we will return a timeout error
+/// instead of kept waiting for it.
+///
+/// For examples, if we set timeout to 60 seconds and speed to 1MiB/s, then:
+///
+/// - If `stat` didn't return in 60 seconds, we will return a timeout error.
+/// - If `Reader::read` didn't return in 60 seconds, we will return a timeout error.
+/// - For `Writer::write(vec![1024*1024*1024])`
+/// - didn't return in 60s, it's ok, we will keep waiting.
+/// - didn't return in 1024s (1GiB/1MiB), we will return a timeout error.
+///
+/// # Default
+///
+/// - timeout: 60 seconds
+/// - speed: 1024 bytes per second, aka, 1KiB/s.
+///
+/// # Examples
+///
+/// ```
+/// use anyhow::Result;
+/// use opendal::layers::TimeoutLayer;
+/// use opendal::services;
+/// use opendal::Operator;
+/// use opendal::Scheme;
+///
+/// let _ = Operator::new(services::Memory::default())
+/// .expect("must init")
+/// .layer(TimeoutLayer::default())
+/// .finish();
+/// ```
+#[derive(Clone)]
+pub struct TimeoutLayer {
+ timeout: Duration,
+ speed: u64,
+}
+
+impl Default for TimeoutLayer {
+ fn default() -> Self {
+ Self {
+ timeout: Duration::from_secs(60),
+ speed: 1024,
+ }
+ }
+}
+
+impl TimeoutLayer {
+ /// Create a new `TimeoutLayer` with default settings.
+ ///
+ /// - timeout: 60 seconds
+ /// - speed: 1024 bytes per second, aka, 1KiB/s.
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// Set timeout for TimeoutLayer with given value.
+ pub fn with_timeout(mut self, timeout: Duration) -> Self {
+ self.timeout = timeout;
+ self
+ }
+
+ /// Set speed for TimeoutLayer with given value.
+ ///
+ /// # Panics
+ ///
+ /// This function will panic if speed is 0.
+ pub fn with_speed(mut self, speed: u64) -> Self {
+ assert_ne!(speed, 0, "TimeoutLayer speed must not be 0");
+
+ self.speed = speed;
+ self
+ }
+}
+
+impl<A: Accessor> Layer<A> for TimeoutLayer {
+ type LayeredAccessor = TimeoutAccessor<A>;
+
+ fn layer(&self, inner: A) -> Self::LayeredAccessor {
+ TimeoutAccessor {
+ inner,
+
+ timeout: self.timeout,
+ speed: self.speed,
+ }
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct TimeoutAccessor<A: Accessor> {
+ inner: A,
+
+ timeout: Duration,
+ speed: u64,
+}
+
+#[async_trait]
+impl<A: Accessor> LayeredAccessor for TimeoutAccessor<A> {
+ type Inner = A;
+ type Reader = TimeoutWrapper<A::Reader>;
+ type BlockingReader = A::BlockingReader;
+ type Writer = TimeoutWrapper<A::Writer>;
+ type BlockingWriter = A::BlockingWriter;
+ type Appender = TimeoutWrapper<A::Appender>;
+ type Pager = TimeoutWrapper<A::Pager>;
+ type BlockingPager = A::BlockingPager;
+
+ fn inner(&self) -> &Self::Inner {
+ &self.inner
+ }
+
+ async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+ tokio::time::timeout(self.timeout, self.inner.read(path, args))
+ .await
+ .map_err(|_| {
+ Error::new(ErrorKind::Unexpected, "operation timeout")
+ .with_operation(Operation::Read)
+ .with_context("timeout", self.timeout.as_secs_f64().to_string())
+ .set_temporary()
+ })?
+ .map(|(rp, r)| (rp, TimeoutWrapper::new(r, self.timeout, self.speed)))
+ }
+
+ async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+ tokio::time::timeout(self.timeout, self.inner.write(path, args))
+ .await
+ .map_err(|_| {
+ Error::new(ErrorKind::Unexpected, "operation timeout")
+ .with_operation(Operation::Write)
+ .with_context("timeout", self.timeout.as_secs_f64().to_string())
+ .set_temporary()
+ })?
+ .map(|(rp, r)| (rp, TimeoutWrapper::new(r, self.timeout, self.speed)))
+ }
+
+ async fn append(&self, path: &str, args: OpAppend) -> Result<(RpAppend, Self::Appender)> {
+ tokio::time::timeout(self.timeout, self.inner.append(path, args))
+ .await
+ .map_err(|_| {
+ Error::new(ErrorKind::Unexpected, "operation timeout")
+ .with_operation(Operation::Append)
+ .with_context("timeout", self.timeout.as_secs_f64().to_string())
+ .set_temporary()
+ })?
+ .map(|(rp, r)| (rp, TimeoutWrapper::new(r, self.timeout, self.speed)))
+ }
+
+ async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {
+ tokio::time::timeout(self.timeout, self.inner.list(path, args))
+ .await
+ .map_err(|_| {
+ Error::new(ErrorKind::Unexpected, "operation timeout")
+ .with_operation(Operation::List)
+ .with_context("timeout", self.timeout.as_secs_f64().to_string())
+ .set_temporary()
+ })?
+ .map(|(rp, r)| (rp, TimeoutWrapper::new(r, self.timeout, self.speed)))
+ }
+
+ fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
+ self.inner.blocking_read(path, args)
+ }
+
+ fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
+ self.inner.blocking_write(path, args)
+ }
+
+ fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingPager)> {
+ self.inner.blocking_list(path, args)
+ }
+}
+
+pub struct TimeoutWrapper<R> {
+ inner: R,
+
+ timeout: Duration,
+ speed: u64,
+
+ start: Option<Instant>,
+}
+
+impl<R> TimeoutWrapper<R> {
+ fn new(inner: R, timeout: Duration, speed: u64) -> Self {
+ Self {
+ inner,
+ timeout,
+ speed,
+ start: None,
+ }
+ }
+
+ fn io_timeout(&self, size: u64) -> Duration {
+ let timeout = Duration::from_millis(size * 1000 / self.speed + 1);
+
+ timeout.max(self.timeout)
+ }
+}
+
+impl<R: oio::Read> oio::Read for TimeoutWrapper<R> {
+ fn poll_read(&mut self, cx: &mut Context<'_>, buf: &mut [u8]) -> Poll<Result<usize>> {
+ match self.start {
+ Some(start) => {
+ if start.elapsed() > self.timeout {
+ // Clean up the start time before return ready.
+ self.start = None;
+
+ return Poll::Ready(Err(Error::new(
+ ErrorKind::Unexpected,
+ "operation timeout",
+ )
+ .with_operation(ReadOperation::Read)
+ .with_context("timeout", self.timeout.as_secs_f64().to_string())
+ .set_temporary()));
+ }
+ }
+ None => {
+ self.start = Some(Instant::now());
+ }
+ }
+
+ match self.inner.poll_read(cx, buf) {
+ Poll::Pending => Poll::Pending,
+ Poll::Ready(v) => {
+ self.start = None;
+ Poll::Ready(v)
+ }
+ }
+ }
+
+ fn poll_seek(&mut self, cx: &mut Context<'_>, pos: SeekFrom) -> Poll<Result<u64>> {
+ match self.start {
+ Some(start) => {
+ if start.elapsed() > self.timeout {
+ // Clean up the start time before return ready.
+ self.start = None;
+
+ return Poll::Ready(Err(Error::new(
+ ErrorKind::Unexpected,
+ "operation timeout",
+ )
+ .with_operation(ReadOperation::Seek)
+ .with_context("timeout", self.timeout.as_secs_f64().to_string())
+ .set_temporary()));
+ }
+ }
+ None => {
+ self.start = Some(Instant::now());
+ }
+ }
+
+ match self.inner.poll_seek(cx, pos) {
+ Poll::Pending => Poll::Pending,
+ Poll::Ready(v) => {
+ self.start = None;
+ Poll::Ready(v)
+ }
+ }
+ }
+
+ fn poll_next(&mut self, cx: &mut Context<'_>) -> Poll<Option<Result<Bytes>>> {
+ match self.start {
+ Some(start) => {
+ if start.elapsed() > self.timeout {
+ // Clean up the start time before return ready.
+ self.start = None;
+
+ return Poll::Ready(Some(Err(Error::new(
+ ErrorKind::Unexpected,
+ "operation timeout",
+ )
+ .with_operation(ReadOperation::Next)
+ .with_context("timeout", self.timeout.as_secs_f64().to_string())
+ .set_temporary())));
+ }
+ }
+ None => {
+ self.start = Some(Instant::now());
+ }
+ }
+
+ match self.inner.poll_next(cx) {
+ Poll::Pending => Poll::Pending,
+ Poll::Ready(v) => {
+ self.start = None;
+ Poll::Ready(v)
+ }
+ }
+ }
+}
+
+#[async_trait]
+impl<R: oio::Write> oio::Write for TimeoutWrapper<R> {
+ async fn write(&mut self, bs: Bytes) -> Result<()> {
+ let timeout = self.io_timeout(bs.len() as u64);
+
+ tokio::time::timeout(timeout, self.inner.write(bs))
+ .await
+ .map_err(|_| {
+ Error::new(ErrorKind::Unexpected, "operation timeout")
+ .with_operation(WriteOperation::Write)
+ .with_context("timeout", timeout.as_secs_f64().to_string())
+ .set_temporary()
+ })?
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ tokio::time::timeout(self.timeout, self.inner.abort())
+ .await
+ .map_err(|_| {
+ Error::new(ErrorKind::Unexpected, "operation timeout")
+ .with_operation(WriteOperation::Abort)
+ .with_context("timeout", self.timeout.as_secs_f64().to_string())
+ .set_temporary()
+ })?
+ }
+
+ async fn close(&mut self) -> Result<()> {
+ tokio::time::timeout(self.timeout, self.inner.close())
+ .await
+ .map_err(|_| {
+ Error::new(ErrorKind::Unexpected, "operation timeout")
+ .with_operation(WriteOperation::Close)
+ .with_context("timeout", self.timeout.as_secs_f64().to_string())
+ .set_temporary()
+ })?
+ }
+}
+
+#[async_trait]
+impl<A: oio::Append> oio::Append for TimeoutWrapper<A> {
+ async fn append(&mut self, bs: Bytes) -> Result<()> {
+ let timeout = self.io_timeout(bs.len() as u64);
+
+ tokio::time::timeout(timeout, self.inner.append(bs))
+ .await
+ .map_err(|_| {
+ Error::new(ErrorKind::Unexpected, "operation timeout")
+ .with_operation(AppendOperation::Append)
+ .with_context("timeout", timeout.as_secs_f64().to_string())
+ .set_temporary()
+ })?
+ }
+
+ async fn close(&mut self) -> Result<()> {
+ tokio::time::timeout(self.timeout, self.inner.close())
+ .await
+ .map_err(|_| {
+ Error::new(ErrorKind::Unexpected, "operation timeout")
+ .with_operation(AppendOperation::Close)
+ .with_context("timeout", self.timeout.as_secs_f64().to_string())
+ .set_temporary()
+ })?
+ }
+}
+
+#[async_trait]
+impl<R: oio::Page> oio::Page for TimeoutWrapper<R> {
+ async fn next(&mut self) -> Result<Option<Vec<oio::Entry>>> {
+ tokio::time::timeout(self.timeout, self.inner.next())
+ .await
+ .map_err(|_| {
+ Error::new(ErrorKind::Unexpected, "operation timeout")
+ .with_operation(PageOperation::Next)
+ .with_context("timeout", self.timeout.as_secs_f64().to_string())
+ .set_temporary()
+ })?
+ }
+}
diff --git a/core/tests/behavior/utils.rs b/core/tests/behavior/utils.rs
index 8f30bbdc5..9c2543df9 100644
--- a/core/tests/behavior/utils.rs
+++ b/core/tests/behavior/utils.rs
@@ -28,6 +28,7 @@ use log::debug;
use log::warn;
use opendal::layers::LoggingLayer;
use opendal::layers::RetryLayer;
+use opendal::layers::TimeoutLayer;
use opendal::*;
use rand::distributions::uniform::SampleRange;
use rand::prelude::*;
@@ -84,6 +85,7 @@ pub fn init_service<B: Builder>() -> Option<Operator> {
let op = op
.layer(LoggingLayer::default())
+ .layer(TimeoutLayer::new())
.layer(RetryLayer::new())
.finish();