You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/09/11 08:48:17 UTC
[incubator-opendal] branch poll-write created (now b24330c4d)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a change to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
at b24330c4d Polish logging
This branch includes the following new commits:
new e1d7ce09c Save work
new 91a30b252 Save work
new 89d5bed9b Save work
new 0dcbc79a4 Save work
new 5d78f4f65 save work
new cd167a5b0 Fix writer
new a20afd7b0 Fix build
new 34fa1e9b7 save work
new 977440d1a cleanup gcs
new cdc4c8771 Fix build
new c7c4819c2 Fix build for hdfs
new 2b75cf72f Fix ipmfs
new 8ad74fb8c Fix sftp
new a2575f825 Fix build
new 25cb7b8af Fix build for wasabi
new a5e534617 Save work
new 7bc8f73da Save
new f6a4188a5 Save
new fdd7bb002 Save
new 919d554ce Build pass
new 1f60e8410 Improve writer
new 77f36bba2 Fix test
new 90292ce9d Fix azblob
new 9b2534b2f Refactor gcs
new f46df2490 Make clippy happy
new c5fed2524 Make clippy happy
new 19aff6cda refactor
new db50a60f3 Fix build
new 42b2a9c66 Cleanup
new ace5c58b0 Fix fs abort
new b24330c4d Polish logging
The 31 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-opendal] 18/31: Save
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit f6a4188a5482754068950acbe9b230c20b0cf5f2
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 17:00:56 2023 +0800
Save
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/layers/complete.rs | 22 ++--
core/src/layers/error_context.rs | 6 +-
core/src/layers/logging.rs | 20 ++--
core/src/layers/madsim.rs | 16 +--
core/src/layers/metrics.rs | 9 +-
core/src/layers/minitrace.rs | 30 ++----
core/src/layers/prometheus.rs | 9 +-
core/src/layers/retry.rs | 128 +++++++++++++++++------
core/src/layers/throttle.rs | 14 +--
core/src/layers/timeout.rs | 95 +++++++++++++----
core/src/layers/tracing.rs | 6 +-
core/src/raw/adapters/kv/backend.rs | 5 +
core/src/raw/adapters/typed_kv/backend.rs | 5 +
core/src/raw/oio/write/append_object_write.rs | 20 ++--
core/src/raw/oio/write/multipart_upload_write.rs | 5 +
core/src/raw/oio/write/one_shot_write.rs | 5 +
core/src/services/azdfs/backend.rs | 4 +-
core/src/services/azdfs/writer.rs | 25 ++---
core/src/services/cos/writer.rs | 7 +-
core/src/services/dropbox/backend.rs | 8 +-
core/src/services/fs/writer.rs | 5 +
core/src/services/ftp/writer.rs | 5 +
core/src/services/gdrive/backend.rs | 8 +-
core/src/services/ghac/writer.rs | 5 +
core/src/services/ipmfs/writer.rs | 10 +-
core/src/services/obs/writer.rs | 7 +-
core/src/services/onedrive/backend.rs | 4 +-
core/src/services/oss/writer.rs | 7 +-
core/src/services/s3/writer.rs | 7 +-
core/src/services/supabase/backend.rs | 4 +-
core/src/services/vercel_artifacts/backend.rs | 8 +-
core/src/services/wasabi/backend.rs | 4 +-
core/src/services/webdav/backend.rs | 7 +-
core/src/services/webhdfs/backend.rs | 4 +-
34 files changed, 334 insertions(+), 190 deletions(-)
diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs
index 3e9e2bba9..4b856e8a1 100644
--- a/core/src/layers/complete.rs
+++ b/core/src/layers/complete.rs
@@ -19,8 +19,8 @@ use std::fmt::Debug;
use std::fmt::Formatter;
use std::io;
use std::sync::Arc;
-use std::task::Context;
use std::task::Poll;
+use std::task::{ready, Context};
use async_trait::async_trait;
use bytes::Bytes;
@@ -715,22 +715,22 @@ where
let w = self.inner.as_mut().ok_or_else(|| {
Error::new(ErrorKind::Unexpected, "writer has been closed or aborted")
})?;
- let n = w.write(bs).await?;
+ let n = ready!(w.poll_write(cx, bs))?;
self.written += n as u64;
if let Some(size) = self.size {
if self.written > size {
- return Err(Error::new(
+ return Poll::Ready(Err(Error::new(
ErrorKind::ContentTruncated,
&format!(
"writer got too much data, expect: {size}, actual: {}",
self.written + n as u64
),
- ));
+ )));
}
}
- Ok(n)
+ Poll::Ready(Ok(n))
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
@@ -738,22 +738,22 @@ where
Error::new(ErrorKind::Unexpected, "writer has been closed or aborted")
})?;
- w.abort().await?;
+ ready!(w.poll_abort(cx))?;
self.inner = None;
- Ok(())
+ Poll::Ready(Ok(()))
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
if let Some(size) = self.size {
if self.written < size {
- return Err(Error::new(
+ return Poll::Ready(Err(Error::new(
ErrorKind::ContentIncomplete,
&format!(
"writer got too less data, expect: {size}, actual: {}",
self.written
),
- ));
+ )));
}
}
@@ -761,10 +761,10 @@ where
Error::new(ErrorKind::Unexpected, "writer has been closed or aborted")
})?;
- w.close().await?;
+ ready!(w.poll_close(cx))?;
self.inner = None;
- Ok(())
+ Poll::Ready(Ok(()))
}
}
diff --git a/core/src/layers/error_context.rs b/core/src/layers/error_context.rs
index 7ce05fc18..1fd5e608c 100644
--- a/core/src/layers/error_context.rs
+++ b/core/src/layers/error_context.rs
@@ -404,7 +404,7 @@ impl<T: oio::BlockingRead> oio::BlockingRead for ErrorContextWrapper<T> {
#[async_trait::async_trait]
impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- self.inner.write(bs).await.map_err(|err| {
+ self.inner.poll_write(cx, bs).map_err(|err| {
err.with_operation(WriteOperation::Write)
.with_context("service", self.scheme)
.with_context("path", &self.path)
@@ -412,7 +412,7 @@ impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.abort().await.map_err(|err| {
+ self.inner.poll_abort(cx).map_err(|err| {
err.with_operation(WriteOperation::Abort)
.with_context("service", self.scheme)
.with_context("path", &self.path)
@@ -420,7 +420,7 @@ impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.close().await.map_err(|err| {
+ self.inner.poll_close(cx).map_err(|err| {
err.with_operation(WriteOperation::Close)
.with_context("service", self.scheme)
.with_context("path", &self.path)
diff --git a/core/src/layers/logging.rs b/core/src/layers/logging.rs
index 7da2da55b..42084eede 100644
--- a/core/src/layers/logging.rs
+++ b/core/src/layers/logging.rs
@@ -17,8 +17,8 @@
use std::fmt::Debug;
use std::io;
-use std::task::Context;
use std::task::Poll;
+use std::task::{ready, Context};
use async_trait::async_trait;
use bytes::Bytes;
@@ -1253,7 +1253,7 @@ impl<W> LoggingWriter<W> {
#[async_trait]
impl<W: oio::Write> oio::Write for LoggingWriter<W> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- match self.inner.write(bs).await {
+ match ready!(self.inner.poll_write(cx, bs)) {
Ok(n) => {
self.written += n as u64;
trace!(
@@ -1265,7 +1265,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
self.written,
n
);
- Ok(n)
+ Poll::Ready(Ok(n))
}
Err(err) => {
if let Some(lvl) = self.ctx.error_level(&err) {
@@ -1280,13 +1280,13 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
self.ctx.error_print(&err),
)
}
- Err(err)
+ Poll::Ready(Err(err))
}
}
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- match self.inner.abort().await {
+ match ready!(self.inner.poll_abort(cx)) {
Ok(_) => {
trace!(
target: LOGGING_TARGET,
@@ -1296,7 +1296,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
self.path,
self.written,
);
- Ok(())
+ Poll::Ready(Ok(()))
}
Err(err) => {
if let Some(lvl) = self.ctx.error_level(&err) {
@@ -1311,13 +1311,13 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
self.ctx.error_print(&err),
)
}
- Err(err)
+ Poll::Ready(Err(err))
}
}
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- match self.inner.close().await {
+ match ready!(self.inner.poll_close(cx)) {
Ok(_) => {
debug!(
target: LOGGING_TARGET,
@@ -1327,7 +1327,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
self.path,
self.written
);
- Ok(())
+ Poll::Ready(Ok(()))
}
Err(err) => {
if let Some(lvl) = self.ctx.error_level(&err) {
@@ -1342,7 +1342,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
self.ctx.error_print(&err),
)
}
- Err(err)
+ Poll::Ready(Err(err))
}
}
}
diff --git a/core/src/layers/madsim.rs b/core/src/layers/madsim.rs
index f58eb1997..3c107fd83 100644
--- a/core/src/layers/madsim.rs
+++ b/core/src/layers/madsim.rs
@@ -302,7 +302,11 @@ pub struct MadsimWriter {
#[async_trait]
impl oio::Write for MadsimWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> crate::Result<usize> {
+ fn poll_write(
+ &mut self,
+ cx: &mut Context<'_>,
+ bs: &dyn oio::WriteBuf,
+ ) -> Poll<crate::Result<usize>> {
#[cfg(madsim)]
{
let req = Request::Write(self.path.to_string(), bs);
@@ -318,15 +322,15 @@ impl oio::Write for MadsimWriter {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> crate::Result<()> {
- Err(Error::new(
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<crate::Result<()>> {
+ Poll::Ready(Err(Error::new(
ErrorKind::Unsupported,
"will be supported in the future",
- ))
+ )))
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> crate::Result<()> {
- Ok(())
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<crate::Result<()>> {
+ Poll::Ready(Ok(()))
}
}
diff --git a/core/src/layers/metrics.rs b/core/src/layers/metrics.rs
index 3cebf46b1..bcbb72131 100644
--- a/core/src/layers/metrics.rs
+++ b/core/src/layers/metrics.rs
@@ -849,9 +849,8 @@ impl<R: oio::BlockingRead> oio::BlockingRead for MetricWrapper<R> {
impl<R: oio::Write> oio::Write for MetricWrapper<R> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
self.inner
- .write(bs)
- .await
- .map(|n| {
+ .poll_write(cx, bs)
+ .map_ok(|n| {
self.bytes += n as u64;
n
})
@@ -862,14 +861,14 @@ impl<R: oio::Write> oio::Write for MetricWrapper<R> {
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.abort().await.map_err(|err| {
+ self.inner.poll_abort(cx).map_err(|err| {
self.handle.increment_errors_total(self.op, err.kind());
err
})
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.close().await.map_err(|err| {
+ self.inner.poll_close(cx).map_err(|err| {
self.handle.increment_errors_total(self.op, err.kind());
err
})
diff --git a/core/src/layers/minitrace.rs b/core/src/layers/minitrace.rs
index 7c6fad770..00e1239d2 100644
--- a/core/src/layers/minitrace.rs
+++ b/core/src/layers/minitrace.rs
@@ -338,33 +338,21 @@ impl<R: oio::BlockingRead> oio::BlockingRead for MinitraceWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for MinitraceWrapper<R> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- self.inner
- .write(bs)
- .in_span(Span::enter_with_parent(
- WriteOperation::Write.into_static(),
- &self.span,
- ))
- .await
+ let _g = self.span.set_local_parent();
+ let _span = LocalSpan::enter_with_local_parent(WriteOperation::Write.into_static());
+ self.inner.poll_write(cx, bs)
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner
- .abort()
- .in_span(Span::enter_with_parent(
- WriteOperation::Abort.into_static(),
- &self.span,
- ))
- .await
+ let _g = self.span.set_local_parent();
+ let _span = LocalSpan::enter_with_local_parent(WriteOperation::Abort.into_static());
+ self.inner.poll_abort(cx)
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner
- .close()
- .in_span(Span::enter_with_parent(
- WriteOperation::Close.into_static(),
- &self.span,
- ))
- .await
+ let _g = self.span.set_local_parent();
+ let _span = LocalSpan::enter_with_local_parent(WriteOperation::Close.into_static());
+ self.inner.poll_close(cx)
}
}
diff --git a/core/src/layers/prometheus.rs b/core/src/layers/prometheus.rs
index b3f704e6d..2558bb50b 100644
--- a/core/src/layers/prometheus.rs
+++ b/core/src/layers/prometheus.rs
@@ -664,9 +664,8 @@ impl<R: oio::BlockingRead> oio::BlockingRead for PrometheusMetricWrapper<R> {
impl<R: oio::Write> oio::Write for PrometheusMetricWrapper<R> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
self.inner
- .write(bs)
- .await
- .map(|n| {
+ .poll_write(cx, bs)
+ .map_ok(|n| {
self.stats
.bytes_total
.with_label_values(&[&self.scheme, Operation::Write.into_static()])
@@ -680,14 +679,14 @@ impl<R: oio::Write> oio::Write for PrometheusMetricWrapper<R> {
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.abort().await.map_err(|err| {
+ self.inner.poll_abort(cx).map_err(|err| {
self.stats.increment_errors_total(self.op, err.kind());
err
})
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.close().await.map_err(|err| {
+ self.inner.poll_close(cx).map_err(|err| {
self.stats.increment_errors_total(self.op, err.kind());
err
})
diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs
index 6a5dba57c..7b2549dc0 100644
--- a/core/src/layers/retry.rs
+++ b/core/src/layers/retry.rs
@@ -873,79 +873,139 @@ impl<R: oio::BlockingRead, I: RetryInterceptor> oio::BlockingRead for RetryWrapp
#[async_trait]
impl<R: oio::Write, I: RetryInterceptor> oio::Write for RetryWrapper<R, I> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let mut backoff = self.builder.build();
+ if let Some(sleep) = self.sleep.as_mut() {
+ ready!(sleep.poll_unpin(cx));
+ self.sleep = None;
+ }
- loop {
- match self.inner.write(bs).await {
- Ok(v) => return Ok(v),
- Err(e) if !e.is_temporary() => return Err(e),
- Err(e) => match backoff.next() {
- None => return Err(e),
+ match ready!(self.inner.poll_write(cx, bs)) {
+ Ok(v) => {
+ self.current_backoff = None;
+ Poll::Ready(Ok(v))
+ }
+ Err(err) if !err.is_temporary() => {
+ self.current_backoff = None;
+ Poll::Ready(Err(err))
+ }
+ Err(err) => {
+ let backoff = match self.current_backoff.as_mut() {
+ Some(backoff) => backoff,
+ None => {
+ self.current_backoff = Some(self.builder.build());
+ self.current_backoff.as_mut().unwrap()
+ }
+ };
+
+ match backoff.next() {
+ None => {
+ self.current_backoff = None;
+ Poll::Ready(Err(err))
+ }
Some(dur) => {
self.notify.intercept(
- &e,
+ &err,
dur,
&[
("operation", WriteOperation::Write.into_static()),
("path", &self.path),
],
);
- tokio::time::sleep(dur).await;
- continue;
+ self.sleep = Some(Box::pin(tokio::time::sleep(dur)));
+ self.poll_write(cx, bs)
}
- },
+ }
}
}
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- let mut backoff = self.builder.build();
+ if let Some(sleep) = self.sleep.as_mut() {
+ ready!(sleep.poll_unpin(cx));
+ self.sleep = None;
+ }
- loop {
- match self.inner.abort().await {
- Ok(v) => return Ok(v),
- Err(e) if !e.is_temporary() => return Err(e),
- Err(e) => match backoff.next() {
- None => return Err(e),
+ match ready!(self.inner.poll_abort(cx)) {
+ Ok(v) => {
+ self.current_backoff = None;
+ Poll::Ready(Ok(v))
+ }
+ Err(err) if !err.is_temporary() => {
+ self.current_backoff = None;
+ Poll::Ready(Err(err))
+ }
+ Err(err) => {
+ let backoff = match self.current_backoff.as_mut() {
+ Some(backoff) => backoff,
+ None => {
+ self.current_backoff = Some(self.builder.build());
+ self.current_backoff.as_mut().unwrap()
+ }
+ };
+
+ match backoff.next() {
+ None => {
+ self.current_backoff = None;
+ Poll::Ready(Err(err))
+ }
Some(dur) => {
self.notify.intercept(
- &e,
+ &err,
dur,
&[
("operation", WriteOperation::Abort.into_static()),
("path", &self.path),
],
);
- tokio::time::sleep(dur).await;
- continue;
+ self.sleep = Some(Box::pin(tokio::time::sleep(dur)));
+ self.poll_abort(cx)
}
- },
+ }
}
}
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- let mut backoff = self.builder.build();
+ if let Some(sleep) = self.sleep.as_mut() {
+ ready!(sleep.poll_unpin(cx));
+ self.sleep = None;
+ }
- loop {
- match self.inner.close().await {
- Ok(v) => return Ok(v),
- Err(e) if !e.is_temporary() => return Err(e),
- Err(e) => match backoff.next() {
- None => return Err(e),
+ match ready!(self.inner.poll_close(cx)) {
+ Ok(v) => {
+ self.current_backoff = None;
+ Poll::Ready(Ok(v))
+ }
+ Err(err) if !err.is_temporary() => {
+ self.current_backoff = None;
+ Poll::Ready(Err(err))
+ }
+ Err(err) => {
+ let backoff = match self.current_backoff.as_mut() {
+ Some(backoff) => backoff,
+ None => {
+ self.current_backoff = Some(self.builder.build());
+ self.current_backoff.as_mut().unwrap()
+ }
+ };
+
+ match backoff.next() {
+ None => {
+ self.current_backoff = None;
+ Poll::Ready(Err(err))
+ }
Some(dur) => {
self.notify.intercept(
- &e,
+ &err,
dur,
&[
- ("operation", WriteOperation::Close.into_static()),
+ ("operation", WriteOperation::Abort.into_static()),
("path", &self.path),
],
);
- tokio::time::sleep(dur).await;
- continue;
+ self.sleep = Some(Box::pin(tokio::time::sleep(dur)));
+ self.poll_close(cx)
}
- },
+ }
}
}
}
diff --git a/core/src/layers/throttle.rs b/core/src/layers/throttle.rs
index f731ab83c..72e63abe0 100644
--- a/core/src/layers/throttle.rs
+++ b/core/src/layers/throttle.rs
@@ -221,20 +221,22 @@ impl<R: oio::Write> oio::Write for ThrottleWrapper<R> {
loop {
match self.limiter.check_n(buf_length) {
- Ok(_) => return self.inner.write(bs).await,
+ Ok(_) => return self.inner.poll_write(cx, bs),
Err(negative) => match negative {
// the query is valid but the Decider can not accommodate them.
NegativeMultiDecision::BatchNonConforming(_, not_until) => {
let wait_time = not_until.wait_time_from(DefaultClock::default().now());
// TODO: Should lock the limiter and wait for the wait_time, or should let other small requests go first?
- tokio::time::sleep(wait_time).await;
+
+ // FIXME: we should sleep here.
+ // tokio::time::sleep(wait_time).await;
}
// the query was invalid as the rate limit parameters can "never" accommodate the number of cells queried for.
NegativeMultiDecision::InsufficientCapacity(_) => {
- return Err(Error::new(
+ return Poll::Ready(Err(Error::new(
ErrorKind::RateLimited,
"InsufficientCapacity due to burst size being smaller than the request size",
- ))
+ )))
}
},
}
@@ -242,11 +244,11 @@ impl<R: oio::Write> oio::Write for ThrottleWrapper<R> {
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.abort().await
+ self.inner.poll_abort(cx)
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.close().await
+ self.inner.poll_close(cx)
}
}
diff --git a/core/src/layers/timeout.rs b/core/src/layers/timeout.rs
index 5871818b3..d6cd4d24a 100644
--- a/core/src/layers/timeout.rs
+++ b/core/src/layers/timeout.rs
@@ -323,38 +323,93 @@ impl<R: oio::Read> oio::Read for TimeoutWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for TimeoutWrapper<R> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let timeout = self.io_timeout(bs.remaining() as u64);
+ match self.start {
+ Some(start) => {
+ if start.elapsed() > self.timeout {
+ // Clean up the start time before return ready.
+ self.start = None;
- tokio::time::timeout(timeout, self.inner.write(bs))
- .await
- .map_err(|_| {
- Error::new(ErrorKind::Unexpected, "operation timeout")
+ return Poll::Ready(Err(Error::new(
+ ErrorKind::Unexpected,
+ "operation timeout",
+ )
.with_operation(WriteOperation::Write)
- .with_context("timeout", timeout.as_secs_f64().to_string())
- .set_temporary()
- })?
+ .with_context("timeout", self.timeout.as_secs_f64().to_string())
+ .set_temporary()));
+ }
+ }
+ None => {
+ self.start = Some(Instant::now());
+ }
+ }
+
+ match self.inner.poll_write(cx, bs) {
+ Poll::Pending => Poll::Pending,
+ Poll::Ready(v) => {
+ self.start = None;
+ Poll::Ready(v)
+ }
+ }
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- tokio::time::timeout(self.timeout, self.inner.abort())
- .await
- .map_err(|_| {
- Error::new(ErrorKind::Unexpected, "operation timeout")
+ match self.start {
+ Some(start) => {
+ if start.elapsed() > self.timeout {
+ // Clean up the start time before return ready.
+ self.start = None;
+
+ return Poll::Ready(Err(Error::new(
+ ErrorKind::Unexpected,
+ "operation timeout",
+ )
.with_operation(WriteOperation::Abort)
.with_context("timeout", self.timeout.as_secs_f64().to_string())
- .set_temporary()
- })?
+ .set_temporary()));
+ }
+ }
+ None => {
+ self.start = Some(Instant::now());
+ }
+ }
+
+ match self.inner.poll_abort(cx) {
+ Poll::Pending => Poll::Pending,
+ Poll::Ready(v) => {
+ self.start = None;
+ Poll::Ready(v)
+ }
+ }
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- tokio::time::timeout(self.timeout, self.inner.close())
- .await
- .map_err(|_| {
- Error::new(ErrorKind::Unexpected, "operation timeout")
+ match self.start {
+ Some(start) => {
+ if start.elapsed() > self.timeout {
+ // Clean up the start time before return ready.
+ self.start = None;
+
+ return Poll::Ready(Err(Error::new(
+ ErrorKind::Unexpected,
+ "operation timeout",
+ )
.with_operation(WriteOperation::Close)
.with_context("timeout", self.timeout.as_secs_f64().to_string())
- .set_temporary()
- })?
+ .set_temporary()));
+ }
+ }
+ None => {
+ self.start = Some(Instant::now());
+ }
+ }
+
+ match self.inner.poll_close(cx) {
+ Poll::Pending => Poll::Pending,
+ Poll::Ready(v) => {
+ self.start = None;
+ Poll::Ready(v)
+ }
+ }
}
}
diff --git a/core/src/layers/tracing.rs b/core/src/layers/tracing.rs
index cb71388cb..9bb0b54c2 100644
--- a/core/src/layers/tracing.rs
+++ b/core/src/layers/tracing.rs
@@ -325,7 +325,7 @@ impl<R: oio::Write> oio::Write for TracingWrapper<R> {
level = "trace",
skip_all)]
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- self.inner.write(bs).await
+ self.inner.poll_write(cx, bs)
}
#[tracing::instrument(
@@ -333,7 +333,7 @@ impl<R: oio::Write> oio::Write for TracingWrapper<R> {
level = "trace",
skip_all)]
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.abort().await
+ self.inner.poll_abort(cx)
}
#[tracing::instrument(
@@ -341,7 +341,7 @@ impl<R: oio::Write> oio::Write for TracingWrapper<R> {
level = "trace",
skip_all)]
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.close().await
+ self.inner.poll_close(cx)
}
}
diff --git a/core/src/raw/adapters/kv/backend.rs b/core/src/raw/adapters/kv/backend.rs
index a3ead2e7a..48c327b0a 100644
--- a/core/src/raw/adapters/kv/backend.rs
+++ b/core/src/raw/adapters/kv/backend.rs
@@ -390,6 +390,11 @@ impl<S> KvWriter<S> {
}
}
+/// # Safety
+///
+/// We will only take `&mut Self` reference for KvWriter.
+unsafe impl<S: Adapter> Sync for KvWriter<S> {}
+
#[async_trait]
impl<S: Adapter> oio::Write for KvWriter<S> {
// TODO: we need to support append in the future.
diff --git a/core/src/raw/adapters/typed_kv/backend.rs b/core/src/raw/adapters/typed_kv/backend.rs
index 4a3cdf37b..fda56bf28 100644
--- a/core/src/raw/adapters/typed_kv/backend.rs
+++ b/core/src/raw/adapters/typed_kv/backend.rs
@@ -368,6 +368,11 @@ pub struct KvWriter<S> {
future: Option<BoxFuture<'static, Result<()>>>,
}
+/// # Safety
+///
+/// We will only take `&mut Self` reference for KvWriter.
+unsafe impl<S: Adapter> Sync for KvWriter<S> {}
+
impl<S> KvWriter<S> {
fn new(kv: Arc<S>, path: String, op: OpWrite) -> Self {
KvWriter {
diff --git a/core/src/raw/oio/write/append_object_write.rs b/core/src/raw/oio/write/append_object_write.rs
index 2c0bd5555..393254097 100644
--- a/core/src/raw/oio/write/append_object_write.rs
+++ b/core/src/raw/oio/write/append_object_write.rs
@@ -59,6 +59,11 @@ enum State<W> {
Append(BoxFuture<'static, (W, Result<usize>)>),
}
+/// # Safety
+///
+/// We will only take `&mut Self` reference for State.
+unsafe impl<S: AppendObjectWrite> Sync for State<S> {}
+
impl<W: AppendObjectWrite> AppendObjectWriter<W> {
/// Create a new AppendObjectWriter.
pub fn new(inner: W) -> Self {
@@ -67,17 +72,6 @@ impl<W: AppendObjectWrite> AppendObjectWriter<W> {
offset: None,
}
}
-
- async fn offset(&mut self) -> Result<u64> {
- if let Some(offset) = self.offset {
- return Ok(offset);
- }
-
- let offset = self.inner.offset().await?;
- self.offset = Some(offset);
-
- Ok(offset)
- }
}
#[async_trait]
@@ -103,9 +97,9 @@ where
}
None => {
self.state = State::Offset(Box::pin(async move {
- let offset = w.offset().await?;
+ let offset = w.offset().await;
- (w, Ok(offset))
+ (w, offset)
}));
}
}
diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs
index 482954dc9..97b2f0220 100644
--- a/core/src/raw/oio/write/multipart_upload_write.rs
+++ b/core/src/raw/oio/write/multipart_upload_write.rs
@@ -96,6 +96,11 @@ enum State<W> {
Abort(BoxFuture<'static, (W, Result<()>)>),
}
+/// # Safety
+///
+/// We will only take `&mut Self` reference for State.
+unsafe impl<S: MultipartUploadWrite> Sync for State<S> {}
+
impl<W: MultipartUploadWrite> MultipartUploadWriter<W> {
/// Create a new MultipartUploadWriter.
pub fn new(inner: W) -> Self {
diff --git a/core/src/raw/oio/write/one_shot_write.rs b/core/src/raw/oio/write/one_shot_write.rs
index a0a743915..6cc083491 100644
--- a/core/src/raw/oio/write/one_shot_write.rs
+++ b/core/src/raw/oio/write/one_shot_write.rs
@@ -47,6 +47,11 @@ enum State<W> {
Write(BoxFuture<'static, (W, Result<usize>)>),
}
+/// # Safety
+///
+/// We will only take `&mut Self` reference for State.
+unsafe impl<S: OneShotWrite> Sync for State<S> {}
+
impl<W: OneShotWrite> OneShotWriter<W> {
/// Create a new one shot writer.
pub fn new(inner: W) -> Self {
diff --git a/core/src/services/azdfs/backend.rs b/core/src/services/azdfs/backend.rs
index 3882f4fe2..1c0dd3a8c 100644
--- a/core/src/services/azdfs/backend.rs
+++ b/core/src/services/azdfs/backend.rs
@@ -230,7 +230,7 @@ pub struct AzdfsBackend {
impl Accessor for AzdfsBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = AzdfsWriter;
+ type Writer = oio::OneShotWriter<AzdfsWriter>;
type BlockingWriter = ();
type Pager = AzdfsPager;
type BlockingPager = ();
@@ -305,7 +305,7 @@ impl Accessor for AzdfsBackend {
Ok((
RpWrite::default(),
- AzdfsWriter::new(self.core.clone(), args, path.to_string()),
+ oio::OneShotWriter::new(AzdfsWriter::new(self.core.clone(), args, path.to_string())),
))
}
diff --git a/core/src/services/azdfs/writer.rs b/core/src/services/azdfs/writer.rs
index 774c9ee4b..1da698dd4 100644
--- a/core/src/services/azdfs/writer.rs
+++ b/core/src/services/azdfs/writer.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
use std::task::{Context, Poll};
use async_trait::async_trait;
+use bytes::Bytes;
use http::StatusCode;
use super::core::AzdfsCore;
@@ -40,8 +41,8 @@ impl AzdfsWriter {
}
#[async_trait]
-impl oio::Write for AzdfsWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
+impl oio::OneShotWrite for AzdfsWriter {
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
let mut req = self.core.azdfs_create_request(
&self.path,
"file",
@@ -66,13 +67,11 @@ impl oio::Write for AzdfsWriter {
}
}
- let size = bs.remaining();
+ let size = bs.len();
- let mut req = self.core.azdfs_update_request(
- &self.path,
- Some(size),
- AsyncBody::Bytes(bs.copy_to_bytes(size)),
- )?;
+ let mut req =
+ self.core
+ .azdfs_update_request(&self.path, Some(size), AsyncBody::Bytes(bs))?;
self.core.sign(&mut req).await?;
@@ -82,19 +81,11 @@ impl oio::Write for AzdfsWriter {
match status {
StatusCode::OK | StatusCode::ACCEPTED => {
resp.into_body().consume().await?;
- Ok(size)
+ Ok(())
}
_ => Err(parse_error(resp)
.await?
.with_operation("Backend::azdfs_update_request")),
}
}
-
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Poll::Ready(Ok(()))
- }
-
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Poll::Ready(Ok(()))
- }
}
diff --git a/core/src/services/cos/writer.rs b/core/src/services/cos/writer.rs
index e0cc8be73..fba7cea09 100644
--- a/core/src/services/cos/writer.rs
+++ b/core/src/services/cos/writer.rs
@@ -18,6 +18,7 @@
use std::sync::Arc;
use async_trait::async_trait;
+use bytes::Bytes;
use http::StatusCode;
use super::core::*;
@@ -50,15 +51,15 @@ impl CosWriter {
#[async_trait]
impl oio::OneShotWrite for CosWriter {
- async fn write_once(&self, buf: &dyn oio::WriteBuf) -> Result<()> {
- let size = buf.remaining();
+ async fn write_once(&self, buf: Bytes) -> Result<()> {
+ let size = buf.len();
let mut req = self.core.cos_put_object_request(
&self.path,
Some(size as u64),
self.op.content_type(),
self.op.content_disposition(),
self.op.cache_control(),
- AsyncBody::Bytes(buf.copy_to_bytes(size)),
+ AsyncBody::Bytes(buf),
)?;
self.core.sign(&mut req).await?;
diff --git a/core/src/services/dropbox/backend.rs b/core/src/services/dropbox/backend.rs
index ebb468728..029f1fe7f 100644
--- a/core/src/services/dropbox/backend.rs
+++ b/core/src/services/dropbox/backend.rs
@@ -49,7 +49,7 @@ pub struct DropboxBackend {
impl Accessor for DropboxBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = DropboxWriter;
+ type Writer = oio::OneShotWriter<DropboxWriter>;
type BlockingWriter = ();
type Pager = ();
type BlockingPager = ();
@@ -114,7 +114,11 @@ impl Accessor for DropboxBackend {
}
Ok((
RpWrite::default(),
- DropboxWriter::new(self.core.clone(), args, String::from(path)),
+ oio::OneShotWriter::new(DropboxWriter::new(
+ self.core.clone(),
+ args,
+ String::from(path),
+ )),
))
}
diff --git a/core/src/services/fs/writer.rs b/core/src/services/fs/writer.rs
index a71de17d5..1a0aa49eb 100644
--- a/core/src/services/fs/writer.rs
+++ b/core/src/services/fs/writer.rs
@@ -48,6 +48,11 @@ impl<F> FsWriter<F> {
}
}
+/// # Safety
+///
+/// We will only take `&mut Self` reference for FsWriter.
+unsafe impl<F> Sync for FsWriter<F> {}
+
#[async_trait]
impl oio::Write for FsWriter<tokio::fs::File> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
diff --git a/core/src/services/ftp/writer.rs b/core/src/services/ftp/writer.rs
index 34ed85b37..ec3c3f0e2 100644
--- a/core/src/services/ftp/writer.rs
+++ b/core/src/services/ftp/writer.rs
@@ -46,6 +46,11 @@ impl FtpWriter {
}
}
+/// # Safety
+///
+/// We will only take `&mut Self` reference for FtpWriter.
+unsafe impl Sync for FtpWriter {}
+
#[async_trait]
impl oio::Write for FtpWriter {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs
index df8ee7451..f50713ac7 100644
--- a/core/src/services/gdrive/backend.rs
+++ b/core/src/services/gdrive/backend.rs
@@ -40,7 +40,7 @@ pub struct GdriveBackend {
impl Accessor for GdriveBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = GdriveWriter;
+ type Writer = oio::OneShotWriter<GdriveWriter>;
type BlockingWriter = ();
type Pager = ();
type BlockingPager = ();
@@ -194,7 +194,11 @@ impl Accessor for GdriveBackend {
Ok((
RpWrite::default(),
- GdriveWriter::new(self.core.clone(), String::from(path), file_id),
+ oio::OneShotWriter::new(GdriveWriter::new(
+ self.core.clone(),
+ String::from(path),
+ file_id,
+ )),
))
}
diff --git a/core/src/services/ghac/writer.rs b/core/src/services/ghac/writer.rs
index 0be63ce22..c780c2710 100644
--- a/core/src/services/ghac/writer.rs
+++ b/core/src/services/ghac/writer.rs
@@ -47,6 +47,11 @@ enum State {
Commit(BoxFuture<'static, (GhacBackend, Result<()>)>),
}
+/// # Safety
+///
+/// We will only take `&mut Self` reference for State.
+unsafe impl Sync for State {}
+
#[async_trait]
impl oio::Write for GhacWriter {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
diff --git a/core/src/services/ipmfs/writer.rs b/core/src/services/ipmfs/writer.rs
index 0d1673693..feb5e5630 100644
--- a/core/src/services/ipmfs/writer.rs
+++ b/core/src/services/ipmfs/writer.rs
@@ -16,6 +16,7 @@
// under the License.
use async_trait::async_trait;
+use bytes::Bytes;
use http::StatusCode;
use std::task::{Context, Poll};
@@ -39,12 +40,9 @@ impl IpmfsWriter {
#[async_trait]
impl oio::OneShotWrite for IpmfsWriter {
- async fn write_once(&self, bs: &dyn WriteBuf) -> Result<()> {
- let size = bs.remaining();
- let resp = self
- .backend
- .ipmfs_write(&self.path, bs.copy_to_bytes(size))
- .await?;
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
+ let size = bs.len();
+ let resp = self.backend.ipmfs_write(&self.path, bs).await?;
let status = resp.status();
diff --git a/core/src/services/obs/writer.rs b/core/src/services/obs/writer.rs
index 38dc660db..6b0aed175 100644
--- a/core/src/services/obs/writer.rs
+++ b/core/src/services/obs/writer.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use bytes::Bytes;
use std::sync::Arc;
use async_trait::async_trait;
@@ -51,14 +52,14 @@ impl ObsWriter {
#[async_trait]
impl oio::OneShotWrite for ObsWriter {
- async fn write_once(&self, bs: &dyn oio::WriteBuf) -> Result<()> {
- let size = bs.remaining();
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
+ let size = bs.len();
let mut req = self.core.obs_put_object_request(
&self.path,
Some(size as u64),
self.op.content_type(),
self.op.cache_control(),
- AsyncBody::Bytes(bs.copy_to_bytes(size)),
+ AsyncBody::Bytes(bs),
)?;
self.core.sign(&mut req).await?;
diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs
index 09ef04765..2b32abbaf 100644
--- a/core/src/services/onedrive/backend.rs
+++ b/core/src/services/onedrive/backend.rs
@@ -64,7 +64,7 @@ impl Debug for OnedriveBackend {
impl Accessor for OnedriveBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = OneDriveWriter;
+ type Writer = oio::OneShotWriter<OneDriveWriter>;
type BlockingWriter = ();
type Pager = OnedrivePager;
type BlockingPager = ();
@@ -114,7 +114,7 @@ impl Accessor for OnedriveBackend {
Ok((
RpWrite::default(),
- OneDriveWriter::new(self.clone(), args, path),
+ oio::OneShotWriter::new(OneDriveWriter::new(self.clone(), args, path)),
))
}
diff --git a/core/src/services/oss/writer.rs b/core/src/services/oss/writer.rs
index 8fd7ac656..56d262f17 100644
--- a/core/src/services/oss/writer.rs
+++ b/core/src/services/oss/writer.rs
@@ -18,6 +18,7 @@
use std::sync::Arc;
use async_trait::async_trait;
+use bytes::Bytes;
use http::StatusCode;
use super::core::*;
@@ -50,15 +51,15 @@ impl OssWriter {
#[async_trait]
impl oio::OneShotWrite for OssWriter {
- async fn write_once(&self, bs: &dyn oio::WriteBuf) -> Result<()> {
- let size = bs.remaining();
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
+ let size = bs.len();
let mut req = self.core.oss_put_object_request(
&self.path,
Some(size as u64),
self.op.content_type(),
self.op.content_disposition(),
self.op.cache_control(),
- AsyncBody::Bytes(bs.copy_to_bytes(size)),
+ AsyncBody::Bytes(bs),
false,
)?;
diff --git a/core/src/services/s3/writer.rs b/core/src/services/s3/writer.rs
index 7c5ddf924..a3a1bd5bd 100644
--- a/core/src/services/s3/writer.rs
+++ b/core/src/services/s3/writer.rs
@@ -18,6 +18,7 @@
use std::sync::Arc;
use async_trait::async_trait;
+use bytes::Bytes;
use http::StatusCode;
use super::core::*;
@@ -47,8 +48,8 @@ impl S3Writer {
#[async_trait]
impl oio::OneShotWrite for S3Writer {
- async fn write_once(&self, bs: &dyn oio::WriteBuf) -> Result<()> {
- let size = bs.remaining();
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
+ let size = bs.len();
let mut req = self.core.s3_put_object_request(
&self.path,
@@ -56,7 +57,7 @@ impl oio::OneShotWrite for S3Writer {
self.op.content_type(),
self.op.content_disposition(),
self.op.cache_control(),
- AsyncBody::Bytes(bs.copy_to_bytes(size)),
+ AsyncBody::Bytes(bs),
)?;
self.core.sign(&mut req).await?;
diff --git a/core/src/services/supabase/backend.rs b/core/src/services/supabase/backend.rs
index 89996fa8c..402c3abe7 100644
--- a/core/src/services/supabase/backend.rs
+++ b/core/src/services/supabase/backend.rs
@@ -158,7 +158,7 @@ pub struct SupabaseBackend {
impl Accessor for SupabaseBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = SupabaseWriter;
+ type Writer = oio::OneShotWriter<SupabaseWriter>;
type BlockingWriter = ();
// todo: implement Pager to support list and scan
type Pager = ();
@@ -233,7 +233,7 @@ impl Accessor for SupabaseBackend {
Ok((
RpWrite::default(),
- SupabaseWriter::new(self.core.clone(), path, args),
+ oio::OneShotWriter::new(SupabaseWriter::new(self.core.clone(), path, args)),
))
}
diff --git a/core/src/services/vercel_artifacts/backend.rs b/core/src/services/vercel_artifacts/backend.rs
index 70cd9048b..4b88be664 100644
--- a/core/src/services/vercel_artifacts/backend.rs
+++ b/core/src/services/vercel_artifacts/backend.rs
@@ -46,7 +46,7 @@ impl Debug for VercelArtifactsBackend {
impl Accessor for VercelArtifactsBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = VercelArtifactsWriter;
+ type Writer = oio::OneShotWriter<VercelArtifactsWriter>;
type BlockingWriter = ();
type Pager = ();
type BlockingPager = ();
@@ -93,7 +93,11 @@ impl Accessor for VercelArtifactsBackend {
Ok((
RpWrite::default(),
- VercelArtifactsWriter::new(self.clone(), args, path.to_string()),
+ oio::OneShotWriter::new(VercelArtifactsWriter::new(
+ self.clone(),
+ args,
+ path.to_string(),
+ )),
))
}
diff --git a/core/src/services/wasabi/backend.rs b/core/src/services/wasabi/backend.rs
index f2aedab2a..835492bad 100644
--- a/core/src/services/wasabi/backend.rs
+++ b/core/src/services/wasabi/backend.rs
@@ -670,7 +670,7 @@ pub struct WasabiBackend {
impl Accessor for WasabiBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = WasabiWriter;
+ type Writer = oio::OneShotWriter<WasabiWriter>;
type BlockingWriter = ();
type Pager = WasabiPager;
type BlockingPager = ();
@@ -759,7 +759,7 @@ impl Accessor for WasabiBackend {
Ok((
RpWrite::default(),
- WasabiWriter::new(self.core.clone(), args, path.to_string()),
+ oio::OneShotWriter::new(WasabiWriter::new(self.core.clone(), args, path.to_string())),
))
}
diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs
index 509d4c612..cca8728ba 100644
--- a/core/src/services/webdav/backend.rs
+++ b/core/src/services/webdav/backend.rs
@@ -221,7 +221,7 @@ impl Debug for WebdavBackend {
impl Accessor for WebdavBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = WebdavWriter;
+ type Writer = oio::OneShotWriter<WebdavWriter>;
type BlockingWriter = ();
type Pager = Option<WebdavPager>;
type BlockingPager = ();
@@ -286,7 +286,10 @@ impl Accessor for WebdavBackend {
let p = build_abs_path(&self.root, path);
- Ok((RpWrite::default(), WebdavWriter::new(self.clone(), args, p)))
+ Ok((
+ RpWrite::default(),
+ oio::OneShotWriter::new(WebdavWriter::new(self.clone(), args, p)),
+ ))
}
/// # Notes
diff --git a/core/src/services/webhdfs/backend.rs b/core/src/services/webhdfs/backend.rs
index db8e0490a..e8a3a8f04 100644
--- a/core/src/services/webhdfs/backend.rs
+++ b/core/src/services/webhdfs/backend.rs
@@ -399,7 +399,7 @@ impl WebhdfsBackend {
impl Accessor for WebhdfsBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = WebhdfsWriter;
+ type Writer = oio::OneShotWriter<WebhdfsWriter>;
type BlockingWriter = ();
type Pager = WebhdfsPager;
type BlockingPager = ();
@@ -483,7 +483,7 @@ impl Accessor for WebhdfsBackend {
Ok((
RpWrite::default(),
- WebhdfsWriter::new(self.clone(), args, path.to_string()),
+ oio::OneShotWriter::new(WebhdfsWriter::new(self.clone(), args, path.to_string())),
))
}
[incubator-opendal] 31/31: Polish logging
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit b24330c4d74db8672734df96d59b006928509b91
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Sep 11 16:48:03 2023 +0800
Polish logging
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/layers/logging.rs | 24 +++++++++++++-----------
core/src/raw/oio/write/range_write.rs | 8 ++++++--
2 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/core/src/layers/logging.rs b/core/src/layers/logging.rs
index 42084eede..71bb7c099 100644
--- a/core/src/layers/logging.rs
+++ b/core/src/layers/logging.rs
@@ -1258,12 +1258,13 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
self.written += n as u64;
trace!(
target: LOGGING_TARGET,
- "service={} operation={} path={} written={} -> data write {}B",
+ "service={} operation={} path={} written={}B -> input data {}B, write {}B",
self.ctx.scheme,
WriteOperation::Write,
self.path,
self.written,
- n
+ bs.remaining(),
+ n,
);
Poll::Ready(Ok(n))
}
@@ -1272,7 +1273,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
log!(
target: LOGGING_TARGET,
lvl,
- "service={} operation={} path={} written={} -> data write failed: {}",
+ "service={} operation={} path={} written={}B -> data write failed: {}",
self.ctx.scheme,
WriteOperation::Write,
self.path,
@@ -1290,7 +1291,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
Ok(_) => {
trace!(
target: LOGGING_TARGET,
- "service={} operation={} path={} written={} -> abort writer",
+ "service={} operation={} path={} written={}B -> abort writer",
self.ctx.scheme,
WriteOperation::Abort,
self.path,
@@ -1303,7 +1304,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
log!(
target: LOGGING_TARGET,
lvl,
- "service={} operation={} path={} written={} -> abort writer failed: {}",
+ "service={} operation={} path={} written={}B -> abort writer failed: {}",
self.ctx.scheme,
WriteOperation::Abort,
self.path,
@@ -1321,7 +1322,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
Ok(_) => {
debug!(
target: LOGGING_TARGET,
- "service={} operation={} path={} written={} -> data written finished",
+ "service={} operation={} path={} written={}B -> data written finished",
self.ctx.scheme,
self.op,
self.path,
@@ -1334,7 +1335,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
log!(
target: LOGGING_TARGET,
lvl,
- "service={} operation={} path={} written={} -> data close failed: {}",
+ "service={} operation={} path={} written={}B -> data close failed: {}",
self.ctx.scheme,
WriteOperation::Close,
self.path,
@@ -1355,11 +1356,12 @@ impl<W: oio::BlockingWrite> oio::BlockingWrite for LoggingWriter<W> {
self.written += n as u64;
trace!(
target: LOGGING_TARGET,
- "service={} operation={} path={} written={} -> data write {}B",
+ "service={} operation={} path={} written={}B -> input data {}B, write {}B",
self.ctx.scheme,
WriteOperation::BlockingWrite,
self.path,
self.written,
+ bs.remaining(),
n
);
Ok(n)
@@ -1369,7 +1371,7 @@ impl<W: oio::BlockingWrite> oio::BlockingWrite for LoggingWriter<W> {
log!(
target: LOGGING_TARGET,
lvl,
- "service={} operation={} path={} written={} -> data write failed: {}",
+ "service={} operation={} path={} written={}B -> data write failed: {}",
self.ctx.scheme,
WriteOperation::BlockingWrite,
self.path,
@@ -1387,7 +1389,7 @@ impl<W: oio::BlockingWrite> oio::BlockingWrite for LoggingWriter<W> {
Ok(_) => {
debug!(
target: LOGGING_TARGET,
- "service={} operation={} path={} written={} -> data written finished",
+ "service={} operation={} path={} written={}B -> data written finished",
self.ctx.scheme,
self.op,
self.path,
@@ -1400,7 +1402,7 @@ impl<W: oio::BlockingWrite> oio::BlockingWrite for LoggingWriter<W> {
log!(
target: LOGGING_TARGET,
lvl,
- "service={} operation={} path={} written={} -> data close failed: {}",
+ "service={} operation={} path={} written={}B -> data close failed: {}",
self.ctx.scheme,
WriteOperation::BlockingClose,
self.path,
diff --git a/core/src/raw/oio/write/range_write.rs b/core/src/raw/oio/write/range_write.rs
index ae4558a4c..f01f7d40f 100644
--- a/core/src/raw/oio/write/range_write.rs
+++ b/core/src/raw/oio/write/range_write.rs
@@ -128,7 +128,6 @@ impl<W: RangeWrite> oio::Write for RangeWriter<W> {
loop {
match &mut self.state {
State::Idle(w) => {
- let w = w.take().unwrap();
match self.location.clone() {
Some(location) => {
let remaining = bs.remaining();
@@ -152,6 +151,7 @@ impl<W: RangeWrite> oio::Write for RangeWriter<W> {
align_buffer.push(bs);
let written = self.written;
+ let w = w.take().unwrap();
let fut = async move {
let size = align_buffer.len() as u64;
let res = w
@@ -168,6 +168,7 @@ impl<W: RangeWrite> oio::Write for RangeWriter<W> {
self.state = State::Write(Box::pin(fut));
}
None => {
+ let w = w.take().unwrap();
let fut = async move {
let res = w.initiate_range().await;
@@ -188,7 +189,10 @@ impl<W: RangeWrite> oio::Write for RangeWriter<W> {
let (consume, written) = res?;
self.written += written;
self.align_buffer.clear();
- return Poll::Ready(Ok(consume));
+ // It's possible that the buffer is already aligned, no bytes has been consumed.
+ if consume != 0 {
+ return Poll::Ready(Ok(consume));
+ }
}
State::Complete(_) => {
unreachable!("RangeWriter must not go into State::Complete during poll_write")
[incubator-opendal] 24/31: Refactor gcs
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 9b2534b2f72e5439a24f92f78ea3a2b3518db07b
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Sep 11 14:03:36 2023 +0800
Refactor gcs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/oio/write/mod.rs | 4 +
core/src/raw/oio/write/multipart_upload_write.rs | 16 +-
core/src/raw/oio/write/range_write.rs | 269 +++++++++++++++++++++++
core/src/services/gcs/backend.rs | 15 +-
core/src/services/gcs/core.rs | 26 +--
core/src/services/gcs/writer.rs | 178 +++++----------
6 files changed, 357 insertions(+), 151 deletions(-)
diff --git a/core/src/raw/oio/write/mod.rs b/core/src/raw/oio/write/mod.rs
index ff6d8377e..df0b83b28 100644
--- a/core/src/raw/oio/write/mod.rs
+++ b/core/src/raw/oio/write/mod.rs
@@ -42,3 +42,7 @@ pub use one_shot_write::OneShotWriter;
mod exact_buf_write;
pub use exact_buf_write::ExactBufWriter;
+
+mod range_write;
+pub use range_write::RangeWrite;
+pub use range_write::RangeWriter;
diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs
index 81ee1f1ee..1b2148cb4 100644
--- a/core/src/raw/oio/write/multipart_upload_write.rs
+++ b/core/src/raw/oio/write/multipart_upload_write.rs
@@ -166,12 +166,12 @@ where
}
State::Close(_) => {
unreachable!(
- "MultipartUploadWriter must not go into State:Close during poll_write"
+ "MultipartUploadWriter must not go into State::Close during poll_write"
)
}
State::Abort(_) => {
unreachable!(
- "MultipartUploadWriter must not go into State:Abort during poll_write"
+ "MultipartUploadWriter must not go into State::Abort during poll_write"
)
}
}
@@ -200,13 +200,13 @@ where
return Poll::Ready(res);
}
State::Init(_) => unreachable!(
- "MultipartUploadWriter must not go into State:Init during poll_close"
+ "MultipartUploadWriter must not go into State::Init during poll_close"
),
State::Write(_) => unreachable!(
- "MultipartUploadWriter must not go into State:Write during poll_close"
+ "MultipartUploadWriter must not go into State::Write during poll_close"
),
State::Abort(_) => unreachable!(
- "MultipartUploadWriter must not go into State:Abort during poll_close"
+ "MultipartUploadWriter must not go into State::Abort during poll_close"
),
}
}
@@ -233,13 +233,13 @@ where
return Poll::Ready(res);
}
State::Init(_) => unreachable!(
- "MultipartUploadWriter must not go into State:Init during poll_abort"
+ "MultipartUploadWriter must not go into State::Init during poll_abort"
),
State::Write(_) => unreachable!(
- "MultipartUploadWriter must not go into State:Write during poll_abort"
+ "MultipartUploadWriter must not go into State::Write during poll_abort"
),
State::Close(_) => unreachable!(
- "MultipartUploadWriter must not go into State:Close during poll_abort"
+ "MultipartUploadWriter must not go into State::Close during poll_abort"
),
}
}
diff --git a/core/src/raw/oio/write/range_write.rs b/core/src/raw/oio/write/range_write.rs
new file mode 100644
index 000000000..a96af8d3d
--- /dev/null
+++ b/core/src/raw/oio/write/range_write.rs
@@ -0,0 +1,269 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use async_trait::async_trait;
+use futures::future::BoxFuture;
+use futures::FutureExt;
+use std::task::{ready, Context, Poll};
+
+use crate::raw::oio::WriteBuf;
+use crate::raw::*;
+use crate::*;
+
+#[async_trait]
+pub trait RangeWrite: Send + Sync + Unpin + 'static {
+ async fn initiate_range(&self) -> Result<String>;
+
+ async fn write_range(
+ &self,
+ location: &str,
+ written: u64,
+ size: u64,
+ body: AsyncBody,
+ ) -> Result<()>;
+
+ async fn complete_range(
+ &self,
+ location: &str,
+ written: u64,
+ size: u64,
+ body: AsyncBody,
+ ) -> Result<()>;
+
+ async fn abort_range(&self, location: &str) -> Result<()>;
+}
+
+pub struct RangeWriter<W: RangeWrite> {
+ location: Option<String>,
+ written: u64,
+ align_size: usize,
+ align_buffer: oio::ChunkedCursor,
+
+ state: State<W>,
+}
+
+enum State<W> {
+ Idle(Option<W>),
+ Init(BoxFuture<'static, (W, Result<String>)>),
+ /// The returning value is (consume size, written size).
+ Write(BoxFuture<'static, (W, Result<(usize, u64)>)>),
+ Complete(BoxFuture<'static, (W, Result<()>)>),
+ Abort(BoxFuture<'static, (W, Result<()>)>),
+}
+
+/// # Safety
+///
+/// We will only take `&mut Self` reference for State.
+unsafe impl<W: RangeWrite> Sync for State<W> {}
+
+impl<W: RangeWrite> RangeWriter<W> {
+ /// Create a new MultipartUploadWriter.
+ pub fn new(inner: W) -> Self {
+ Self {
+ state: State::Idle(Some(inner)),
+
+ location: None,
+ written: 0,
+ align_size: 256 * 1024,
+ align_buffer: oio::ChunkedCursor::default(),
+ }
+ }
+
+ pub fn with_align_size(mut self, size: usize) -> Self {
+ self.align_size = size;
+ self
+ }
+}
+
+impl<W: RangeWrite> RangeWriter<W> {
+ fn align(&mut self, bs: &dyn WriteBuf) -> usize {
+ let remaining = bs.remaining();
+ let current_size = self.align_buffer.len();
+
+ let total_size = current_size + remaining;
+ if total_size <= self.align_size {
+ return remaining;
+ }
+
+ total_size - total_size % self.align_size - current_size
+ }
+}
+
+impl<W: RangeWrite> oio::Write for RangeWriter<W> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn WriteBuf) -> Poll<Result<usize>> {
+ loop {
+ match &mut self.state {
+ State::Idle(w) => {
+ let w = w.take().unwrap();
+ match self.location.clone() {
+ Some(location) => {
+ let remaining = bs.remaining();
+ let current_size = self.align_buffer.len();
+ let mut total_size = current_size + remaining;
+
+ if total_size <= self.align_size {
+ let bs = bs.copy_to_bytes(remaining);
+ self.align_buffer.push(bs);
+ return Poll::Ready(Ok(remaining));
+ }
+ // If total_size is aligned, we need to write one less chunk to make sure
+ // that the file has at least one chunk during complete stage.
+ if total_size % self.align_size == 0 {
+ total_size -= self.align_size;
+ }
+
+ let consume = total_size - total_size % self.align_size - current_size;
+ let mut align_buffer = self.align_buffer.clone();
+ let bs = bs.copy_to_bytes(consume);
+ align_buffer.push(bs);
+
+ let written = self.written;
+ let fut = async move {
+ let size = align_buffer.len() as u64;
+ let res = w
+ .write_range(
+ &location,
+ written,
+ size,
+ AsyncBody::Stream(Box::new(align_buffer)),
+ )
+ .await;
+
+ (w, res.map(|_| (consume, size)))
+ };
+ self.state = State::Write(Box::pin(fut));
+ }
+ None => {
+ let fut = async move {
+ let res = w.initiate_range().await;
+
+ (w, res)
+ };
+ self.state = State::Init(Box::pin(fut));
+ }
+ }
+ }
+ State::Init(fut) => {
+ let (w, res) = ready!(fut.poll_unpin(cx));
+ self.state = State::Idle(Some(w));
+ self.location = Some(res?);
+ }
+ State::Write(fut) => {
+ let (w, res) = ready!(fut.poll_unpin(cx));
+ self.state = State::Idle(Some(w));
+ let (consume, written) = res?;
+ self.written += written;
+ self.align_buffer.clear();
+ return Poll::Ready(Ok(consume));
+ }
+ State::Complete(_) => {
+ unreachable!("RangeWriter must not go into State::Complete during poll_write")
+ }
+ State::Abort(_) => {
+ unreachable!("RangeWriter must not go into State::Abort during poll_write")
+ }
+ }
+ }
+ }
+
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ loop {
+ match &mut self.state {
+ State::Idle(w) => {
+ let w = w.take().unwrap();
+ match self.location.clone() {
+ Some(location) => {
+ debug_assert!(
+ self.align_buffer.len() > 0,
+ "RangeWriter requires to have last chunk"
+ );
+ let mut align_buffer = self.align_buffer.clone();
+
+ let written = self.written;
+ let fut = async move {
+ let size = align_buffer.len() as u64;
+ let res = w
+ .complete_range(
+ &location,
+ written,
+ size,
+ AsyncBody::Stream(Box::new(align_buffer)),
+ )
+ .await;
+
+ (w, res)
+ };
+ self.state = State::Complete(Box::pin(fut));
+ }
+ None => return Poll::Ready(Ok(())),
+ }
+ }
+ State::Init(_) => {
+ unreachable!("RangeWriter must not go into State::Init during poll_close")
+ }
+ State::Write(_) => {
+ unreachable!("RangeWriter must not go into State::Write during poll_close")
+ }
+ State::Complete(fut) => {
+ let (w, res) = ready!(fut.poll_unpin(cx));
+ self.state = State::Idle(Some(w));
+ self.align_buffer.clear();
+ return Poll::Ready(res);
+ }
+ State::Abort(_) => {
+ unreachable!("RangeWriter must not go into State::Abort during poll_close")
+ }
+ }
+ }
+ }
+
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ loop {
+ match &mut self.state {
+ State::Idle(w) => {
+ let w = w.take().unwrap();
+ match self.location.clone() {
+ Some(location) => {
+ let fut = async move {
+ let res = w.abort_range(&location).await;
+
+ (w, res)
+ };
+ self.state = State::Complete(Box::pin(fut));
+ }
+ None => return Poll::Ready(Ok(())),
+ }
+ }
+ State::Init(_) => {
+ unreachable!("RangeWriter must not go into State::Init during poll_close")
+ }
+ State::Write(_) => {
+ unreachable!("RangeWriter must not go into State::Write during poll_close")
+ }
+ State::Complete(_) => {
+ unreachable!("RangeWriter must not go into State::Complete during poll_close")
+ }
+ State::Abort(fut) => {
+ let (w, res) = ready!(fut.poll_unpin(cx));
+ self.state = State::Idle(Some(w));
+ self.align_buffer.clear();
+ return Poll::Ready(res);
+ }
+ }
+ }
+ }
+}
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index 77549908e..f91c792a0 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -35,6 +35,7 @@ use super::error::parse_error;
use super::pager::GcsPager;
use super::writer::GcsWriter;
use crate::raw::*;
+use crate::services::gcs::writer::GcsWriters;
use crate::*;
const DEFAULT_GCS_ENDPOINT: &str = "https://storage.googleapis.com";
@@ -336,7 +337,7 @@ pub struct GcsBackend {
impl Accessor for GcsBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = GcsWriter;
+ type Writer = GcsWriters;
type BlockingWriter = ();
type Pager = GcsPager;
type BlockingPager = ();
@@ -418,10 +419,14 @@ impl Accessor for GcsBackend {
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
- Ok((
- RpWrite::default(),
- GcsWriter::new(self.core.clone(), path, args),
- ))
+ let w = GcsWriter::new(self.core.clone(), path, args.clone());
+ let w = if args.content_length().is_some() {
+ GcsWriters::One(oio::OneShotWriter::new(w))
+ } else {
+ GcsWriters::Two(oio::RangeWriter::new(w))
+ };
+
+ Ok((RpWrite::default(), w))
}
async fn copy(&self, from: &str, to: &str, _: OpCopy) -> Result<RpCopy> {
diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs
index b8eca1ece..82c6d9054 100644
--- a/core/src/services/gcs/core.rs
+++ b/core/src/services/gcs/core.rs
@@ -558,21 +558,11 @@ impl GcsCore {
location: &str,
size: u64,
written: u64,
- is_last_part: bool,
body: AsyncBody,
) -> Result<Request<AsyncBody>> {
let mut req = Request::put(location);
- let range_header = if is_last_part {
- format!(
- "bytes {}-{}/{}",
- written,
- written + size - 1,
- written + size
- )
- } else {
- format!("bytes {}-{}/*", written, written + size - 1)
- };
+ let range_header = format!("bytes {}-{}/*", written, written + size - 1);
req = req
.header(CONTENT_LENGTH, size)
@@ -587,22 +577,22 @@ impl GcsCore {
pub async fn gcs_complete_resumable_upload(
&self,
location: &str,
- written_bytes: u64,
- bs: Bytes,
+ written: u64,
+ size: u64,
+ body: AsyncBody,
) -> Result<Response<IncomingAsyncBody>> {
- let size = bs.len() as u64;
let mut req = Request::post(location)
.header(CONTENT_LENGTH, size)
.header(
CONTENT_RANGE,
format!(
"bytes {}-{}/{}",
- written_bytes,
- written_bytes + size - 1,
- written_bytes + size
+ written,
+ written + size - 1,
+ written + size
),
)
- .body(AsyncBody::Bytes(bs))
+ .body(body)
.map_err(new_request_build_error)?;
self.sign(&mut req).await?;
diff --git a/core/src/services/gcs/writer.rs b/core/src/services/gcs/writer.rs
index c61c7df4c..6b66240f7 100644
--- a/core/src/services/gcs/writer.rs
+++ b/core/src/services/gcs/writer.rs
@@ -15,11 +15,14 @@
// specific language governing permissions and limitations
// under the License.
+use std::fmt::{Debug, Formatter};
use std::sync::Arc;
-use std::task::{Context, Poll};
+use std::task::{ready, Context, Poll};
use async_trait::async_trait;
use bytes::Bytes;
+use futures::future::BoxFuture;
+use futures::FutureExt;
use http::StatusCode;
use super::core::GcsCore;
@@ -27,38 +30,33 @@ use super::error::parse_error;
use crate::raw::*;
use crate::*;
+pub type GcsWriters =
+ oio::TwoWaysWriter<oio::OneShotWriter<GcsWriter>, oio::RangeWriter<GcsWriter>>;
+
pub struct GcsWriter {
core: Arc<GcsCore>,
path: String,
op: OpWrite,
-
- location: Option<String>,
- written: u64,
- buffer: oio::VectorCursor,
- write_fixed_size: usize,
}
impl GcsWriter {
pub fn new(core: Arc<GcsCore>, path: &str, op: OpWrite) -> Self {
- let write_fixed_size = core.write_fixed_size;
GcsWriter {
core,
path: path.to_string(),
op,
-
- location: None,
- written: 0,
- buffer: oio::VectorCursor::new(),
- write_fixed_size,
}
}
+}
- async fn write_oneshot(&self, size: u64, body: AsyncBody) -> Result<()> {
+#[async_trait]
+impl oio::OneShotWrite for GcsWriter {
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
let mut req = self.core.gcs_insert_object_request(
&percent_encode_path(&self.path),
- Some(size),
+ Some(bs.len() as u64),
&self.op,
- body,
+ AsyncBody::Bytes(bs),
)?;
self.core.sign(&mut req).await?;
@@ -75,8 +73,11 @@ impl GcsWriter {
_ => Err(parse_error(resp).await?),
}
}
+}
- async fn initiate_upload(&self) -> Result<String> {
+#[async_trait]
+impl oio::RangeWrite for GcsWriter {
+ async fn initiate_range(&self) -> Result<String> {
let resp = self.core.gcs_initiate_resumable_upload(&self.path).await?;
let status = resp.status();
@@ -96,14 +97,16 @@ impl GcsWriter {
}
}
- async fn write_part(&self, location: &str, bs: Bytes) -> Result<()> {
- let mut req = self.core.gcs_upload_in_resumable_upload(
- location,
- bs.len() as u64,
- self.written,
- false,
- AsyncBody::Bytes(bs),
- )?;
+ async fn write_range(
+ &self,
+ location: &str,
+ written: u64,
+ size: u64,
+ body: AsyncBody,
+ ) -> Result<()> {
+ let mut req = self
+ .core
+ .gcs_upload_in_resumable_upload(location, size, written, body)?;
self.core.sign(&mut req).await?;
@@ -115,105 +118,40 @@ impl GcsWriter {
_ => Err(parse_error(resp).await?),
}
}
-}
-#[async_trait]
-impl oio::Write for GcsWriter {
- fn poll_write(&mut self, _: &mut Context<'_>, _: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- // let size = bs.remaining();
- //
- // let location = match &self.location {
- // Some(location) => location,
- // None => {
- // if self.op.content_length().unwrap_or_default() == size as u64 && self.written == 0
- // {
- // self.write_oneshot(size as u64, AsyncBody::Bytes(bs.copy_to_bytes(size)))
- // .await?;
- //
- // return Ok(size);
- // } else {
- // let location = self.initiate_upload().await?;
- // self.location = Some(location);
- // self.location.as_deref().unwrap()
- // }
- // }
- // };
- //
- // self.buffer.push(bs.copy_to_bytes(size));
- // // Return directly if the buffer is not full
- // if self.buffer.len() <= self.write_fixed_size {
- // return Ok(size);
- // }
- //
- // let bs = self.buffer.peak_exact(self.write_fixed_size);
- //
- // match self.write_part(location, bs).await {
- // Ok(_) => {
- // self.buffer.take(self.write_fixed_size);
- // self.written += self.write_fixed_size as u64;
- // Ok(size)
- // }
- // Err(e) => {
- // // If the upload fails, we should pop the given bs to make sure
- // // write is re-enter safe.
- // self.buffer.pop();
- // Err(e)
- // }
- // }
-
- todo!()
- }
+ async fn complete_range(
+ &self,
+ location: &str,
+ written: u64,
+ size: u64,
+ body: AsyncBody,
+ ) -> Result<()> {
+ let resp = self
+ .core
+ .gcs_complete_resumable_upload(location, written, size, body)
+ .await?;
- fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
- // let location = if let Some(location) = &self.location {
- // location
- // } else {
- // return Ok(());
- // };
- //
- // let resp = self.core.gcs_abort_resumable_upload(location).await?;
- //
- // match resp.status().as_u16() {
- // // gcs returns 499 if the upload aborted successfully
- // // reference: https://cloud.google.com/storage/docs/performing-resumable-uploads#cancel-upload-json
- // 499 => {
- // resp.into_body().consume().await?;
- // self.location = None;
- // self.buffer.clear();
- // Ok(())
- // }
- // _ => Err(parse_error(resp).await?),
- // }
-
- todo!()
+ let status = resp.status();
+ match status {
+ StatusCode::OK => {
+ resp.into_body().consume().await?;
+ Ok(())
+ }
+ _ => Err(parse_error(resp).await?),
+ }
}
- fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
- // let location = if let Some(location) = &self.location {
- // location
- // } else {
- // return Ok(());
- // };
- //
- // let bs = self.buffer.peak_exact(self.buffer.len());
- //
- // let resp = self
- // .core
- // .gcs_complete_resumable_upload(location, self.written, bs)
- // .await?;
- //
- // let status = resp.status();
- // match status {
- // StatusCode::OK => {
- // resp.into_body().consume().await?;
- //
- // self.location = None;
- // self.buffer.clear();
- // Ok(())
- // }
- // _ => Err(parse_error(resp).await?),
- // }
-
- todo!()
+ async fn abort_range(&self, location: &str) -> Result<()> {
+ let resp = self.core.gcs_abort_resumable_upload(location).await?;
+
+ match resp.status().as_u16() {
+ // gcs returns 499 if the upload aborted successfully
+ // reference: https://cloud.google.com/storage/docs/performing-resumable-uploads#cancel-upload-json
+ 499 => {
+ resp.into_body().consume().await?;
+ Ok(())
+ }
+ _ => Err(parse_error(resp).await?),
+ }
}
}
[incubator-opendal] 19/31: Save
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit fdd7bb002a1f46e65336d44db7bb85d3668963c1
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 17:06:05 2023 +0800
Save
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/layers/blocking.rs | 7 +++++--
core/src/raw/adapters/kv/backend.rs | 5 ++---
core/src/raw/adapters/typed_kv/backend.rs | 5 ++---
core/src/types/operator/operator.rs | 1 +
4 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/core/src/layers/blocking.rs b/core/src/layers/blocking.rs
index 6beb8881b..7dd2d189c 100644
--- a/core/src/layers/blocking.rs
+++ b/core/src/layers/blocking.rs
@@ -18,6 +18,7 @@
use async_trait::async_trait;
use bytes;
use bytes::Bytes;
+use futures::future::poll_fn;
use tokio::runtime::Handle;
use crate::raw::oio::ReadExt;
@@ -197,11 +198,13 @@ impl<I: oio::Read + 'static> oio::BlockingRead for BlockingWrapper<I> {
impl<I: oio::Write + 'static> oio::BlockingWrite for BlockingWrapper<I> {
fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
- self.handle.block_on(self.inner.write(bs))
+ self.handle
+ .block_on(poll_fn(|cx| self.inner.poll_write(cx, bs)))
}
fn close(&mut self) -> Result<()> {
- self.handle.block_on(self.inner.close())
+ self.handle
+ .block_on(poll_fn(|cx| self.inner.poll_close(cx)))
}
}
diff --git a/core/src/raw/adapters/kv/backend.rs b/core/src/raw/adapters/kv/backend.rs
index 48c327b0a..b664a76a9 100644
--- a/core/src/raw/adapters/kv/backend.rs
+++ b/core/src/raw/adapters/kv/backend.rs
@@ -20,6 +20,7 @@ use std::task::{ready, Context, Poll};
use async_trait::async_trait;
use futures::future::BoxFuture;
+use futures::FutureExt;
use super::Adapter;
use crate::raw::*;
@@ -445,9 +446,7 @@ impl<S: Adapter> oio::Write for KvWriter<S> {
None => return Poll::Ready(Ok(())),
};
- let fut = async move {
- kv.set(&path, &buf).await?;
- };
+ let fut = async move { kv.set(&path, &buf).await };
self.future = Some(Box::pin(fut));
}
}
diff --git a/core/src/raw/adapters/typed_kv/backend.rs b/core/src/raw/adapters/typed_kv/backend.rs
index fda56bf28..ce0f9f073 100644
--- a/core/src/raw/adapters/typed_kv/backend.rs
+++ b/core/src/raw/adapters/typed_kv/backend.rs
@@ -21,6 +21,7 @@ use std::task::{ready, Context, Poll};
use async_trait::async_trait;
use bytes::Bytes;
use futures::future::BoxFuture;
+use futures::FutureExt;
use super::Adapter;
use super::Value;
@@ -455,9 +456,7 @@ impl<S: Adapter> oio::Write for KvWriter<S> {
let path = self.path.clone();
let value = self.build();
- let fut = async move {
- kv.set(&path, value).await?;
- };
+ let fut = async move { kv.set(&path, value).await };
self.future = Some(Box::pin(fut));
}
}
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index b51c2bc60..1aa814658 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -27,6 +27,7 @@ use tokio::io::ReadBuf;
use super::BlockingOperator;
use crate::operator_futures::*;
+use crate::raw::oio::WriteExt;
use crate::raw::*;
use crate::*;
[incubator-opendal] 01/31: Save work
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit e1d7ce09c2f56a9898d4916c2fae45e10a81b200
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 10:57:10 2023 +0800
Save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/benches/oio/utils.rs | 10 +-
core/src/layers/complete.rs | 6 +-
core/src/layers/concurrent_limit.rs | 6 +-
core/src/layers/error_context.rs | 6 +-
core/src/layers/logging.rs | 6 +-
core/src/layers/madsim.rs | 6 +-
core/src/layers/metrics.rs | 6 +-
core/src/layers/minitrace.rs | 6 +-
core/src/layers/oteltrace.rs | 6 +-
core/src/layers/prometheus.rs | 6 +-
core/src/layers/retry.rs | 6 +-
core/src/layers/throttle.rs | 6 +-
core/src/layers/timeout.rs | 6 +-
core/src/layers/tracing.rs | 6 +-
core/src/raw/adapters/kv/backend.rs | 59 +++++++++---
core/src/raw/adapters/typed_kv/backend.rs | 54 +++++++++--
core/src/raw/oio/write/api.rs | 112 +++++++++++++++++++----
core/src/raw/oio/write/append_object_write.rs | 6 +-
core/src/raw/oio/write/compose_write.rs | 12 +--
core/src/raw/oio/write/exact_buf_write.rs | 30 +++---
core/src/raw/oio/write/mod.rs | 1 +
core/src/raw/oio/write/multipart_upload_write.rs | 6 +-
core/src/raw/oio/write/one_shot_write.rs | 6 +-
core/src/services/azblob/writer.rs | 6 +-
core/src/services/azdfs/writer.rs | 6 +-
core/src/services/dropbox/writer.rs | 6 +-
core/src/services/fs/writer.rs | 6 +-
core/src/services/ftp/writer.rs | 6 +-
core/src/services/gcs/writer.rs | 6 +-
core/src/services/gdrive/writer.rs | 6 +-
core/src/services/ghac/writer.rs | 6 +-
core/src/services/hdfs/writer.rs | 6 +-
core/src/services/ipmfs/writer.rs | 6 +-
core/src/services/onedrive/writer.rs | 6 +-
core/src/services/sftp/writer.rs | 6 +-
core/src/services/supabase/writer.rs | 6 +-
core/src/services/vercel_artifacts/writer.rs | 6 +-
core/src/services/wasabi/writer.rs | 6 +-
core/src/services/webdav/writer.rs | 6 +-
core/src/services/webhdfs/writer.rs | 6 +-
core/src/types/writer.rs | 4 +-
41 files changed, 316 insertions(+), 164 deletions(-)
diff --git a/core/benches/oio/utils.rs b/core/benches/oio/utils.rs
index 1671057f0..67b4fd451 100644
--- a/core/benches/oio/utils.rs
+++ b/core/benches/oio/utils.rs
@@ -26,15 +26,19 @@ pub struct BlackHoleWriter;
#[async_trait]
impl oio::Write for BlackHoleWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> opendal::Result<usize> {
+ fn poll_write(
+ &mut self,
+ cx: &mut Context<'_>,
+ bs: &dyn oio::WriteBuf,
+ ) -> opendal::Result<usize> {
Ok(bs.remaining())
}
- async fn abort(&mut self) -> opendal::Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> opendal::Result<()> {
Ok(())
}
- async fn close(&mut self) -> opendal::Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> opendal::Result<()> {
Ok(())
}
}
diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs
index 048905bae..19429cff8 100644
--- a/core/src/layers/complete.rs
+++ b/core/src/layers/complete.rs
@@ -711,7 +711,7 @@ impl<W> oio::Write for CompleteWriter<W>
where
W: oio::Write,
{
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let w = self.inner.as_mut().ok_or_else(|| {
Error::new(ErrorKind::Unexpected, "writer has been closed or aborted")
})?;
@@ -733,7 +733,7 @@ where
Ok(n)
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
let w = self.inner.as_mut().ok_or_else(|| {
Error::new(ErrorKind::Unexpected, "writer has been closed or aborted")
})?;
@@ -744,7 +744,7 @@ where
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
if let Some(size) = self.size {
if self.written < size {
return Err(Error::new(
diff --git a/core/src/layers/concurrent_limit.rs b/core/src/layers/concurrent_limit.rs
index bdc473384..f726d2207 100644
--- a/core/src/layers/concurrent_limit.rs
+++ b/core/src/layers/concurrent_limit.rs
@@ -285,15 +285,15 @@ impl<R: oio::BlockingRead> oio::BlockingRead for ConcurrentLimitWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for ConcurrentLimitWrapper<R> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
self.inner.write(bs).await
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.abort().await
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.close().await
}
}
diff --git a/core/src/layers/error_context.rs b/core/src/layers/error_context.rs
index 15d7a0dd0..c6b3bd2be 100644
--- a/core/src/layers/error_context.rs
+++ b/core/src/layers/error_context.rs
@@ -403,7 +403,7 @@ impl<T: oio::BlockingRead> oio::BlockingRead for ErrorContextWrapper<T> {
#[async_trait::async_trait]
impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
self.inner.write(bs).await.map_err(|err| {
err.with_operation(WriteOperation::Write)
.with_context("service", self.scheme)
@@ -411,7 +411,7 @@ impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
})
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.abort().await.map_err(|err| {
err.with_operation(WriteOperation::Abort)
.with_context("service", self.scheme)
@@ -419,7 +419,7 @@ impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
})
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.close().await.map_err(|err| {
err.with_operation(WriteOperation::Close)
.with_context("service", self.scheme)
diff --git a/core/src/layers/logging.rs b/core/src/layers/logging.rs
index f0cea32e8..f55bbd627 100644
--- a/core/src/layers/logging.rs
+++ b/core/src/layers/logging.rs
@@ -1252,7 +1252,7 @@ impl<W> LoggingWriter<W> {
#[async_trait]
impl<W: oio::Write> oio::Write for LoggingWriter<W> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
match self.inner.write(bs).await {
Ok(n) => {
self.written += n as u64;
@@ -1285,7 +1285,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
match self.inner.abort().await {
Ok(_) => {
trace!(
@@ -1316,7 +1316,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
}
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
match self.inner.close().await {
Ok(_) => {
debug!(
diff --git a/core/src/layers/madsim.rs b/core/src/layers/madsim.rs
index b0b5cf179..f58eb1997 100644
--- a/core/src/layers/madsim.rs
+++ b/core/src/layers/madsim.rs
@@ -302,7 +302,7 @@ pub struct MadsimWriter {
#[async_trait]
impl oio::Write for MadsimWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> crate::Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> crate::Result<usize> {
#[cfg(madsim)]
{
let req = Request::Write(self.path.to_string(), bs);
@@ -318,14 +318,14 @@ impl oio::Write for MadsimWriter {
}
}
- async fn abort(&mut self) -> crate::Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> crate::Result<()> {
Err(Error::new(
ErrorKind::Unsupported,
"will be supported in the future",
))
}
- async fn close(&mut self) -> crate::Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> crate::Result<()> {
Ok(())
}
}
diff --git a/core/src/layers/metrics.rs b/core/src/layers/metrics.rs
index b16f87a40..7a7b73fc4 100644
--- a/core/src/layers/metrics.rs
+++ b/core/src/layers/metrics.rs
@@ -847,7 +847,7 @@ impl<R: oio::BlockingRead> oio::BlockingRead for MetricWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for MetricWrapper<R> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
self.inner
.write(bs)
.await
@@ -861,14 +861,14 @@ impl<R: oio::Write> oio::Write for MetricWrapper<R> {
})
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.abort().await.map_err(|err| {
self.handle.increment_errors_total(self.op, err.kind());
err
})
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.close().await.map_err(|err| {
self.handle.increment_errors_total(self.op, err.kind());
err
diff --git a/core/src/layers/minitrace.rs b/core/src/layers/minitrace.rs
index f54f40593..8d8662846 100644
--- a/core/src/layers/minitrace.rs
+++ b/core/src/layers/minitrace.rs
@@ -337,7 +337,7 @@ impl<R: oio::BlockingRead> oio::BlockingRead for MinitraceWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for MinitraceWrapper<R> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
self.inner
.write(bs)
.in_span(Span::enter_with_parent(
@@ -347,7 +347,7 @@ impl<R: oio::Write> oio::Write for MinitraceWrapper<R> {
.await
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner
.abort()
.in_span(Span::enter_with_parent(
@@ -357,7 +357,7 @@ impl<R: oio::Write> oio::Write for MinitraceWrapper<R> {
.await
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner
.close()
.in_span(Span::enter_with_parent(
diff --git a/core/src/layers/oteltrace.rs b/core/src/layers/oteltrace.rs
index cceca3a69..f0a8ae82b 100644
--- a/core/src/layers/oteltrace.rs
+++ b/core/src/layers/oteltrace.rs
@@ -313,15 +313,15 @@ impl<R: oio::BlockingRead> oio::BlockingRead for OtelTraceWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for OtelTraceWrapper<R> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
self.inner.write(bs).await
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.abort().await
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.close().await
}
}
diff --git a/core/src/layers/prometheus.rs b/core/src/layers/prometheus.rs
index 2e294c9e5..98637752b 100644
--- a/core/src/layers/prometheus.rs
+++ b/core/src/layers/prometheus.rs
@@ -662,7 +662,7 @@ impl<R: oio::BlockingRead> oio::BlockingRead for PrometheusMetricWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for PrometheusMetricWrapper<R> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
self.inner
.write(bs)
.await
@@ -679,14 +679,14 @@ impl<R: oio::Write> oio::Write for PrometheusMetricWrapper<R> {
})
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.abort().await.map_err(|err| {
self.stats.increment_errors_total(self.op, err.kind());
err
})
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.close().await.map_err(|err| {
self.stats.increment_errors_total(self.op, err.kind());
err
diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs
index 3746008ab..3a1c5521f 100644
--- a/core/src/layers/retry.rs
+++ b/core/src/layers/retry.rs
@@ -872,7 +872,7 @@ impl<R: oio::BlockingRead, I: RetryInterceptor> oio::BlockingRead for RetryWrapp
#[async_trait]
impl<R: oio::Write, I: RetryInterceptor> oio::Write for RetryWrapper<R, I> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let mut backoff = self.builder.build();
loop {
@@ -898,7 +898,7 @@ impl<R: oio::Write, I: RetryInterceptor> oio::Write for RetryWrapper<R, I> {
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
let mut backoff = self.builder.build();
loop {
@@ -924,7 +924,7 @@ impl<R: oio::Write, I: RetryInterceptor> oio::Write for RetryWrapper<R, I> {
}
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
let mut backoff = self.builder.build();
loop {
diff --git a/core/src/layers/throttle.rs b/core/src/layers/throttle.rs
index aea7b6381..7787c77d8 100644
--- a/core/src/layers/throttle.rs
+++ b/core/src/layers/throttle.rs
@@ -216,7 +216,7 @@ impl<R: oio::BlockingRead> oio::BlockingRead for ThrottleWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for ThrottleWrapper<R> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let buf_length = NonZeroU32::new(bs.remaining() as u32).unwrap();
loop {
@@ -241,11 +241,11 @@ impl<R: oio::Write> oio::Write for ThrottleWrapper<R> {
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.abort().await
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.close().await
}
}
diff --git a/core/src/layers/timeout.rs b/core/src/layers/timeout.rs
index c02a4d58a..751caa915 100644
--- a/core/src/layers/timeout.rs
+++ b/core/src/layers/timeout.rs
@@ -322,7 +322,7 @@ impl<R: oio::Read> oio::Read for TimeoutWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for TimeoutWrapper<R> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let timeout = self.io_timeout(bs.remaining() as u64);
tokio::time::timeout(timeout, self.inner.write(bs))
@@ -335,7 +335,7 @@ impl<R: oio::Write> oio::Write for TimeoutWrapper<R> {
})?
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
tokio::time::timeout(self.timeout, self.inner.abort())
.await
.map_err(|_| {
@@ -346,7 +346,7 @@ impl<R: oio::Write> oio::Write for TimeoutWrapper<R> {
})?
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
tokio::time::timeout(self.timeout, self.inner.close())
.await
.map_err(|_| {
diff --git a/core/src/layers/tracing.rs b/core/src/layers/tracing.rs
index b829f8376..35f718cb5 100644
--- a/core/src/layers/tracing.rs
+++ b/core/src/layers/tracing.rs
@@ -324,7 +324,7 @@ impl<R: oio::Write> oio::Write for TracingWrapper<R> {
parent = &self.span,
level = "trace",
skip_all)]
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
self.inner.write(bs).await
}
@@ -332,7 +332,7 @@ impl<R: oio::Write> oio::Write for TracingWrapper<R> {
parent = &self.span,
level = "trace",
skip_all)]
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.abort().await
}
@@ -340,7 +340,7 @@ impl<R: oio::Write> oio::Write for TracingWrapper<R> {
parent = &self.span,
level = "trace",
skip_all)]
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.inner.close().await
}
}
diff --git a/core/src/raw/adapters/kv/backend.rs b/core/src/raw/adapters/kv/backend.rs
index 513bda67e..a3ead2e7a 100644
--- a/core/src/raw/adapters/kv/backend.rs
+++ b/core/src/raw/adapters/kv/backend.rs
@@ -16,8 +16,10 @@
// under the License.
use std::sync::Arc;
+use std::task::{ready, Context, Poll};
use async_trait::async_trait;
+use futures::future::BoxFuture;
use super::Adapter;
use crate::raw::*;
@@ -374,6 +376,7 @@ pub struct KvWriter<S> {
/// TODO: if kv supports append, we can use them directly.
buf: Option<Vec<u8>>,
+ future: Option<BoxFuture<'static, Result<()>>>,
}
impl<S> KvWriter<S> {
@@ -382,6 +385,7 @@ impl<S> KvWriter<S> {
kv,
path,
buf: None,
+ future: None,
}
}
}
@@ -389,29 +393,60 @@ impl<S> KvWriter<S> {
#[async_trait]
impl<S: Adapter> oio::Write for KvWriter<S> {
// TODO: we need to support append in the future.
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, _: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
+ if self.future.is_some() {
+ self.future = None;
+ return Poll::Ready(Err(Error::new(
+ ErrorKind::Unexpected,
+ "there is a future on going, it's maybe a bug to go into this case",
+ )));
+ }
+
let size = bs.chunk().len();
let mut buf = self.buf.take().unwrap_or_else(|| Vec::with_capacity(size));
buf.extend_from_slice(bs.chunk());
self.buf = Some(buf);
- Ok(size)
+ Poll::Ready(Ok(size))
}
- async fn abort(&mut self) -> Result<()> {
- Err(Error::new(
- ErrorKind::Unsupported,
- "output writer doesn't support abort",
- ))
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
+ if self.future.is_some() {
+ self.future = None;
+ return Poll::Ready(Err(Error::new(
+ ErrorKind::Unexpected,
+ "there is a future on going, it's maybe a bug to go into this case",
+ )));
+ }
+
+ self.buf = None;
+ Poll::Ready(Ok(()))
}
- async fn close(&mut self) -> Result<()> {
- if let Some(buf) = self.buf.as_deref() {
- self.kv.set(&self.path, buf).await?;
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ loop {
+ match self.future.as_mut() {
+ Some(fut) => {
+ ready!(fut.poll_unpin(cx))?;
+ self.future = None;
+ return Poll::Ready(Ok(()));
+ }
+ None => {
+ let kv = self.kv.clone();
+ let path = self.path.clone();
+ let buf = match self.buf.take() {
+ Some(buf) => buf,
+ None => return Poll::Ready(Ok(())),
+ };
+
+ let fut = async move {
+ kv.set(&path, &buf).await?;
+ };
+ self.future = Some(Box::pin(fut));
+ }
+ }
}
-
- Ok(())
}
}
diff --git a/core/src/raw/adapters/typed_kv/backend.rs b/core/src/raw/adapters/typed_kv/backend.rs
index 1e0864de7..4a3cdf37b 100644
--- a/core/src/raw/adapters/typed_kv/backend.rs
+++ b/core/src/raw/adapters/typed_kv/backend.rs
@@ -16,9 +16,11 @@
// under the License.
use std::sync::Arc;
+use std::task::{ready, Context, Poll};
use async_trait::async_trait;
use bytes::Bytes;
+use futures::future::BoxFuture;
use super::Adapter;
use super::Value;
@@ -363,6 +365,7 @@ pub struct KvWriter<S> {
op: OpWrite,
buf: Option<Vec<u8>>,
+ future: Option<BoxFuture<'static, Result<()>>>,
}
impl<S> KvWriter<S> {
@@ -372,6 +375,7 @@ impl<S> KvWriter<S> {
path,
op,
buf: None,
+ future: None,
}
}
@@ -401,7 +405,15 @@ impl<S> KvWriter<S> {
#[async_trait]
impl<S: Adapter> oio::Write for KvWriter<S> {
// TODO: we need to support append in the future.
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
+ if self.future.is_some() {
+ self.future = None;
+ return Poll::Ready(Err(Error::new(
+ ErrorKind::Unexpected,
+ "there is a future on going, it's maybe a bug to go into this case",
+ )));
+ }
+
let size = bs.chunk().len();
let mut buf = self.buf.take().unwrap_or_else(|| Vec::with_capacity(size));
@@ -409,20 +421,42 @@ impl<S: Adapter> oio::Write for KvWriter<S> {
self.buf = Some(buf);
- Ok(size)
+ Poll::Ready(Ok(size))
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
+ if self.future.is_some() {
+ self.future = None;
+ return Poll::Ready(Err(Error::new(
+ ErrorKind::Unexpected,
+ "there is a future on going, it's maybe a bug to go into this case",
+ )));
+ }
+
self.buf = None;
- Ok(())
+ Poll::Ready(Ok(()))
}
- async fn close(&mut self) -> Result<()> {
- let kv = self.kv.clone();
- let value = self.build();
-
- kv.set(&self.path, value).await?;
- Ok(())
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ loop {
+ match self.future.as_mut() {
+ Some(fut) => {
+ ready!(fut.poll_unpin(cx))?;
+ self.future = None;
+ return Poll::Ready(Ok(()));
+ }
+ None => {
+ let kv = self.kv.clone();
+ let path = self.path.clone();
+ let value = self.build();
+
+ let fut = async move {
+ kv.set(&path, value).await?;
+ };
+ self.future = Some(Box::pin(fut));
+ }
+ }
+ }
}
}
diff --git a/core/src/raw/oio/write/api.rs b/core/src/raw/oio/write/api.rs
index 622303ff4..2463068e9 100644
--- a/core/src/raw/oio/write/api.rs
+++ b/core/src/raw/oio/write/api.rs
@@ -17,8 +17,12 @@
use std::fmt::Display;
use std::fmt::Formatter;
+use std::future::Future;
+use std::pin::Pin;
+use std::task::{Context, Poll};
use async_trait::async_trait;
+use pin_project::pin_project;
use crate::raw::*;
use crate::*;
@@ -81,35 +85,33 @@ pub trait Write: Unpin + Send + Sync {
///
/// It's possible that `n < bs.len()`, caller should pass the remaining bytes
/// repeatedly until all bytes has been written.
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize>;
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>>;
/// Abort the pending writer.
- async fn abort(&mut self) -> Result<()>;
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>>;
/// Close the writer and make sure all data has been flushed.
- async fn close(&mut self) -> Result<()>;
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>>;
}
#[async_trait]
impl Write for () {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
- let _ = bs;
-
+ fn poll_write(&mut self, _: &mut Context<'_>, _: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
unimplemented!("write is required to be implemented for oio::Write")
}
- async fn abort(&mut self) -> Result<()> {
- Err(Error::new(
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
+ Poll::Ready(Err(Error::new(
ErrorKind::Unsupported,
"output writer doesn't support abort",
- ))
+ )))
}
- async fn close(&mut self) -> Result<()> {
- Err(Error::new(
+ fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
+ Poll::Ready(Err(Error::new(
ErrorKind::Unsupported,
"output writer doesn't support close",
- ))
+ )))
}
}
@@ -118,16 +120,90 @@ impl Write for () {
/// To make Writer work as expected, we must add this impl.
#[async_trait]
impl<T: Write + ?Sized> Write for Box<T> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
- (**self).write(bs).await
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
+ (**self).poll_write(cx, bs)
+ }
+
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ (**self).poll_abort(cx)
+ }
+
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ (**self).poll_close(cx)
+ }
+}
+
+/// Impl WriteExt for all T: Write
+impl<T: Write> WriteExt for T {}
+
+/// Extension of [`Read`] to make it easier for use.
+pub trait WriteExt: Write {
+ /// Build a future for `poll_write`.
+ fn write<'a>(&'a mut self, buf: &'a dyn oio::WriteBuf) -> WriteFuture<'a, Self> {
+ WriteFuture { writer: self, buf }
+ }
+
+ fn abort(&mut self) -> AbortFuture<Self> {
+ AbortFuture { writer: self }
}
- async fn abort(&mut self) -> Result<()> {
- (**self).abort().await
+ fn close(&mut self) -> CloseFuture<Self> {
+ CloseFuture { writer: self }
}
+}
+
+/// Make this future `!Unpin` for compatibility with async trait methods.
+#[pin_project(!Unpin)]
+pub struct WriteFuture<'a, W: Write + Unpin + ?Sized> {
+ writer: &'a mut W,
+ buf: &'a dyn oio::WriteBuf,
+}
+
+impl<W> Future for WriteFuture<'_, W>
+where
+ W: Write + Unpin + ?Sized,
+{
+ type Output = Result<usize>;
+
+ fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<usize>> {
+ let this = self.project();
+ Pin::new(this.writer).poll_write(cx, this.buf)
+ }
+}
+
+/// Make this future `!Unpin` for compatibility with async trait methods.
+#[pin_project(!Unpin)]
+pub struct AbortFuture<'a, W: Write + Unpin + ?Sized> {
+ writer: &'a mut W,
+}
+
+impl<W> Future for AbortFuture<'_, W>
+where
+ W: Write + Unpin + ?Sized,
+{
+ type Output = Result<usize>;
+
+ fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ let this = self.project();
+ Pin::new(this.writer).poll_abort(cx)
+ }
+}
+
+/// Make this future `!Unpin` for compatibility with async trait methods.
+#[pin_project(!Unpin)]
+pub struct CloseFuture<'a, W: Write + Unpin + ?Sized> {
+ writer: &'a mut W,
+}
+
+impl<W> Future for CloseFuture<'_, W>
+where
+ W: Write + Unpin + ?Sized,
+{
+ type Output = Result<usize>;
- async fn close(&mut self) -> Result<()> {
- (**self).close().await
+ fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ let this = self.project();
+ Pin::new(this.writer).poll_close(cx)
}
}
diff --git a/core/src/raw/oio/write/append_object_write.rs b/core/src/raw/oio/write/append_object_write.rs
index 2a5e30609..ed2a456d9 100644
--- a/core/src/raw/oio/write/append_object_write.rs
+++ b/core/src/raw/oio/write/append_object_write.rs
@@ -77,7 +77,7 @@ impl<W> oio::Write for AppendObjectWriter<W>
where
W: AppendObjectWrite,
{
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let offset = self.offset().await?;
let size = bs.remaining();
@@ -94,11 +94,11 @@ where
Ok(size)
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/raw/oio/write/compose_write.rs b/core/src/raw/oio/write/compose_write.rs
index e2c6638af..c6504ee93 100644
--- a/core/src/raw/oio/write/compose_write.rs
+++ b/core/src/raw/oio/write/compose_write.rs
@@ -55,21 +55,21 @@ pub enum TwoWaysWriter<ONE: oio::Write, TWO: oio::Write> {
#[async_trait]
impl<ONE: oio::Write, TWO: oio::Write> oio::Write for TwoWaysWriter<ONE, TWO> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
match self {
Self::One(one) => one.write(bs).await,
Self::Two(two) => two.write(bs).await,
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
match self {
Self::One(one) => one.abort().await,
Self::Two(two) => two.abort().await,
}
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
match self {
Self::One(one) => one.close().await,
Self::Two(two) => two.close().await,
@@ -93,7 +93,7 @@ pub enum ThreeWaysWriter<ONE: oio::Write, TWO: oio::Write, THREE: oio::Write> {
impl<ONE: oio::Write, TWO: oio::Write, THREE: oio::Write> oio::Write
for ThreeWaysWriter<ONE, TWO, THREE>
{
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
match self {
Self::One(one) => one.write(bs).await,
Self::Two(two) => two.write(bs).await,
@@ -101,7 +101,7 @@ impl<ONE: oio::Write, TWO: oio::Write, THREE: oio::Write> oio::Write
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
match self {
Self::One(one) => one.abort().await,
Self::Two(two) => two.abort().await,
@@ -109,7 +109,7 @@ impl<ONE: oio::Write, TWO: oio::Write, THREE: oio::Write> oio::Write
}
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
match self {
Self::One(one) => one.close().await,
Self::Two(two) => two.close().await,
diff --git a/core/src/raw/oio/write/exact_buf_write.rs b/core/src/raw/oio/write/exact_buf_write.rs
index d4a868472..38e00f057 100644
--- a/core/src/raw/oio/write/exact_buf_write.rs
+++ b/core/src/raw/oio/write/exact_buf_write.rs
@@ -16,11 +16,13 @@
// under the License.
use std::cmp::min;
+use std::task::{ready, Context, Poll};
use async_trait::async_trait;
use bytes::{Bytes, BytesMut};
use crate::raw::oio::WriteBuf;
+use crate::raw::oio::WriteExt;
use crate::raw::*;
use crate::*;
@@ -62,20 +64,20 @@ enum Buffer {
#[async_trait]
impl<W: oio::Write> oio::Write for ExactBufWriter<W> {
- async fn write(&mut self, bs: &dyn WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn WriteBuf) -> Poll<Result<usize>> {
loop {
match &mut self.buffer {
Buffer::Empty => {
if bs.remaining() >= self.buffer_size {
self.buffer = Buffer::Consuming(bs.copy_to_bytes(self.buffer_size));
- return Ok(self.buffer_size);
+ return Poll::Ready(Ok(self.buffer_size));
}
let chunk = bs.chunk();
let mut fill = BytesMut::with_capacity(chunk.len());
fill.extend_from_slice(chunk);
self.buffer = Buffer::Filling(fill);
- return Ok(chunk.len());
+ return Poll::Ready(Ok(chunk.len()));
}
Buffer::Filling(fill) => {
if fill.len() >= self.buffer_size {
@@ -85,14 +87,14 @@ impl<W: oio::Write> oio::Write for ExactBufWriter<W> {
let size = min(self.buffer_size - fill.len(), bs.chunk().len());
fill.extend_from_slice(&bs.chunk()[..size]);
- return Ok(size);
+ return Poll::Ready(Ok(size));
}
Buffer::Consuming(consume) => {
// Make sure filled buffer has been flushed.
//
// TODO: maybe we can re-fill it after a successful write.
while !consume.is_empty() {
- let n = self.inner.write(consume).await?;
+ let n = ready!(self.inner.poll_write(cx, consume)?);
consume.advance(n);
}
self.buffer = Buffer::Empty;
@@ -101,12 +103,12 @@ impl<W: oio::Write> oio::Write for ExactBufWriter<W> {
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.buffer = Buffer::Empty;
- self.inner.abort().await
+ self.inner.poll_abort(cx)
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
loop {
match &mut self.buffer {
Buffer::Empty => break,
@@ -119,7 +121,7 @@ impl<W: oio::Write> oio::Write for ExactBufWriter<W> {
//
// TODO: maybe we can re-fill it after a successful write.
while !consume.is_empty() {
- let n = self.inner.write(&consume).await?;
+ let n = ready!(self.inner.poll_write(cx, &consume))?;
consume.advance(n);
}
self.buffer = Buffer::Empty;
@@ -128,7 +130,7 @@ impl<W: oio::Write> oio::Write for ExactBufWriter<W> {
}
}
- self.inner.close().await
+ self.inner.poll_close(cx)
}
}
@@ -151,21 +153,21 @@ mod tests {
#[async_trait]
impl Write for MockWriter {
- async fn write(&mut self, bs: &dyn WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn WriteBuf) -> Poll<Result<usize>> {
debug!(
"test_fuzz_exact_buf_writer: flush size: {}",
bs.chunk().len()
);
self.buf.extend_from_slice(bs.chunk());
- Ok(bs.chunk().len())
+ Poll::Ready(Ok(bs.chunk().len()))
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/raw/oio/write/mod.rs b/core/src/raw/oio/write/mod.rs
index dfaf6c4ee..ff6d8377e 100644
--- a/core/src/raw/oio/write/mod.rs
+++ b/core/src/raw/oio/write/mod.rs
@@ -19,6 +19,7 @@ mod api;
pub use api::BlockingWrite;
pub use api::BlockingWriter;
pub use api::Write;
+pub use api::WriteExt;
pub use api::WriteOperation;
pub use api::Writer;
diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs
index 39660a2f7..237b5774e 100644
--- a/core/src/raw/oio/write/multipart_upload_write.rs
+++ b/core/src/raw/oio/write/multipart_upload_write.rs
@@ -119,7 +119,7 @@ impl<W> oio::Write for MultipartUploadWriter<W>
where
W: MultipartUploadWrite,
{
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let upload_id = self.upload_id().await?;
let size = bs.remaining();
@@ -137,7 +137,7 @@ where
Ok(size)
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
let upload_id = if let Some(upload_id) = &self.upload_id {
upload_id
} else {
@@ -147,7 +147,7 @@ where
self.inner.complete_part(upload_id, &self.parts).await
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
let upload_id = if let Some(upload_id) = &self.upload_id {
upload_id
} else {
diff --git a/core/src/raw/oio/write/one_shot_write.rs b/core/src/raw/oio/write/one_shot_write.rs
index a02c64445..9d387aedb 100644
--- a/core/src/raw/oio/write/one_shot_write.rs
+++ b/core/src/raw/oio/write/one_shot_write.rs
@@ -48,17 +48,17 @@ impl<W: OneShotWrite> OneShotWriter<W> {
#[async_trait]
impl<W: OneShotWrite> oio::Write for OneShotWriter<W> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
self.inner.write_once(bs).await?;
Ok(size)
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/azblob/writer.rs b/core/src/services/azblob/writer.rs
index 9745b1387..680bc8f53 100644
--- a/core/src/services/azblob/writer.rs
+++ b/core/src/services/azblob/writer.rs
@@ -160,7 +160,7 @@ impl AzblobWriter {
#[async_trait]
impl oio::Write for AzblobWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
if self.op.append() {
@@ -181,11 +181,11 @@ impl oio::Write for AzblobWriter {
Ok(size)
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/azdfs/writer.rs b/core/src/services/azdfs/writer.rs
index f9bec7346..0dfaa320b 100644
--- a/core/src/services/azdfs/writer.rs
+++ b/core/src/services/azdfs/writer.rs
@@ -40,7 +40,7 @@ impl AzdfsWriter {
#[async_trait]
impl oio::Write for AzdfsWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let mut req = self.core.azdfs_create_request(
&self.path,
"file",
@@ -89,11 +89,11 @@ impl oio::Write for AzdfsWriter {
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/dropbox/writer.rs b/core/src/services/dropbox/writer.rs
index c39e95eaf..6331727b1 100644
--- a/core/src/services/dropbox/writer.rs
+++ b/core/src/services/dropbox/writer.rs
@@ -39,7 +39,7 @@ impl DropboxWriter {
#[async_trait]
impl oio::Write for DropboxWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
let resp = self
@@ -61,11 +61,11 @@ impl oio::Write for DropboxWriter {
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/fs/writer.rs b/core/src/services/fs/writer.rs
index 13e03ae2c..fd4a9535b 100644
--- a/core/src/services/fs/writer.rs
+++ b/core/src/services/fs/writer.rs
@@ -43,18 +43,18 @@ impl<F> FsWriter<F> {
#[async_trait]
impl oio::Write for FsWriter<tokio::fs::File> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
self.f.write(bs.chunk()).await.map_err(parse_io_error)
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Err(Error::new(
ErrorKind::Unsupported,
"output writer doesn't support abort",
))
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.f.sync_all().await.map_err(parse_io_error)?;
if let Some(tmp_path) = &self.tmp_path {
diff --git a/core/src/services/ftp/writer.rs b/core/src/services/ftp/writer.rs
index 2b0673d67..f0e61d537 100644
--- a/core/src/services/ftp/writer.rs
+++ b/core/src/services/ftp/writer.rs
@@ -40,7 +40,7 @@ impl FtpWriter {
#[async_trait]
impl oio::Write for FtpWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
let bs = bs.copy_to_bytes(size);
@@ -55,11 +55,11 @@ impl oio::Write for FtpWriter {
Ok(size)
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/gcs/writer.rs b/core/src/services/gcs/writer.rs
index d8dbe7597..9ae9b2fb3 100644
--- a/core/src/services/gcs/writer.rs
+++ b/core/src/services/gcs/writer.rs
@@ -118,7 +118,7 @@ impl GcsWriter {
#[async_trait]
impl oio::Write for GcsWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
let location = match &self.location {
@@ -161,7 +161,7 @@ impl oio::Write for GcsWriter {
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
let location = if let Some(location) = &self.location {
location
} else {
@@ -183,7 +183,7 @@ impl oio::Write for GcsWriter {
}
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
let location = if let Some(location) = &self.location {
location
} else {
diff --git a/core/src/services/gdrive/writer.rs b/core/src/services/gdrive/writer.rs
index c77a8989d..2fc5f229a 100644
--- a/core/src/services/gdrive/writer.rs
+++ b/core/src/services/gdrive/writer.rs
@@ -95,7 +95,7 @@ impl GdriveWriter {
#[async_trait]
impl oio::Write for GdriveWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
if self.file_id.is_none() {
self.write_create(size as u64, bs.copy_to_bytes(size))
@@ -108,11 +108,11 @@ impl oio::Write for GdriveWriter {
Ok(size)
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/ghac/writer.rs b/core/src/services/ghac/writer.rs
index 13b27ccf4..7314d82fc 100644
--- a/core/src/services/ghac/writer.rs
+++ b/core/src/services/ghac/writer.rs
@@ -41,7 +41,7 @@ impl GhacWriter {
#[async_trait]
impl oio::Write for GhacWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
let req = self
@@ -66,11 +66,11 @@ impl oio::Write for GhacWriter {
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
let req = self.backend.ghac_commit(self.cache_id, self.size).await?;
let resp = self.backend.client.send(req).await?;
diff --git a/core/src/services/hdfs/writer.rs b/core/src/services/hdfs/writer.rs
index 47cdb53fe..e5f140dca 100644
--- a/core/src/services/hdfs/writer.rs
+++ b/core/src/services/hdfs/writer.rs
@@ -36,18 +36,18 @@ impl<F> HdfsWriter<F> {
#[async_trait]
impl oio::Write for HdfsWriter<hdrs::AsyncFile> {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
self.f.write(bs.chunk()).await.map_err(parse_io_error)
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Err(Error::new(
ErrorKind::Unsupported,
"output writer doesn't support abort",
))
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
self.f.close().await.map_err(parse_io_error)?;
Ok(())
diff --git a/core/src/services/ipmfs/writer.rs b/core/src/services/ipmfs/writer.rs
index 6b6ab6a4d..2c82ec0c6 100644
--- a/core/src/services/ipmfs/writer.rs
+++ b/core/src/services/ipmfs/writer.rs
@@ -37,7 +37,7 @@ impl IpmfsWriter {
#[async_trait]
impl oio::Write for IpmfsWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
let resp = self
.backend
@@ -55,11 +55,11 @@ impl oio::Write for IpmfsWriter {
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/onedrive/writer.rs b/core/src/services/onedrive/writer.rs
index d9eabb8eb..9d0c6a992 100644
--- a/core/src/services/onedrive/writer.rs
+++ b/core/src/services/onedrive/writer.rs
@@ -45,7 +45,7 @@ impl OneDriveWriter {
#[async_trait]
impl oio::Write for OneDriveWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
let bs = bs.copy_to_bytes(size);
@@ -58,11 +58,11 @@ impl oio::Write for OneDriveWriter {
Ok(size)
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/sftp/writer.rs b/core/src/services/sftp/writer.rs
index 90e605152..9abd24f25 100644
--- a/core/src/services/sftp/writer.rs
+++ b/core/src/services/sftp/writer.rs
@@ -33,20 +33,20 @@ impl SftpWriter {
#[async_trait]
impl oio::Write for SftpWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = self.file.write(bs.chunk()).await?;
Ok(size)
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Err(Error::new(
ErrorKind::Unsupported,
"SFTP does not support aborting writes",
))
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/supabase/writer.rs b/core/src/services/supabase/writer.rs
index 95a2ee92f..8830ca849 100644
--- a/core/src/services/supabase/writer.rs
+++ b/core/src/services/supabase/writer.rs
@@ -67,20 +67,20 @@ impl SupabaseWriter {
#[async_trait]
impl oio::Write for SupabaseWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
self.upload(bs.copy_to_bytes(size)).await?;
Ok(size)
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Err(Error::new(
ErrorKind::Unsupported,
"The abort operation is not yet supported for Supabase backend",
))
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/vercel_artifacts/writer.rs b/core/src/services/vercel_artifacts/writer.rs
index 36e62b734..52acee034 100644
--- a/core/src/services/vercel_artifacts/writer.rs
+++ b/core/src/services/vercel_artifacts/writer.rs
@@ -38,7 +38,7 @@ impl VercelArtifactsWriter {
#[async_trait]
impl oio::Write for VercelArtifactsWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
let resp = self
@@ -61,11 +61,11 @@ impl oio::Write for VercelArtifactsWriter {
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/wasabi/writer.rs b/core/src/services/wasabi/writer.rs
index d0509ebbc..4736e348b 100644
--- a/core/src/services/wasabi/writer.rs
+++ b/core/src/services/wasabi/writer.rs
@@ -40,7 +40,7 @@ impl WasabiWriter {
#[async_trait]
impl oio::Write for WasabiWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
let resp = self
@@ -64,11 +64,11 @@ impl oio::Write for WasabiWriter {
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/webdav/writer.rs b/core/src/services/webdav/writer.rs
index 413fe891a..cccf177ed 100644
--- a/core/src/services/webdav/writer.rs
+++ b/core/src/services/webdav/writer.rs
@@ -61,7 +61,7 @@ impl WebdavWriter {
#[async_trait]
impl oio::Write for WebdavWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
self.write_oneshot(size as u64, AsyncBody::Bytes(bs.copy_to_bytes(size)))
@@ -70,11 +70,11 @@ impl oio::Write for WebdavWriter {
Ok(size)
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/services/webhdfs/writer.rs b/core/src/services/webhdfs/writer.rs
index 34e6b20fa..c3ce0b5be 100644
--- a/core/src/services/webhdfs/writer.rs
+++ b/core/src/services/webhdfs/writer.rs
@@ -38,7 +38,7 @@ impl WebhdfsWriter {
#[async_trait]
impl oio::Write for WebhdfsWriter {
- async fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
let size = bs.remaining();
let req = self
@@ -63,11 +63,11 @@ impl oio::Write for WebhdfsWriter {
}
}
- async fn abort(&mut self) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
- async fn close(&mut self) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
Ok(())
}
}
diff --git a/core/src/types/writer.rs b/core/src/types/writer.rs
index 0474d6ea4..da384729f 100644
--- a/core/src/types/writer.rs
+++ b/core/src/types/writer.rs
@@ -205,7 +205,7 @@ impl Writer {
///
/// Abort should only be called when the writer is not closed or
/// aborted, otherwise an unexpected error could be returned.
- pub async fn abort(&mut self) -> Result<()> {
+ pub fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
if let State::Idle(Some(w)) = &mut self.state {
w.abort().await
} else {
@@ -222,7 +222,7 @@ impl Writer {
///
/// Close should only be called when the writer is not closed or
/// aborted, otherwise an unexpected error could be returned.
- pub async fn close(&mut self) -> Result<()> {
+ pub fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
if let State::Idle(Some(w)) = &mut self.state {
w.close().await
} else {
[incubator-opendal] 17/31: Save
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 7bc8f73da5438d6831b2c7df41c4ebf77ea9073a
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 16:33:42 2023 +0800
Save
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/layers/concurrent_limit.rs | 6 +++---
core/src/raw/oio/write/api.rs | 4 ++--
core/src/services/dropbox/writer.rs | 1 +
core/src/services/gdrive/writer.rs | 1 +
core/src/services/ipmfs/writer.rs | 1 +
core/src/services/onedrive/writer.rs | 1 +
core/src/services/supabase/writer.rs | 1 +
core/src/services/vercel_artifacts/writer.rs | 1 +
core/src/services/wasabi/writer.rs | 1 +
core/src/services/webhdfs/writer.rs | 1 +
core/src/types/writer.rs | 6 +++---
11 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/core/src/layers/concurrent_limit.rs b/core/src/layers/concurrent_limit.rs
index ef620e037..b09408d45 100644
--- a/core/src/layers/concurrent_limit.rs
+++ b/core/src/layers/concurrent_limit.rs
@@ -286,15 +286,15 @@ impl<R: oio::BlockingRead> oio::BlockingRead for ConcurrentLimitWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for ConcurrentLimitWrapper<R> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- self.inner.write(bs).await
+ self.inner.poll_write(cx, bs)
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.abort().await
+ self.inner.poll_abort(cx)
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.close().await
+ self.inner.poll_close(cx)
}
}
diff --git a/core/src/raw/oio/write/api.rs b/core/src/raw/oio/write/api.rs
index 76b534888..41bd93491 100644
--- a/core/src/raw/oio/write/api.rs
+++ b/core/src/raw/oio/write/api.rs
@@ -181,7 +181,7 @@ impl<W> Future for AbortFuture<'_, W>
where
W: Write + Unpin + ?Sized,
{
- type Output = Result<usize>;
+ type Output = Result<()>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
let this = self.project();
@@ -199,7 +199,7 @@ impl<W> Future for CloseFuture<'_, W>
where
W: Write + Unpin + ?Sized,
{
- type Output = Result<usize>;
+ type Output = Result<()>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>> {
let this = self.project();
diff --git a/core/src/services/dropbox/writer.rs b/core/src/services/dropbox/writer.rs
index 5e3e19a92..3a5c6cdd7 100644
--- a/core/src/services/dropbox/writer.rs
+++ b/core/src/services/dropbox/writer.rs
@@ -38,6 +38,7 @@ impl DropboxWriter {
}
}
+#[async_trait]
impl oio::OneShotWrite for DropboxWriter {
async fn write_once(&self, bs: Bytes) -> Result<()> {
let size = bs.len();
diff --git a/core/src/services/gdrive/writer.rs b/core/src/services/gdrive/writer.rs
index f2f0b4e1c..2b0ac8c55 100644
--- a/core/src/services/gdrive/writer.rs
+++ b/core/src/services/gdrive/writer.rs
@@ -87,6 +87,7 @@ impl GdriveWriter {
}
}
+#[async_trait]
impl oio::OneShotWrite for GdriveWriter {
async fn write_once(&self, bs: Bytes) -> Result<()> {
let size = bs.len();
diff --git a/core/src/services/ipmfs/writer.rs b/core/src/services/ipmfs/writer.rs
index e3395dfaf..0d1673693 100644
--- a/core/src/services/ipmfs/writer.rs
+++ b/core/src/services/ipmfs/writer.rs
@@ -37,6 +37,7 @@ impl IpmfsWriter {
}
}
+#[async_trait]
impl oio::OneShotWrite for IpmfsWriter {
async fn write_once(&self, bs: &dyn WriteBuf) -> Result<()> {
let size = bs.remaining();
diff --git a/core/src/services/onedrive/writer.rs b/core/src/services/onedrive/writer.rs
index 76fc734d1..727d8f7d3 100644
--- a/core/src/services/onedrive/writer.rs
+++ b/core/src/services/onedrive/writer.rs
@@ -44,6 +44,7 @@ impl OneDriveWriter {
}
}
+#[async_trait]
impl oio::OneShotWrite for OneDriveWriter {
async fn write_once(&self, bs: Bytes) -> Result<()> {
let size = bs.len();
diff --git a/core/src/services/supabase/writer.rs b/core/src/services/supabase/writer.rs
index 3b692fd08..52292a737 100644
--- a/core/src/services/supabase/writer.rs
+++ b/core/src/services/supabase/writer.rs
@@ -44,6 +44,7 @@ impl SupabaseWriter {
}
}
+#[async_trait]
impl oio::OneShotWrite for SupabaseWriter {
async fn write_once(&self, bs: Bytes) -> Result<()> {
let size = bs.len();
diff --git a/core/src/services/vercel_artifacts/writer.rs b/core/src/services/vercel_artifacts/writer.rs
index 596c933cf..61e5681e0 100644
--- a/core/src/services/vercel_artifacts/writer.rs
+++ b/core/src/services/vercel_artifacts/writer.rs
@@ -37,6 +37,7 @@ impl VercelArtifactsWriter {
}
}
+#[async_trait]
impl oio::OneShotWrite for VercelArtifactsWriter {
async fn write_once(&self, bs: Bytes) -> Result<()> {
let size = bs.len();
diff --git a/core/src/services/wasabi/writer.rs b/core/src/services/wasabi/writer.rs
index 685848084..254810aa9 100644
--- a/core/src/services/wasabi/writer.rs
+++ b/core/src/services/wasabi/writer.rs
@@ -40,6 +40,7 @@ impl WasabiWriter {
}
}
+#[async_trait]
impl oio::OneShotWrite for WasabiWriter {
async fn write_once(&self, bs: Bytes) -> Result<()> {
let size = bs.len();
diff --git a/core/src/services/webhdfs/writer.rs b/core/src/services/webhdfs/writer.rs
index 22901b496..b323c0173 100644
--- a/core/src/services/webhdfs/writer.rs
+++ b/core/src/services/webhdfs/writer.rs
@@ -37,6 +37,7 @@ impl WebhdfsWriter {
}
}
+#[async_trait]
impl oio::OneShotWrite for WebhdfsWriter {
async fn write_once(&self, bs: Bytes) -> Result<()> {
let size = bs.len();
diff --git a/core/src/types/writer.rs b/core/src/types/writer.rs
index 77b88899e..9ab2b69dd 100644
--- a/core/src/types/writer.rs
+++ b/core/src/types/writer.rs
@@ -28,8 +28,8 @@ use futures::AsyncWrite;
use futures::FutureExt;
use futures::TryStreamExt;
-use crate::raw::oio::Write;
use crate::raw::oio::WriteBuf;
+use crate::raw::oio::{Write, WriteExt};
use crate::raw::*;
use crate::*;
@@ -205,7 +205,7 @@ impl Writer {
///
/// Abort should only be called when the writer is not closed or
/// aborted, otherwise an unexpected error could be returned.
- pub fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ pub async fn abort(&mut self) -> Result<()> {
if let State::Idle(Some(w)) = &mut self.state {
w.abort().await
} else {
@@ -222,7 +222,7 @@ impl Writer {
///
/// Close should only be called when the writer is not closed or
/// aborted, otherwise an unexpected error could be returned.
- pub fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ pub async fn close(&mut self) -> Result<()> {
if let State::Idle(Some(w)) = &mut self.state {
w.close().await
} else {
[incubator-opendal] 23/31: Fix azblob
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 90292ce9d95cd3d78853ef0d86a4dfd6e75024ae
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Sat Sep 9 03:12:52 2023 +0800
Fix azblob
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/azblob/backend.rs | 15 ++++---
core/src/services/azblob/core.rs | 15 +------
core/src/services/azblob/writer.rs | 84 +++++++++++--------------------------
3 files changed, 37 insertions(+), 77 deletions(-)
diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs
index 4b04d5c7e..7e5fe259e 100644
--- a/core/src/services/azblob/backend.rs
+++ b/core/src/services/azblob/backend.rs
@@ -38,6 +38,7 @@ use super::pager::AzblobPager;
use super::writer::AzblobWriter;
use crate::raw::*;
use crate::services::azblob::core::AzblobCore;
+use crate::services::azblob::writer::AzblobWriters;
use crate::types::Metadata;
use crate::*;
@@ -506,7 +507,7 @@ pub struct AzblobBackend {
impl Accessor for AzblobBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = AzblobWriter;
+ type Writer = AzblobWriters;
type BlockingWriter = ();
type Pager = AzblobPager;
type BlockingPager = ();
@@ -601,10 +602,14 @@ impl Accessor for AzblobBackend {
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
- Ok((
- RpWrite::default(),
- AzblobWriter::new(self.core.clone(), args, path.to_string()),
- ))
+ let w = AzblobWriter::new(self.core.clone(), args.clone(), path.to_string());
+ let w = if args.append() {
+ AzblobWriters::Two(oio::AppendObjectWriter::new(w))
+ } else {
+ AzblobWriters::One(oio::OneShotWriter::new(w))
+ };
+
+ Ok((RpWrite::default(), w))
}
async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result<RpCopy> {
diff --git a/core/src/services/azblob/core.rs b/core/src/services/azblob/core.rs
index 6382817bb..5eb46f464 100644
--- a/core/src/services/azblob/core.rs
+++ b/core/src/services/azblob/core.rs
@@ -352,12 +352,6 @@ impl AzblobCore {
///
/// - The maximum size of the content could be appended is 4MB.
/// - `Append Block` succeeds only if the blob already exists.
- /// - It does not need to provide append position.
- /// - But it could use append position to verify the content is appended to the right position.
- ///
- /// Since the `appendpos` only returned by the append operation response,
- /// we could not use it when we want to append content to the blob first time.
- /// (The first time of the appender, not the blob)
///
/// # Reference
///
@@ -365,8 +359,8 @@ impl AzblobCore {
pub fn azblob_append_blob_request(
&self,
path: &str,
+ position: u64,
size: u64,
- position: Option<u64>,
body: AsyncBody,
) -> Result<Request<AsyncBody>> {
let p = build_abs_path(&self.root, path);
@@ -385,12 +379,7 @@ impl AzblobCore {
req = req.header(CONTENT_LENGTH, size);
- if let Some(pos) = position {
- req = req.header(
- HeaderName::from_static(constants::X_MS_BLOB_CONDITION_APPENDPOS),
- pos.to_string(),
- );
- }
+ req = req.header(constants::X_MS_BLOB_CONDITION_APPENDPOS, position);
let req = req.body(body).map_err(new_request_build_error)?;
diff --git a/core/src/services/azblob/writer.rs b/core/src/services/azblob/writer.rs
index 15a63caca..029278ad5 100644
--- a/core/src/services/azblob/writer.rs
+++ b/core/src/services/azblob/writer.rs
@@ -16,9 +16,9 @@
// under the License.
use std::sync::Arc;
-use std::task::{Context, Poll};
use async_trait::async_trait;
+use bytes::Bytes;
use http::StatusCode;
use super::core::AzblobCore;
@@ -27,34 +27,32 @@ use crate::raw::*;
use crate::*;
const X_MS_BLOB_TYPE: &str = "x-ms-blob-type";
-const X_MS_BLOB_APPEND_OFFSET: &str = "x-ms-blob-append-offset";
+
+pub type AzblobWriters =
+ oio::TwoWaysWriter<oio::OneShotWriter<AzblobWriter>, oio::AppendObjectWriter<AzblobWriter>>;
pub struct AzblobWriter {
core: Arc<AzblobCore>,
op: OpWrite,
path: String,
-
- position: Option<u64>,
}
impl AzblobWriter {
pub fn new(core: Arc<AzblobCore>, op: OpWrite, path: String) -> Self {
- AzblobWriter {
- core,
- op,
- path,
- position: None,
- }
+ AzblobWriter { core, op, path }
}
+}
- async fn write_oneshot(&self, size: u64, body: AsyncBody) -> Result<()> {
+#[async_trait]
+impl oio::OneShotWrite for AzblobWriter {
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
let mut req = self.core.azblob_put_blob_request(
&self.path,
- Some(size),
+ Some(bs.len() as u64),
self.op.content_type(),
self.op.cache_control(),
- body,
+ AsyncBody::Bytes(bs),
)?;
self.core.sign(&mut req).await?;
@@ -71,13 +69,11 @@ impl AzblobWriter {
_ => Err(parse_error(resp).await?),
}
}
+}
- async fn current_position(&mut self) -> Result<Option<u64>> {
- if let Some(v) = self.position {
- return Ok(Some(v));
- }
-
- // TODO: we should check with current etag to make sure file not changed.
+#[async_trait]
+impl oio::AppendObjectWrite for AzblobWriter {
+ async fn offset(&self) -> Result<u64> {
let resp = self
.core
.azblob_get_blob_properties(&self.path, None, None)
@@ -86,9 +82,6 @@ impl AzblobWriter {
let status = resp.status();
match status {
- // Just check the blob type.
- // If it is not an appendable blob, return an error.
- // We can not get the append position of the blob here.
StatusCode::OK => {
let headers = resp.headers();
let blob_type = headers.get(X_MS_BLOB_TYPE).and_then(|v| v.to_str().ok());
@@ -98,9 +91,9 @@ impl AzblobWriter {
"the blob is not an appendable blob.",
));
}
- Ok(None)
+
+ Ok(parse_content_length(headers)?.unwrap_or_default())
}
- // If the blob is not existing, we need to create one.
StatusCode::NOT_FOUND => {
let mut req = self.core.azblob_init_appendable_blob_request(
&self.path,
@@ -121,20 +114,16 @@ impl AzblobWriter {
return Err(parse_error(resp).await?);
}
}
-
- self.position = Some(0);
- Ok(Some(0))
+ Ok(0)
}
_ => Err(parse_error(resp).await?),
}
}
- async fn append_oneshot(&mut self, size: u64, body: AsyncBody) -> Result<()> {
- let _ = self.current_position().await?;
-
- let mut req =
- self.core
- .azblob_append_blob_request(&self.path, size, self.position, body)?;
+ async fn append(&self, offset: u64, size: u64, body: AsyncBody) -> Result<()> {
+ let mut req = self
+ .core
+ .azblob_append_blob_request(&self.path, offset, size, body)?;
self.core.sign(&mut req).await?;
@@ -143,33 +132,10 @@ impl AzblobWriter {
let status = resp.status();
match status {
StatusCode::CREATED => {
- let headers = resp.headers();
- let position = headers
- .get(X_MS_BLOB_APPEND_OFFSET)
- .and_then(|v| v.to_str().ok())
- .and_then(|v| v.parse::<u64>().ok());
- self.position = position.map(|v| v + size);
- }
- _ => {
- return Err(parse_error(resp).await?);
+ resp.into_body().consume().await?;
+ Ok(())
}
+ _ => Err(parse_error(resp).await?),
}
-
- Ok(())
- }
-}
-
-#[async_trait]
-impl oio::Write for AzblobWriter {
- fn poll_write(&mut self, _: &mut Context<'_>, _: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- todo!()
- }
-
- fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
- Poll::Ready(Ok(()))
- }
-
- fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
- Poll::Ready(Ok(()))
}
}
[incubator-opendal] 26/31: Make clippy happy
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit c5fed25246927b14b0037fd1294b4eae334654e6
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Sep 11 14:16:03 2023 +0800
Make clippy happy
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/oio/write/api.rs | 2 ++
core/src/services/vercel_artifacts/writer.rs | 8 ++++++--
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/core/src/raw/oio/write/api.rs b/core/src/raw/oio/write/api.rs
index c38221675..dfc8ef6ab 100644
--- a/core/src/raw/oio/write/api.rs
+++ b/core/src/raw/oio/write/api.rs
@@ -143,10 +143,12 @@ pub trait WriteExt: Write {
WriteFuture { writer: self, buf }
}
+ /// Build a future for `poll_close`.
fn close(&mut self) -> CloseFuture<Self> {
CloseFuture { writer: self }
}
+ /// Build a future for `poll_abort`.
fn abort(&mut self) -> AbortFuture<Self> {
AbortFuture { writer: self }
}
diff --git a/core/src/services/vercel_artifacts/writer.rs b/core/src/services/vercel_artifacts/writer.rs
index 61e5681e0..f62d76cf5 100644
--- a/core/src/services/vercel_artifacts/writer.rs
+++ b/core/src/services/vercel_artifacts/writer.rs
@@ -26,14 +26,18 @@ use crate::*;
pub struct VercelArtifactsWriter {
backend: VercelArtifactsBackend,
- op: OpWrite,
+ _op: OpWrite,
path: String,
}
impl VercelArtifactsWriter {
pub fn new(backend: VercelArtifactsBackend, op: OpWrite, path: String) -> Self {
- VercelArtifactsWriter { backend, op, path }
+ VercelArtifactsWriter {
+ backend,
+ _op: op,
+ path,
+ }
}
}
[incubator-opendal] 03/31: Save work
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 89d5bed9b71d6352ae9ee8ef42888a573001cf83
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 14:04:14 2023 +0800
Save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/layers/oteltrace.rs | 37 ++---
core/src/raw/oio/write/multipart_upload_write.rs | 181 +++++++++++++++++------
2 files changed, 147 insertions(+), 71 deletions(-)
diff --git a/core/src/layers/oteltrace.rs b/core/src/layers/oteltrace.rs
index df9bc2400..145b997d5 100644
--- a/core/src/layers/oteltrace.rs
+++ b/core/src/layers/oteltrace.rs
@@ -17,6 +17,7 @@
use std::io;
use std::task;
+use std::task::{Context, Poll};
use async_trait::async_trait;
use bytes::Bytes;
@@ -27,7 +28,7 @@ use opentelemetry::trace::FutureExt as TraceFutureExt;
use opentelemetry::trace::Span;
use opentelemetry::trace::TraceContextExt;
use opentelemetry::trace::Tracer;
-use opentelemetry::Context;
+use opentelemetry::Context as TraceContext;
use opentelemetry::KeyValue;
use crate::raw::*;
@@ -89,7 +90,7 @@ impl<A: Accessor> LayeredAccessor for OtelTraceAccessor<A> {
let mut span = tracer.start("create");
span.set_attribute(KeyValue::new("path", path.to_string()));
span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- let cx = Context::current_with_span(span);
+ let cx = TraceContext::current_with_span(span);
self.inner.create_dir(path, args).with_context(cx).await
}
@@ -121,7 +122,7 @@ impl<A: Accessor> LayeredAccessor for OtelTraceAccessor<A> {
span.set_attribute(KeyValue::new("from", from.to_string()));
span.set_attribute(KeyValue::new("to", to.to_string()));
span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- let cx = Context::current_with_span(span);
+ let cx = TraceContext::current_with_span(span);
self.inner().copy(from, to, args).with_context(cx).await
}
@@ -131,7 +132,7 @@ impl<A: Accessor> LayeredAccessor for OtelTraceAccessor<A> {
span.set_attribute(KeyValue::new("from", from.to_string()));
span.set_attribute(KeyValue::new("to", to.to_string()));
span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- let cx = Context::current_with_span(span);
+ let cx = TraceContext::current_with_span(span);
self.inner().rename(from, to, args).with_context(cx).await
}
@@ -140,7 +141,7 @@ impl<A: Accessor> LayeredAccessor for OtelTraceAccessor<A> {
let mut span = tracer.start("stat");
span.set_attribute(KeyValue::new("path", path.to_string()));
span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- let cx = Context::current_with_span(span);
+ let cx = TraceContext::current_with_span(span);
self.inner().stat(path, args).with_context(cx).await
}
@@ -149,7 +150,7 @@ impl<A: Accessor> LayeredAccessor for OtelTraceAccessor<A> {
let mut span = tracer.start("delete");
span.set_attribute(KeyValue::new("path", path.to_string()));
span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- let cx = Context::current_with_span(span);
+ let cx = TraceContext::current_with_span(span);
self.inner().delete(path, args).with_context(cx).await
}
@@ -168,7 +169,7 @@ impl<A: Accessor> LayeredAccessor for OtelTraceAccessor<A> {
let tracer = global::tracer("opendal");
let mut span = tracer.start("batch");
span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- let cx = Context::current_with_span(span);
+ let cx = TraceContext::current_with_span(span);
self.inner().batch(args).with_context(cx).await
}
@@ -177,7 +178,7 @@ impl<A: Accessor> LayeredAccessor for OtelTraceAccessor<A> {
let mut span = tracer.start("presign");
span.set_attribute(KeyValue::new("path", path.to_string()));
span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- let cx = Context::current_with_span(span);
+ let cx = TraceContext::current_with_span(span);
self.inner().presign(path, args).with_context(cx).await
}
@@ -276,23 +277,15 @@ impl<R> OtelTraceWrapper<R> {
}
impl<R: oio::Read> oio::Read for OtelTraceWrapper<R> {
- fn poll_read(
- &mut self,
- cx: &mut task::Context<'_>,
- buf: &mut [u8],
- ) -> task::Poll<Result<usize>> {
+ fn poll_read(&mut self, cx: &mut Context<'_>, buf: &mut [u8]) -> Poll<Result<usize>> {
self.inner.poll_read(cx, buf)
}
- fn poll_seek(
- &mut self,
- cx: &mut task::Context<'_>,
- pos: io::SeekFrom,
- ) -> task::Poll<Result<u64>> {
+ fn poll_seek(&mut self, cx: &mut Context<'_>, pos: io::SeekFrom) -> Poll<Result<u64>> {
self.inner.poll_seek(cx, pos)
}
- fn poll_next(&mut self, cx: &mut task::Context<'_>) -> task::Poll<Option<Result<Bytes>>> {
+ fn poll_next(&mut self, cx: &mut Context<'_>) -> Poll<Option<Result<Bytes>>> {
self.inner.poll_next(cx)
}
}
@@ -314,15 +307,15 @@ impl<R: oio::BlockingRead> oio::BlockingRead for OtelTraceWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for OtelTraceWrapper<R> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- self.inner.write(bs).await
+ self.inner.poll_write(cx, bs)
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.abort().await
+ self.inner.poll_abort(cx)
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.inner.close().await
+ self.inner.poll_close(cx)
}
}
diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs
index 10353714d..55a28e84a 100644
--- a/core/src/raw/oio/write/multipart_upload_write.rs
+++ b/core/src/raw/oio/write/multipart_upload_write.rs
@@ -16,6 +16,9 @@
// under the License.
use async_trait::async_trait;
+use futures::future::BoxFuture;
+use std::sync::Arc;
+use std::task::{Context, Poll};
use crate::raw::*;
use crate::*;
@@ -78,40 +81,31 @@ pub struct MultipartUploadPart {
/// MultipartUploadWriter will implements [`Write`] based on multipart
/// uploads.
-///
-/// ## TODO
-///
-/// - Add threshold for `write_once` to avoid unnecessary multipart uploads.
-/// - Allow users to switch to un-buffered mode if users write 16MiB every time.
pub struct MultipartUploadWriter<W: MultipartUploadWrite> {
- inner: W,
+ state: State<W>,
- upload_id: Option<String>,
+ upload_id: Option<Arc<String>>,
parts: Vec<MultipartUploadPart>,
}
+enum State<W> {
+ Idle(Option<W>),
+ Init(BoxFuture<'static, (W, Result<String>)>),
+ Write(BoxFuture<'static, (W, Result<(usize, MultipartUploadPart)>)>),
+ Close(BoxFuture<'static, (W, Result<()>)>),
+ Abort(BoxFuture<'static, (W, Result<()>)>),
+}
+
impl<W: MultipartUploadWrite> MultipartUploadWriter<W> {
/// Create a new MultipartUploadWriter.
pub fn new(inner: W) -> Self {
Self {
- inner,
+ state: State::Idle(Some(inner)),
upload_id: None,
parts: Vec::new(),
}
}
-
- /// Get the upload id. Initiate a new multipart upload if the upload id is empty.
- pub async fn upload_id(&mut self) -> Result<String> {
- match &self.upload_id {
- Some(upload_id) => Ok(upload_id.to_string()),
- None => {
- let upload_id = self.inner.initiate_part().await?;
- self.upload_id = Some(upload_id.clone());
- Ok(upload_id)
- }
- }
- }
}
#[async_trait]
@@ -120,40 +114,129 @@ where
W: MultipartUploadWrite,
{
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let upload_id = self.upload_id().await?;
-
- let size = bs.remaining();
-
- self.inner
- .write_part(
- &upload_id,
- self.parts.len(),
- size as u64,
- AsyncBody::Bytes(bs.copy_to_bytes(size)),
- )
- .await
- .map(|v| self.parts.push(v))?;
-
- Ok(size)
+ loop {
+ match &mut self.state {
+ State::Idle(w) => {
+ let w = w.take().expect("writer must be valid");
+ match self.upload_id.as_ref() {
+ Some(upload_id) => {
+ let size = bs.remaining();
+ let bs = bs.copy_to_bytes(size);
+ let upload_id = upload_id.clone();
+ let part_number = self.parts.len();
+
+ self.state = State::Write(Box::pin(async move {
+ let part = w
+ .write_part(
+ &upload_id,
+ part_number,
+ size as u64,
+ AsyncBody::Bytes(bs),
+ )
+ .await?;
+
+ (w, Ok((size, part)))
+ }));
+ }
+ None => {
+ self.state = State::Init(Box::pin(async move {
+ let upload_id = w.initiate_part().await;
+ (w, upload_id)
+ }));
+ }
+ }
+ }
+ State::Init(fut) => {
+ let (w, upload_id) = futures::ready!(fut.as_mut().poll(cx));
+ self.state = State::Idle(Some(w));
+ self.upload_id = Some(Arc::new(upload_id?));
+ }
+ State::Write(fut) => {
+ let (w, res) = futures::ready!(fut.as_mut().poll(cx));
+ self.state = State::Idle(Some(w));
+
+ let (written, part) = res?;
+ self.parts.push(part);
+ return Poll::Ready(Ok(written));
+ }
+ State::Close(_) => {
+ unreachable!(
+ "MultipartUploadWriter must not go into State:Close during poll_write"
+ )
+ }
+ State::Abort(_) => {
+ unreachable!(
+ "MultipartUploadWriter must not go into State:Abort during poll_write"
+ )
+ }
+ }
+ }
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- let upload_id = if let Some(upload_id) = &self.upload_id {
- upload_id
- } else {
- return Ok(());
- };
-
- self.inner.complete_part(upload_id, &self.parts).await
+ loop {
+ match &mut self.state {
+ State::Idle(w) => {
+ let w = w.take().expect("writer must be valid");
+ match &self.upload_id {
+ Some(upload_id) => {
+ let parts = self.parts.clone();
+ self.state = State::Close(Box::pin(async move {
+ let res = w.complete_part(&upload_id, &self.parts).await;
+ (w, res)
+ }));
+ }
+ None => return Poll::Ready(Ok(())),
+ }
+ }
+ State::Close(fut) => {
+ let (w, res) = futures::ready!(fut.as_mut().poll(cx));
+ self.state = State::Idle(Some(w));
+ return Poll::Ready(res);
+ }
+ State::Init(_) => unreachable!(
+ "MultipartUploadWriter must not go into State:Init during poll_close"
+ ),
+ State::Write(_) => unreachable!(
+ "MultipartUploadWriter must not go into State:Write during poll_close"
+ ),
+ State::Abort(_) => unreachable!(
+ "MultipartUploadWriter must not go into State:Abort during poll_close"
+ ),
+ }
+ }
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- let upload_id = if let Some(upload_id) = &self.upload_id {
- upload_id
- } else {
- return Ok(());
- };
-
- self.inner.abort_part(upload_id).await
+ loop {
+ match &mut self.state {
+ State::Idle(w) => {
+ let w = w.take().expect("writer must be valid");
+ match &self.upload_id {
+ Some(upload_id) => {
+ self.state = State::Close(Box::pin(async move {
+ let res = w.abort_part(&upload_id).await;
+ (w, res)
+ }));
+ }
+ None => return Poll::Ready(Ok(())),
+ }
+ }
+ State::Abort(fut) => {
+ let (w, res) = futures::ready!(fut.as_mut().poll(cx));
+ self.state = State::Idle(Some(w));
+ return Poll::Ready(res);
+ }
+ State::Init(_) => unreachable!(
+ "MultipartUploadWriter must not go into State:Init during poll_abort"
+ ),
+ State::Write(_) => unreachable!(
+ "MultipartUploadWriter must not go into State:Write during poll_abort"
+ ),
+ State::Close(_) => unreachable!(
+ "MultipartUploadWriter must not go into State:Close during poll_abort"
+ ),
+ }
+ }
}
}
[incubator-opendal] 04/31: Save work
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 0dcbc79a4d1c726a43789e0e92e29cfb7ddcea2e
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 14:33:48 2023 +0800
Save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/oio/write/append_object_write.rs | 68 +++++++++++++++++-------
core/src/raw/oio/write/multipart_upload_write.rs | 6 +--
2 files changed, 53 insertions(+), 21 deletions(-)
diff --git a/core/src/raw/oio/write/append_object_write.rs b/core/src/raw/oio/write/append_object_write.rs
index ba5642dff..2c0bd5555 100644
--- a/core/src/raw/oio/write/append_object_write.rs
+++ b/core/src/raw/oio/write/append_object_write.rs
@@ -16,6 +16,8 @@
// under the License.
use async_trait::async_trait;
+use futures::future::BoxFuture;
+use std::task::{ready, Context, Poll};
use crate::raw::*;
use crate::*;
@@ -46,16 +48,22 @@ pub trait AppendObjectWrite: Send + Sync + Unpin {
///
/// - Allow users to switch to un-buffered mode if users write 16MiB every time.
pub struct AppendObjectWriter<W: AppendObjectWrite> {
- inner: W,
+ state: State<W>,
offset: Option<u64>,
}
+enum State<W> {
+ Idle(Option<W>),
+ Offset(BoxFuture<'static, (W, Result<u64>)>),
+ Append(BoxFuture<'static, (W, Result<usize>)>),
+}
+
impl<W: AppendObjectWrite> AppendObjectWriter<W> {
/// Create a new AppendObjectWriter.
pub fn new(inner: W) -> Self {
Self {
- inner,
+ state: State::Idle(Some(inner)),
offset: None,
}
}
@@ -78,27 +86,51 @@ where
W: AppendObjectWrite,
{
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let offset = self.offset().await?;
-
- let size = bs.remaining();
-
- self.inner
- .append(
- offset,
- size as u64,
- AsyncBody::Bytes(bs.copy_to_bytes(size)),
- )
- .await
- .map(|_| self.offset = Some(offset + size as u64))?;
-
- Ok(size)
+ loop {
+ match &mut self.state {
+ State::Idle(w) => {
+ let w = w.take().expect("writer must be valid");
+ match self.offset {
+ Some(offset) => {
+ let size = bs.remaining();
+ let bs = bs.copy_to_bytes(size);
+
+ self.state = State::Append(Box::pin(async move {
+ w.append(offset, size as u64, AsyncBody::Bytes(bs)).await?;
+
+ (w, Ok(size))
+ }));
+ }
+ None => {
+ self.state = State::Offset(Box::pin(async move {
+ let offset = w.offset().await?;
+
+ (w, Ok(offset))
+ }));
+ }
+ }
+ }
+ State::Offset(fut) => {
+ let (w, offset) = ready!(fut.as_mut().poll(cx));
+ self.state = State::Idle(Some(w));
+ self.offset = Some(offset?);
+ }
+ State::Append(fut) => {
+ let (w, res) = ready!(fut.as_mut().poll(cx));
+ self.state = State::Idle(Some(w));
+
+ let size = res?;
+ return Poll::Ready(Ok(size));
+ }
+ }
+ }
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ Poll::Ready(Ok(()))
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ Poll::Ready(Ok(()))
}
}
diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs
index 55a28e84a..482954dc9 100644
--- a/core/src/raw/oio/write/multipart_upload_write.rs
+++ b/core/src/raw/oio/write/multipart_upload_write.rs
@@ -18,7 +18,7 @@
use async_trait::async_trait;
use futures::future::BoxFuture;
use std::sync::Arc;
-use std::task::{Context, Poll};
+use std::task::{ready, Context, Poll};
use crate::raw::*;
use crate::*;
@@ -147,12 +147,12 @@ where
}
}
State::Init(fut) => {
- let (w, upload_id) = futures::ready!(fut.as_mut().poll(cx));
+ let (w, upload_id) = ready!(fut.as_mut().poll(cx));
self.state = State::Idle(Some(w));
self.upload_id = Some(Arc::new(upload_id?));
}
State::Write(fut) => {
- let (w, res) = futures::ready!(fut.as_mut().poll(cx));
+ let (w, res) = ready!(fut.as_mut().poll(cx));
self.state = State::Idle(Some(w));
let (written, part) = res?;
[incubator-opendal] 13/31: Fix sftp
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 8ad74fb8c43941db0fb454b7937407718e7da58d
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 16:13:47 2023 +0800
Fix sftp
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/sftp/writer.rs | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/core/src/services/sftp/writer.rs b/core/src/services/sftp/writer.rs
index cfe51c7f3..fb1ab0a3e 100644
--- a/core/src/services/sftp/writer.rs
+++ b/core/src/services/sftp/writer.rs
@@ -16,7 +16,10 @@
// under the License.
use async_trait::async_trait;
+use futures::AsyncWrite;
use openssh_sftp_client::file::File;
+use std::pin::Pin;
+use std::task::{Context, Poll};
use crate::raw::oio;
use crate::*;
@@ -34,19 +37,16 @@ impl SftpWriter {
#[async_trait]
impl oio::Write for SftpWriter {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = self.file.write(bs.chunk()).await?;
-
- Ok(size)
+ Pin::new(&mut self.file)
+ .poll_write(cx, bs.chunk())
+ .map_err(Error::from)
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Err(Error::new(
- ErrorKind::Unsupported,
- "SFTP does not support aborting writes",
- ))
+ Poll::Ready(Ok(()))
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ Pin::new(&mut self.file).poll_flush(cx).map_err(Error::from)
}
}
[incubator-opendal] 20/31: Build pass
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 919d554ce0a627455ae85ab2b4cc5ad256d168f2
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 17:27:08 2023 +0800
Build pass
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/benches/oio/utils.rs | 13 ++---
core/benches/oio/write.rs | 2 +-
core/src/raw/oio/write/api.rs | 4 +-
core/src/raw/oio/write/append_object_write.rs | 11 ++---
core/src/raw/oio/write/exact_buf_write.rs | 4 +-
core/src/raw/oio/write/multipart_upload_write.rs | 22 ++++-----
core/src/raw/oio/write/one_shot_write.rs | 4 +-
core/src/services/fs/writer.rs | 4 +-
core/src/services/ftp/writer.rs | 3 +-
core/src/services/ghac/writer.rs | 60 +++++++++++++-----------
core/src/services/sftp/writer.rs | 11 +++--
11 files changed, 74 insertions(+), 64 deletions(-)
diff --git a/core/benches/oio/utils.rs b/core/benches/oio/utils.rs
index 67b4fd451..b3ea04027 100644
--- a/core/benches/oio/utils.rs
+++ b/core/benches/oio/utils.rs
@@ -20,6 +20,7 @@ use bytes::Bytes;
use opendal::raw::oio;
use rand::prelude::ThreadRng;
use rand::RngCore;
+use std::task::{Context, Poll};
/// BlackHoleWriter will discard all data written to it so we can measure the buffer's cost.
pub struct BlackHoleWriter;
@@ -30,16 +31,16 @@ impl oio::Write for BlackHoleWriter {
&mut self,
cx: &mut Context<'_>,
bs: &dyn oio::WriteBuf,
- ) -> opendal::Result<usize> {
- Ok(bs.remaining())
+ ) -> Poll<opendal::Result<usize>> {
+ Poll::Ready(Ok(bs.remaining()))
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> opendal::Result<()> {
- Ok(())
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<opendal::Result<()>> {
+ Poll::Ready(Ok(()))
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> opendal::Result<()> {
- Ok(())
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<opendal::Result<()>> {
+ Poll::Ready(Ok(()))
}
}
diff --git a/core/benches/oio/write.rs b/core/benches/oio/write.rs
index 8da2a6dd0..3ab71cdcf 100644
--- a/core/benches/oio/write.rs
+++ b/core/benches/oio/write.rs
@@ -18,8 +18,8 @@
use bytes::Buf;
use criterion::Criterion;
use once_cell::sync::Lazy;
-use opendal::raw::oio::ExactBufWriter;
use opendal::raw::oio::Write;
+use opendal::raw::oio::{ExactBufWriter, WriteExt};
use rand::thread_rng;
use size::Size;
diff --git a/core/src/raw/oio/write/api.rs b/core/src/raw/oio/write/api.rs
index 41bd93491..c05f061c7 100644
--- a/core/src/raw/oio/write/api.rs
+++ b/core/src/raw/oio/write/api.rs
@@ -166,8 +166,8 @@ where
type Output = Result<usize>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<usize>> {
- let this = self.project();
- Pin::new(this.writer).poll_write(cx, this.buf)
+ let mut this = self.project();
+ Pin::new(this.writer).poll_write(cx, *this.buf)
}
}
diff --git a/core/src/raw/oio/write/append_object_write.rs b/core/src/raw/oio/write/append_object_write.rs
index 393254097..5cbaedb37 100644
--- a/core/src/raw/oio/write/append_object_write.rs
+++ b/core/src/raw/oio/write/append_object_write.rs
@@ -32,7 +32,7 @@ use crate::*;
/// - `AppendObjectWriter` impl `Write`
/// - Expose `AppendObjectWriter` as `Accessor::Writer`
#[async_trait]
-pub trait AppendObjectWrite: Send + Sync + Unpin {
+pub trait AppendObjectWrite: Send + Sync + Unpin + 'static {
/// Get the current offset of the append object.
///
/// Returns `0` if the object is not exist.
@@ -90,9 +90,9 @@ where
let bs = bs.copy_to_bytes(size);
self.state = State::Append(Box::pin(async move {
- w.append(offset, size as u64, AsyncBody::Bytes(bs)).await?;
+ let res = w.append(offset, size as u64, AsyncBody::Bytes(bs)).await;
- (w, Ok(size))
+ (w, res.map(|_| size))
}));
}
None => {
@@ -110,11 +110,10 @@ where
self.offset = Some(offset?);
}
State::Append(fut) => {
- let (w, res) = ready!(fut.as_mut().poll(cx));
+ let (w, size) = ready!(fut.as_mut().poll(cx));
self.state = State::Idle(Some(w));
- let size = res?;
- return Poll::Ready(Ok(size));
+ return Poll::Ready(Ok(size?));
}
}
}
diff --git a/core/src/raw/oio/write/exact_buf_write.rs b/core/src/raw/oio/write/exact_buf_write.rs
index f1615346b..636242280 100644
--- a/core/src/raw/oio/write/exact_buf_write.rs
+++ b/core/src/raw/oio/write/exact_buf_write.rs
@@ -164,11 +164,11 @@ mod tests {
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ Poll::Ready(Ok(()))
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ Poll::Ready(Ok(()))
}
}
diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs
index 97b2f0220..af6f825ac 100644
--- a/core/src/raw/oio/write/multipart_upload_write.rs
+++ b/core/src/raw/oio/write/multipart_upload_write.rs
@@ -33,7 +33,7 @@ use crate::*;
/// - `MultipartUploadWriter` impl `Write`
/// - Expose `MultipartUploadWriter` as `Accessor::Writer`
#[async_trait]
-pub trait MultipartUploadWrite: Send + Sync + Unpin {
+pub trait MultipartUploadWrite: Send + Sync + Unpin + 'static {
/// initiate_part will call start a multipart upload and return the upload id.
///
/// MultipartUploadWriter will call this when:
@@ -72,6 +72,7 @@ pub trait MultipartUploadWrite: Send + Sync + Unpin {
///
/// - `part_number` is the index of the part, starting from 0.
/// - `etag` is the `ETag` of the part.
+#[derive(Clone)]
pub struct MultipartUploadPart {
/// The number of the part, starting from 0.
pub part_number: usize,
@@ -91,7 +92,7 @@ pub struct MultipartUploadWriter<W: MultipartUploadWrite> {
enum State<W> {
Idle(Option<W>),
Init(BoxFuture<'static, (W, Result<String>)>),
- Write(BoxFuture<'static, (W, Result<(usize, MultipartUploadPart)>)>),
+ Write(BoxFuture<'static, (W, usize, Result<MultipartUploadPart>)>),
Close(BoxFuture<'static, (W, Result<()>)>),
Abort(BoxFuture<'static, (W, Result<()>)>),
}
@@ -138,9 +139,9 @@ where
size as u64,
AsyncBody::Bytes(bs),
)
- .await?;
+ .await;
- (w, Ok((size, part)))
+ (w, size, part)
}));
}
None => {
@@ -157,12 +158,11 @@ where
self.upload_id = Some(Arc::new(upload_id?));
}
State::Write(fut) => {
- let (w, res) = ready!(fut.as_mut().poll(cx));
+ let (w, size, part) = ready!(fut.as_mut().poll(cx));
self.state = State::Idle(Some(w));
- let (written, part) = res?;
- self.parts.push(part);
- return Poll::Ready(Ok(written));
+ self.parts.push(part?);
+ return Poll::Ready(Ok(size));
}
State::Close(_) => {
unreachable!(
@@ -183,11 +183,11 @@ where
match &mut self.state {
State::Idle(w) => {
let w = w.take().expect("writer must be valid");
- match &self.upload_id {
+ match self.upload_id.clone() {
Some(upload_id) => {
let parts = self.parts.clone();
self.state = State::Close(Box::pin(async move {
- let res = w.complete_part(&upload_id, &self.parts).await;
+ let res = w.complete_part(&upload_id, &parts).await;
(w, res)
}));
}
@@ -217,7 +217,7 @@ where
match &mut self.state {
State::Idle(w) => {
let w = w.take().expect("writer must be valid");
- match &self.upload_id {
+ match self.upload_id.clone() {
Some(upload_id) => {
self.state = State::Close(Box::pin(async move {
let res = w.abort_part(&upload_id).await;
diff --git a/core/src/raw/oio/write/one_shot_write.rs b/core/src/raw/oio/write/one_shot_write.rs
index 6cc083491..b798da5a4 100644
--- a/core/src/raw/oio/write/one_shot_write.rs
+++ b/core/src/raw/oio/write/one_shot_write.rs
@@ -30,7 +30,7 @@ use crate::*;
///
/// The layout after adopting [`OneShotWrite`]:
#[async_trait]
-pub trait OneShotWrite: Send + Sync + Unpin {
+pub trait OneShotWrite: Send + Sync + Unpin + 'static {
/// write_once write all data at once.
///
/// Implementations should make sure that the data is written correctly at once.
@@ -71,7 +71,7 @@ impl<W: OneShotWrite> oio::Write for OneShotWriter<W> {
let size = bs.remaining();
let bs = bs.copy_to_bytes(size);
- let fut = async {
+ let fut = async move {
let res = w.write_once(bs).await;
(w, res.map(|_| size))
diff --git a/core/src/services/fs/writer.rs b/core/src/services/fs/writer.rs
index 1a0aa49eb..26105115b 100644
--- a/core/src/services/fs/writer.rs
+++ b/core/src/services/fs/writer.rs
@@ -42,7 +42,7 @@ impl<F> FsWriter<F> {
Self {
target_path,
tmp_path,
- f,
+ f: Some(f),
fut: None,
}
}
@@ -80,7 +80,7 @@ impl oio::Write for FsWriter<tokio::fs::File> {
let f = self.f.take().expect("FsWriter must be initialized");
let tmp_path = self.tmp_path.clone();
let target_path = self.target_path.clone();
- self.fut = Some(Box::pin(async {
+ self.fut = Some(Box::pin(async move {
f.sync_all().await.map_err(parse_io_error)?;
if let Some(tmp_path) = &tmp_path {
diff --git a/core/src/services/ftp/writer.rs b/core/src/services/ftp/writer.rs
index ec3c3f0e2..5a759aa21 100644
--- a/core/src/services/ftp/writer.rs
+++ b/core/src/services/ftp/writer.rs
@@ -18,6 +18,7 @@
use async_trait::async_trait;
use futures::future::BoxFuture;
use futures::AsyncWriteExt;
+use futures::FutureExt;
use std::task::{ready, Context, Poll};
use super::backend::FtpBackend;
@@ -66,7 +67,7 @@ impl oio::Write for FtpWriter {
let path = self.path.clone();
let backend = self.backend.clone();
- let fut = async {
+ let fut = async move {
let mut ftp_stream = backend.ftp_connect(Operation::Write).await?;
let mut data_stream = ftp_stream.append_with_stream(&path).await?;
data_stream.write_all(&bs).await.map_err(|err| {
diff --git a/core/src/services/ghac/writer.rs b/core/src/services/ghac/writer.rs
index c780c2710..582a150aa 100644
--- a/core/src/services/ghac/writer.rs
+++ b/core/src/services/ghac/writer.rs
@@ -64,21 +64,24 @@ impl oio::Write for GhacWriter {
let size = bs.remaining();
let bs = bs.copy_to_bytes(size);
- let fut = async {
- let req = backend
- .ghac_upload(cache_id, size as u64, AsyncBody::Bytes(bs))
- .await?;
-
- let resp = backend.client.send(req).await?;
-
- let res = if resp.status().is_success() {
- resp.into_body().consume().await?;
- Ok(size)
- } else {
- Err(parse_error(resp)
- .await
- .map(|err| err.with_operation("Backend::ghac_upload"))?)
- };
+ let fut = async move {
+ let res = async {
+ let req = backend
+ .ghac_upload(cache_id, size as u64, AsyncBody::Bytes(bs))
+ .await?;
+
+ let resp = backend.client.send(req).await?;
+
+ if resp.status().is_success() {
+ resp.into_body().consume().await?;
+ Ok(size)
+ } else {
+ Err(parse_error(resp)
+ .await
+ .map(|err| err.with_operation("Backend::ghac_upload"))?)
+ }
+ }
+ .await;
(backend, res)
};
@@ -114,18 +117,21 @@ impl oio::Write for GhacWriter {
let cache_id = self.cache_id;
let size = self.size;
- let fut = async {
- let req = backend.ghac_commit(cache_id, size).await?;
- let resp = backend.client.send(req).await?;
-
- let res = if resp.status().is_success() {
- resp.into_body().consume().await?;
- Ok(())
- } else {
- Err(parse_error(resp)
- .await
- .map(|err| err.with_operation("Backend::ghac_commit"))?)
- };
+ let fut = async move {
+ let res = async {
+ let req = backend.ghac_commit(cache_id, size).await?;
+ let resp = backend.client.send(req).await?;
+
+ if resp.status().is_success() {
+ resp.into_body().consume().await?;
+ Ok(size as usize)
+ } else {
+ Err(parse_error(resp)
+ .await
+ .map(|err| err.with_operation("Backend::ghac_commit"))?)
+ }
+ }
+ .await;
(backend, res)
};
diff --git a/core/src/services/sftp/writer.rs b/core/src/services/sftp/writer.rs
index fb1ab0a3e..455e0fdbc 100644
--- a/core/src/services/sftp/writer.rs
+++ b/core/src/services/sftp/writer.rs
@@ -37,9 +37,11 @@ impl SftpWriter {
#[async_trait]
impl oio::Write for SftpWriter {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- Pin::new(&mut self.file)
- .poll_write(cx, bs.chunk())
- .map_err(Error::from)
+ // Pin::new(&mut self.file)
+ // .poll_write(cx, bs.chunk())
+ // .map_err(Error::from)
+
+ todo!()
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
@@ -47,6 +49,7 @@ impl oio::Write for SftpWriter {
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Pin::new(&mut self.file).poll_flush(cx).map_err(Error::from)
+ // Pin::new(&mut self.file).poll_flush(cx).map_err(Error::from)
+ todo!()
}
}
[incubator-opendal] 10/31: Fix build
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit cdc4c877163fe0fc20bc80ffea46f7368b5075a3
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 16:06:39 2023 +0800
Fix build
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/ghac/writer.rs | 123 ++++++++++++++++++++++++++++-----------
core/src/services/hdfs/writer.rs | 1 +
2 files changed, 90 insertions(+), 34 deletions(-)
diff --git a/core/src/services/ghac/writer.rs b/core/src/services/ghac/writer.rs
index 886abffb5..0be63ce22 100644
--- a/core/src/services/ghac/writer.rs
+++ b/core/src/services/ghac/writer.rs
@@ -16,6 +16,8 @@
// under the License.
use async_trait::async_trait;
+use futures::future::BoxFuture;
+use std::task::{ready, Context, Poll};
use super::backend::GhacBackend;
use super::error::parse_error;
@@ -23,7 +25,7 @@ use crate::raw::*;
use crate::*;
pub struct GhacWriter {
- backend: GhacBackend,
+ state: State,
cache_id: i64,
size: u64,
@@ -32,55 +34,108 @@ pub struct GhacWriter {
impl GhacWriter {
pub fn new(backend: GhacBackend, cache_id: i64) -> Self {
GhacWriter {
- backend,
+ state: State::Idle(Some(backend)),
cache_id,
size: 0,
}
}
}
+enum State {
+ Idle(Option<GhacBackend>),
+ Upload(BoxFuture<'static, (GhacBackend, Result<usize>)>),
+ Commit(BoxFuture<'static, (GhacBackend, Result<()>)>),
+}
+
#[async_trait]
impl oio::Write for GhacWriter {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
-
- let req = self
- .backend
- .ghac_upload(
- self.cache_id,
- size as u64,
- AsyncBody::Bytes(bs.copy_to_bytes(size)),
- )
- .await?;
-
- let resp = self.backend.client.send(req).await?;
-
- if resp.status().is_success() {
- resp.into_body().consume().await?;
- self.size += size as u64;
- Ok(size)
- } else {
- Err(parse_error(resp)
- .await
- .map(|err| err.with_operation("Backend::ghac_upload"))?)
+ loop {
+ match &mut self.state {
+ State::Idle(backend) => {
+ let backend = backend.take().expect("GhacWriter must be initialized");
+
+ let cache_id = self.cache_id;
+ let size = bs.remaining();
+ let bs = bs.copy_to_bytes(size);
+
+ let fut = async {
+ let req = backend
+ .ghac_upload(cache_id, size as u64, AsyncBody::Bytes(bs))
+ .await?;
+
+ let resp = backend.client.send(req).await?;
+
+ let res = if resp.status().is_success() {
+ resp.into_body().consume().await?;
+ Ok(size)
+ } else {
+ Err(parse_error(resp)
+ .await
+ .map(|err| err.with_operation("Backend::ghac_upload"))?)
+ };
+
+ (backend, res)
+ };
+ self.state = State::Upload(Box::pin(fut));
+ }
+ State::Upload(fut) => {
+ let (backend, res) = ready!(fut.as_mut().poll(cx));
+ self.state = State::Idle(Some(backend));
+
+ let size = res?;
+ self.size += size as u64;
+ return Poll::Ready(Ok(size));
+ }
+ State::Commit(_) => {
+ unreachable!("GhacWriter must not go into State:Commit during poll_write")
+ }
+ }
}
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ self.state = State::Idle(None);
+
+ Poll::Ready(Ok(()))
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- let req = self.backend.ghac_commit(self.cache_id, self.size).await?;
- let resp = self.backend.client.send(req).await?;
-
- if resp.status().is_success() {
- resp.into_body().consume().await?;
- Ok(())
- } else {
- Err(parse_error(resp)
- .await
- .map(|err| err.with_operation("Backend::ghac_commit"))?)
+ loop {
+ match &mut self.state {
+ State::Idle(backend) => {
+ let backend = backend.take().expect("GhacWriter must be initialized");
+
+ let cache_id = self.cache_id;
+ let size = self.size;
+
+ let fut = async {
+ let req = backend.ghac_commit(cache_id, size).await?;
+ let resp = backend.client.send(req).await?;
+
+ let res = if resp.status().is_success() {
+ resp.into_body().consume().await?;
+ Ok(())
+ } else {
+ Err(parse_error(resp)
+ .await
+ .map(|err| err.with_operation("Backend::ghac_commit"))?)
+ };
+
+ (backend, res)
+ };
+ self.state = State::Upload(Box::pin(fut));
+ }
+ State::Upload(_) => {
+ unreachable!("GhacWriter must not go into State:Upload during poll_close")
+ }
+ State::Commit(fut) => {
+ let (backend, res) = ready!(fut.as_mut().poll(cx));
+ self.state = State::Idle(Some(backend));
+
+ return Poll::Ready(res);
+ }
+ }
}
}
}
diff --git a/core/src/services/hdfs/writer.rs b/core/src/services/hdfs/writer.rs
index 4c69d63dc..1b4f549ce 100644
--- a/core/src/services/hdfs/writer.rs
+++ b/core/src/services/hdfs/writer.rs
@@ -16,6 +16,7 @@
// under the License.
use std::io::Write;
+use std::task::Context;
use async_trait::async_trait;
use futures::AsyncWriteExt;
[incubator-opendal] 02/31: Save work
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 91a30b252d37ba80e7f96209f063a0d6261d2d43
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 11:00:44 2023 +0800
Save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/layers/complete.rs | 6 ++--
core/src/layers/concurrent_limit.rs | 6 ++--
core/src/layers/error_context.rs | 6 ++--
core/src/layers/logging.rs | 6 ++--
core/src/layers/metrics.rs | 6 ++--
core/src/layers/minitrace.rs | 6 ++--
core/src/layers/oteltrace.rs | 6 ++--
core/src/layers/prometheus.rs | 6 ++--
core/src/layers/retry.rs | 6 ++--
core/src/layers/throttle.rs | 6 ++--
core/src/layers/timeout.rs | 6 ++--
core/src/layers/tracing.rs | 6 ++--
core/src/raw/oio/write/api.rs | 30 ++++++++---------
core/src/raw/oio/write/append_object_write.rs | 6 ++--
core/src/raw/oio/write/compose_write.rs | 43 ++++++++++++------------
core/src/raw/oio/write/exact_buf_write.rs | 4 +--
core/src/raw/oio/write/multipart_upload_write.rs | 6 ++--
core/src/raw/oio/write/one_shot_write.rs | 6 ++--
core/src/services/azblob/writer.rs | 6 ++--
core/src/services/azdfs/writer.rs | 6 ++--
core/src/services/dropbox/writer.rs | 6 ++--
core/src/services/fs/writer.rs | 6 ++--
core/src/services/ftp/writer.rs | 6 ++--
core/src/services/gcs/writer.rs | 6 ++--
core/src/services/gdrive/writer.rs | 6 ++--
core/src/services/ghac/writer.rs | 6 ++--
core/src/services/hdfs/writer.rs | 6 ++--
core/src/services/ipmfs/writer.rs | 6 ++--
core/src/services/onedrive/writer.rs | 6 ++--
core/src/services/sftp/writer.rs | 6 ++--
core/src/services/supabase/writer.rs | 6 ++--
core/src/services/vercel_artifacts/writer.rs | 6 ++--
core/src/services/wasabi/writer.rs | 6 ++--
core/src/services/webdav/writer.rs | 6 ++--
core/src/services/webhdfs/writer.rs | 6 ++--
core/src/types/writer.rs | 4 +--
36 files changed, 137 insertions(+), 136 deletions(-)
diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs
index 19429cff8..3e9e2bba9 100644
--- a/core/src/layers/complete.rs
+++ b/core/src/layers/complete.rs
@@ -711,7 +711,7 @@ impl<W> oio::Write for CompleteWriter<W>
where
W: oio::Write,
{
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let w = self.inner.as_mut().ok_or_else(|| {
Error::new(ErrorKind::Unexpected, "writer has been closed or aborted")
})?;
@@ -733,7 +733,7 @@ where
Ok(n)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
let w = self.inner.as_mut().ok_or_else(|| {
Error::new(ErrorKind::Unexpected, "writer has been closed or aborted")
})?;
@@ -744,7 +744,7 @@ where
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
if let Some(size) = self.size {
if self.written < size {
return Err(Error::new(
diff --git a/core/src/layers/concurrent_limit.rs b/core/src/layers/concurrent_limit.rs
index f726d2207..ef620e037 100644
--- a/core/src/layers/concurrent_limit.rs
+++ b/core/src/layers/concurrent_limit.rs
@@ -285,15 +285,15 @@ impl<R: oio::BlockingRead> oio::BlockingRead for ConcurrentLimitWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for ConcurrentLimitWrapper<R> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
self.inner.write(bs).await
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.abort().await
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.close().await
}
}
diff --git a/core/src/layers/error_context.rs b/core/src/layers/error_context.rs
index c6b3bd2be..7ce05fc18 100644
--- a/core/src/layers/error_context.rs
+++ b/core/src/layers/error_context.rs
@@ -403,7 +403,7 @@ impl<T: oio::BlockingRead> oio::BlockingRead for ErrorContextWrapper<T> {
#[async_trait::async_trait]
impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
self.inner.write(bs).await.map_err(|err| {
err.with_operation(WriteOperation::Write)
.with_context("service", self.scheme)
@@ -411,7 +411,7 @@ impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
})
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.abort().await.map_err(|err| {
err.with_operation(WriteOperation::Abort)
.with_context("service", self.scheme)
@@ -419,7 +419,7 @@ impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
})
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.close().await.map_err(|err| {
err.with_operation(WriteOperation::Close)
.with_context("service", self.scheme)
diff --git a/core/src/layers/logging.rs b/core/src/layers/logging.rs
index f55bbd627..7da2da55b 100644
--- a/core/src/layers/logging.rs
+++ b/core/src/layers/logging.rs
@@ -1252,7 +1252,7 @@ impl<W> LoggingWriter<W> {
#[async_trait]
impl<W: oio::Write> oio::Write for LoggingWriter<W> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
match self.inner.write(bs).await {
Ok(n) => {
self.written += n as u64;
@@ -1285,7 +1285,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
match self.inner.abort().await {
Ok(_) => {
trace!(
@@ -1316,7 +1316,7 @@ impl<W: oio::Write> oio::Write for LoggingWriter<W> {
}
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
match self.inner.close().await {
Ok(_) => {
debug!(
diff --git a/core/src/layers/metrics.rs b/core/src/layers/metrics.rs
index 7a7b73fc4..3cebf46b1 100644
--- a/core/src/layers/metrics.rs
+++ b/core/src/layers/metrics.rs
@@ -847,7 +847,7 @@ impl<R: oio::BlockingRead> oio::BlockingRead for MetricWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for MetricWrapper<R> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
self.inner
.write(bs)
.await
@@ -861,14 +861,14 @@ impl<R: oio::Write> oio::Write for MetricWrapper<R> {
})
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.abort().await.map_err(|err| {
self.handle.increment_errors_total(self.op, err.kind());
err
})
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.close().await.map_err(|err| {
self.handle.increment_errors_total(self.op, err.kind());
err
diff --git a/core/src/layers/minitrace.rs b/core/src/layers/minitrace.rs
index 8d8662846..7c6fad770 100644
--- a/core/src/layers/minitrace.rs
+++ b/core/src/layers/minitrace.rs
@@ -337,7 +337,7 @@ impl<R: oio::BlockingRead> oio::BlockingRead for MinitraceWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for MinitraceWrapper<R> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
self.inner
.write(bs)
.in_span(Span::enter_with_parent(
@@ -347,7 +347,7 @@ impl<R: oio::Write> oio::Write for MinitraceWrapper<R> {
.await
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner
.abort()
.in_span(Span::enter_with_parent(
@@ -357,7 +357,7 @@ impl<R: oio::Write> oio::Write for MinitraceWrapper<R> {
.await
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner
.close()
.in_span(Span::enter_with_parent(
diff --git a/core/src/layers/oteltrace.rs b/core/src/layers/oteltrace.rs
index f0a8ae82b..df9bc2400 100644
--- a/core/src/layers/oteltrace.rs
+++ b/core/src/layers/oteltrace.rs
@@ -313,15 +313,15 @@ impl<R: oio::BlockingRead> oio::BlockingRead for OtelTraceWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for OtelTraceWrapper<R> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
self.inner.write(bs).await
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.abort().await
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.close().await
}
}
diff --git a/core/src/layers/prometheus.rs b/core/src/layers/prometheus.rs
index 98637752b..b3f704e6d 100644
--- a/core/src/layers/prometheus.rs
+++ b/core/src/layers/prometheus.rs
@@ -662,7 +662,7 @@ impl<R: oio::BlockingRead> oio::BlockingRead for PrometheusMetricWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for PrometheusMetricWrapper<R> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
self.inner
.write(bs)
.await
@@ -679,14 +679,14 @@ impl<R: oio::Write> oio::Write for PrometheusMetricWrapper<R> {
})
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.abort().await.map_err(|err| {
self.stats.increment_errors_total(self.op, err.kind());
err
})
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.close().await.map_err(|err| {
self.stats.increment_errors_total(self.op, err.kind());
err
diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs
index 3a1c5521f..6a5dba57c 100644
--- a/core/src/layers/retry.rs
+++ b/core/src/layers/retry.rs
@@ -872,7 +872,7 @@ impl<R: oio::BlockingRead, I: RetryInterceptor> oio::BlockingRead for RetryWrapp
#[async_trait]
impl<R: oio::Write, I: RetryInterceptor> oio::Write for RetryWrapper<R, I> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let mut backoff = self.builder.build();
loop {
@@ -898,7 +898,7 @@ impl<R: oio::Write, I: RetryInterceptor> oio::Write for RetryWrapper<R, I> {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
let mut backoff = self.builder.build();
loop {
@@ -924,7 +924,7 @@ impl<R: oio::Write, I: RetryInterceptor> oio::Write for RetryWrapper<R, I> {
}
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
let mut backoff = self.builder.build();
loop {
diff --git a/core/src/layers/throttle.rs b/core/src/layers/throttle.rs
index 7787c77d8..f731ab83c 100644
--- a/core/src/layers/throttle.rs
+++ b/core/src/layers/throttle.rs
@@ -216,7 +216,7 @@ impl<R: oio::BlockingRead> oio::BlockingRead for ThrottleWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for ThrottleWrapper<R> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let buf_length = NonZeroU32::new(bs.remaining() as u32).unwrap();
loop {
@@ -241,11 +241,11 @@ impl<R: oio::Write> oio::Write for ThrottleWrapper<R> {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.abort().await
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.close().await
}
}
diff --git a/core/src/layers/timeout.rs b/core/src/layers/timeout.rs
index 751caa915..5871818b3 100644
--- a/core/src/layers/timeout.rs
+++ b/core/src/layers/timeout.rs
@@ -322,7 +322,7 @@ impl<R: oio::Read> oio::Read for TimeoutWrapper<R> {
#[async_trait]
impl<R: oio::Write> oio::Write for TimeoutWrapper<R> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let timeout = self.io_timeout(bs.remaining() as u64);
tokio::time::timeout(timeout, self.inner.write(bs))
@@ -335,7 +335,7 @@ impl<R: oio::Write> oio::Write for TimeoutWrapper<R> {
})?
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
tokio::time::timeout(self.timeout, self.inner.abort())
.await
.map_err(|_| {
@@ -346,7 +346,7 @@ impl<R: oio::Write> oio::Write for TimeoutWrapper<R> {
})?
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
tokio::time::timeout(self.timeout, self.inner.close())
.await
.map_err(|_| {
diff --git a/core/src/layers/tracing.rs b/core/src/layers/tracing.rs
index 35f718cb5..cb71388cb 100644
--- a/core/src/layers/tracing.rs
+++ b/core/src/layers/tracing.rs
@@ -324,7 +324,7 @@ impl<R: oio::Write> oio::Write for TracingWrapper<R> {
parent = &self.span,
level = "trace",
skip_all)]
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
self.inner.write(bs).await
}
@@ -332,7 +332,7 @@ impl<R: oio::Write> oio::Write for TracingWrapper<R> {
parent = &self.span,
level = "trace",
skip_all)]
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.abort().await
}
@@ -340,7 +340,7 @@ impl<R: oio::Write> oio::Write for TracingWrapper<R> {
parent = &self.span,
level = "trace",
skip_all)]
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.inner.close().await
}
}
diff --git a/core/src/raw/oio/write/api.rs b/core/src/raw/oio/write/api.rs
index 2463068e9..76b534888 100644
--- a/core/src/raw/oio/write/api.rs
+++ b/core/src/raw/oio/write/api.rs
@@ -87,11 +87,11 @@ pub trait Write: Unpin + Send + Sync {
/// repeatedly until all bytes has been written.
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>>;
- /// Abort the pending writer.
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>>;
-
/// Close the writer and make sure all data has been flushed.
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>>;
+
+ /// Abort the pending writer.
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>>;
}
#[async_trait]
@@ -100,17 +100,17 @@ impl Write for () {
unimplemented!("write is required to be implemented for oio::Write")
}
- fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
Poll::Ready(Err(Error::new(
ErrorKind::Unsupported,
- "output writer doesn't support abort",
+ "output writer doesn't support close",
)))
}
- fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
Poll::Ready(Err(Error::new(
ErrorKind::Unsupported,
- "output writer doesn't support close",
+ "output writer doesn't support abort",
)))
}
}
@@ -124,13 +124,13 @@ impl<T: Write + ?Sized> Write for Box<T> {
(**self).poll_write(cx, bs)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- (**self).poll_abort(cx)
- }
-
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
(**self).poll_close(cx)
}
+
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ (**self).poll_abort(cx)
+ }
}
/// Impl WriteExt for all T: Write
@@ -143,13 +143,13 @@ pub trait WriteExt: Write {
WriteFuture { writer: self, buf }
}
- fn abort(&mut self) -> AbortFuture<Self> {
- AbortFuture { writer: self }
- }
-
fn close(&mut self) -> CloseFuture<Self> {
CloseFuture { writer: self }
}
+
+ fn abort(&mut self) -> AbortFuture<Self> {
+ AbortFuture { writer: self }
+ }
}
/// Make this future `!Unpin` for compatibility with async trait methods.
diff --git a/core/src/raw/oio/write/append_object_write.rs b/core/src/raw/oio/write/append_object_write.rs
index ed2a456d9..ba5642dff 100644
--- a/core/src/raw/oio/write/append_object_write.rs
+++ b/core/src/raw/oio/write/append_object_write.rs
@@ -77,7 +77,7 @@ impl<W> oio::Write for AppendObjectWriter<W>
where
W: AppendObjectWrite,
{
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let offset = self.offset().await?;
let size = bs.remaining();
@@ -94,11 +94,11 @@ where
Ok(size)
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/raw/oio/write/compose_write.rs b/core/src/raw/oio/write/compose_write.rs
index c6504ee93..2b56e7233 100644
--- a/core/src/raw/oio/write/compose_write.rs
+++ b/core/src/raw/oio/write/compose_write.rs
@@ -39,6 +39,7 @@
//! type_alias_impl_trait has been stabilized.
use async_trait::async_trait;
+use std::task::{Context, Poll};
use crate::raw::*;
use crate::*;
@@ -55,24 +56,24 @@ pub enum TwoWaysWriter<ONE: oio::Write, TWO: oio::Write> {
#[async_trait]
impl<ONE: oio::Write, TWO: oio::Write> oio::Write for TwoWaysWriter<ONE, TWO> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
match self {
- Self::One(one) => one.write(bs).await,
- Self::Two(two) => two.write(bs).await,
+ Self::One(one) => one.poll_write(cx, bs),
+ Self::Two(two) => two.poll_write(cx, bs),
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
match self {
- Self::One(one) => one.abort().await,
- Self::Two(two) => two.abort().await,
+ Self::One(one) => one.poll_close(cx),
+ Self::Two(two) => two.poll_close(cx),
}
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
match self {
- Self::One(one) => one.close().await,
- Self::Two(two) => two.close().await,
+ Self::One(one) => one.poll_abort(cx),
+ Self::Two(two) => two.poll_abort(cx),
}
}
}
@@ -93,27 +94,27 @@ pub enum ThreeWaysWriter<ONE: oio::Write, TWO: oio::Write, THREE: oio::Write> {
impl<ONE: oio::Write, TWO: oio::Write, THREE: oio::Write> oio::Write
for ThreeWaysWriter<ONE, TWO, THREE>
{
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
match self {
- Self::One(one) => one.write(bs).await,
- Self::Two(two) => two.write(bs).await,
- Self::Three(three) => three.write(bs).await,
+ Self::One(one) => one.poll_write(cx, bs),
+ Self::Two(two) => two.poll_write(cx, bs),
+ Self::Three(three) => three.poll_write(cx, bs),
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
match self {
- Self::One(one) => one.abort().await,
- Self::Two(two) => two.abort().await,
- Self::Three(three) => three.abort().await,
+ Self::One(one) => one.poll_close(cx),
+ Self::Two(two) => two.poll_close(cx),
+ Self::Three(three) => three.poll_close(cx),
}
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
match self {
- Self::One(one) => one.close().await,
- Self::Two(two) => two.close().await,
- Self::Three(three) => three.close().await,
+ Self::One(one) => one.poll_abort(cx),
+ Self::Two(two) => two.poll_abort(cx),
+ Self::Three(three) => three.poll_abort(cx),
}
}
}
diff --git a/core/src/raw/oio/write/exact_buf_write.rs b/core/src/raw/oio/write/exact_buf_write.rs
index 38e00f057..f1615346b 100644
--- a/core/src/raw/oio/write/exact_buf_write.rs
+++ b/core/src/raw/oio/write/exact_buf_write.rs
@@ -163,11 +163,11 @@ mod tests {
Poll::Ready(Ok(bs.chunk().len()))
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs
index 237b5774e..10353714d 100644
--- a/core/src/raw/oio/write/multipart_upload_write.rs
+++ b/core/src/raw/oio/write/multipart_upload_write.rs
@@ -119,7 +119,7 @@ impl<W> oio::Write for MultipartUploadWriter<W>
where
W: MultipartUploadWrite,
{
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let upload_id = self.upload_id().await?;
let size = bs.remaining();
@@ -137,7 +137,7 @@ where
Ok(size)
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
let upload_id = if let Some(upload_id) = &self.upload_id {
upload_id
} else {
@@ -147,7 +147,7 @@ where
self.inner.complete_part(upload_id, &self.parts).await
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
let upload_id = if let Some(upload_id) = &self.upload_id {
upload_id
} else {
diff --git a/core/src/raw/oio/write/one_shot_write.rs b/core/src/raw/oio/write/one_shot_write.rs
index 9d387aedb..85ef499d5 100644
--- a/core/src/raw/oio/write/one_shot_write.rs
+++ b/core/src/raw/oio/write/one_shot_write.rs
@@ -48,17 +48,17 @@ impl<W: OneShotWrite> OneShotWriter<W> {
#[async_trait]
impl<W: OneShotWrite> oio::Write for OneShotWriter<W> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
self.inner.write_once(bs).await?;
Ok(size)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/azblob/writer.rs b/core/src/services/azblob/writer.rs
index 680bc8f53..1a228b647 100644
--- a/core/src/services/azblob/writer.rs
+++ b/core/src/services/azblob/writer.rs
@@ -160,7 +160,7 @@ impl AzblobWriter {
#[async_trait]
impl oio::Write for AzblobWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
if self.op.append() {
@@ -181,11 +181,11 @@ impl oio::Write for AzblobWriter {
Ok(size)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/azdfs/writer.rs b/core/src/services/azdfs/writer.rs
index 0dfaa320b..100065a7e 100644
--- a/core/src/services/azdfs/writer.rs
+++ b/core/src/services/azdfs/writer.rs
@@ -40,7 +40,7 @@ impl AzdfsWriter {
#[async_trait]
impl oio::Write for AzdfsWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let mut req = self.core.azdfs_create_request(
&self.path,
"file",
@@ -89,11 +89,11 @@ impl oio::Write for AzdfsWriter {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/dropbox/writer.rs b/core/src/services/dropbox/writer.rs
index 6331727b1..c7887a7d7 100644
--- a/core/src/services/dropbox/writer.rs
+++ b/core/src/services/dropbox/writer.rs
@@ -39,7 +39,7 @@ impl DropboxWriter {
#[async_trait]
impl oio::Write for DropboxWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
let resp = self
@@ -61,11 +61,11 @@ impl oio::Write for DropboxWriter {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/fs/writer.rs b/core/src/services/fs/writer.rs
index fd4a9535b..3d94a8888 100644
--- a/core/src/services/fs/writer.rs
+++ b/core/src/services/fs/writer.rs
@@ -43,18 +43,18 @@ impl<F> FsWriter<F> {
#[async_trait]
impl oio::Write for FsWriter<tokio::fs::File> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
self.f.write(bs.chunk()).await.map_err(parse_io_error)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Err(Error::new(
ErrorKind::Unsupported,
"output writer doesn't support abort",
))
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.f.sync_all().await.map_err(parse_io_error)?;
if let Some(tmp_path) = &self.tmp_path {
diff --git a/core/src/services/ftp/writer.rs b/core/src/services/ftp/writer.rs
index f0e61d537..9fcfd989d 100644
--- a/core/src/services/ftp/writer.rs
+++ b/core/src/services/ftp/writer.rs
@@ -40,7 +40,7 @@ impl FtpWriter {
#[async_trait]
impl oio::Write for FtpWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
let bs = bs.copy_to_bytes(size);
@@ -55,11 +55,11 @@ impl oio::Write for FtpWriter {
Ok(size)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/gcs/writer.rs b/core/src/services/gcs/writer.rs
index 9ae9b2fb3..2436165c4 100644
--- a/core/src/services/gcs/writer.rs
+++ b/core/src/services/gcs/writer.rs
@@ -118,7 +118,7 @@ impl GcsWriter {
#[async_trait]
impl oio::Write for GcsWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
let location = match &self.location {
@@ -161,7 +161,7 @@ impl oio::Write for GcsWriter {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
let location = if let Some(location) = &self.location {
location
} else {
@@ -183,7 +183,7 @@ impl oio::Write for GcsWriter {
}
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
let location = if let Some(location) = &self.location {
location
} else {
diff --git a/core/src/services/gdrive/writer.rs b/core/src/services/gdrive/writer.rs
index 2fc5f229a..b733863a8 100644
--- a/core/src/services/gdrive/writer.rs
+++ b/core/src/services/gdrive/writer.rs
@@ -95,7 +95,7 @@ impl GdriveWriter {
#[async_trait]
impl oio::Write for GdriveWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
if self.file_id.is_none() {
self.write_create(size as u64, bs.copy_to_bytes(size))
@@ -108,11 +108,11 @@ impl oio::Write for GdriveWriter {
Ok(size)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/ghac/writer.rs b/core/src/services/ghac/writer.rs
index 7314d82fc..886abffb5 100644
--- a/core/src/services/ghac/writer.rs
+++ b/core/src/services/ghac/writer.rs
@@ -41,7 +41,7 @@ impl GhacWriter {
#[async_trait]
impl oio::Write for GhacWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
let req = self
@@ -66,11 +66,11 @@ impl oio::Write for GhacWriter {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
let req = self.backend.ghac_commit(self.cache_id, self.size).await?;
let resp = self.backend.client.send(req).await?;
diff --git a/core/src/services/hdfs/writer.rs b/core/src/services/hdfs/writer.rs
index e5f140dca..4c69d63dc 100644
--- a/core/src/services/hdfs/writer.rs
+++ b/core/src/services/hdfs/writer.rs
@@ -36,18 +36,18 @@ impl<F> HdfsWriter<F> {
#[async_trait]
impl oio::Write for HdfsWriter<hdrs::AsyncFile> {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
self.f.write(bs.chunk()).await.map_err(parse_io_error)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Err(Error::new(
ErrorKind::Unsupported,
"output writer doesn't support abort",
))
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
self.f.close().await.map_err(parse_io_error)?;
Ok(())
diff --git a/core/src/services/ipmfs/writer.rs b/core/src/services/ipmfs/writer.rs
index 2c82ec0c6..dcecb66d3 100644
--- a/core/src/services/ipmfs/writer.rs
+++ b/core/src/services/ipmfs/writer.rs
@@ -37,7 +37,7 @@ impl IpmfsWriter {
#[async_trait]
impl oio::Write for IpmfsWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
let resp = self
.backend
@@ -55,11 +55,11 @@ impl oio::Write for IpmfsWriter {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/onedrive/writer.rs b/core/src/services/onedrive/writer.rs
index 9d0c6a992..46aa50a21 100644
--- a/core/src/services/onedrive/writer.rs
+++ b/core/src/services/onedrive/writer.rs
@@ -45,7 +45,7 @@ impl OneDriveWriter {
#[async_trait]
impl oio::Write for OneDriveWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
let bs = bs.copy_to_bytes(size);
@@ -58,11 +58,11 @@ impl oio::Write for OneDriveWriter {
Ok(size)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/sftp/writer.rs b/core/src/services/sftp/writer.rs
index 9abd24f25..cfe51c7f3 100644
--- a/core/src/services/sftp/writer.rs
+++ b/core/src/services/sftp/writer.rs
@@ -33,20 +33,20 @@ impl SftpWriter {
#[async_trait]
impl oio::Write for SftpWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = self.file.write(bs.chunk()).await?;
Ok(size)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Err(Error::new(
ErrorKind::Unsupported,
"SFTP does not support aborting writes",
))
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/supabase/writer.rs b/core/src/services/supabase/writer.rs
index 8830ca849..d8ba4cc16 100644
--- a/core/src/services/supabase/writer.rs
+++ b/core/src/services/supabase/writer.rs
@@ -67,20 +67,20 @@ impl SupabaseWriter {
#[async_trait]
impl oio::Write for SupabaseWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
self.upload(bs.copy_to_bytes(size)).await?;
Ok(size)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Err(Error::new(
ErrorKind::Unsupported,
"The abort operation is not yet supported for Supabase backend",
))
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/vercel_artifacts/writer.rs b/core/src/services/vercel_artifacts/writer.rs
index 52acee034..6579374a7 100644
--- a/core/src/services/vercel_artifacts/writer.rs
+++ b/core/src/services/vercel_artifacts/writer.rs
@@ -38,7 +38,7 @@ impl VercelArtifactsWriter {
#[async_trait]
impl oio::Write for VercelArtifactsWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
let resp = self
@@ -61,11 +61,11 @@ impl oio::Write for VercelArtifactsWriter {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/wasabi/writer.rs b/core/src/services/wasabi/writer.rs
index 4736e348b..3eeaab12d 100644
--- a/core/src/services/wasabi/writer.rs
+++ b/core/src/services/wasabi/writer.rs
@@ -40,7 +40,7 @@ impl WasabiWriter {
#[async_trait]
impl oio::Write for WasabiWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
let resp = self
@@ -64,11 +64,11 @@ impl oio::Write for WasabiWriter {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/webdav/writer.rs b/core/src/services/webdav/writer.rs
index cccf177ed..eaff3d8b4 100644
--- a/core/src/services/webdav/writer.rs
+++ b/core/src/services/webdav/writer.rs
@@ -61,7 +61,7 @@ impl WebdavWriter {
#[async_trait]
impl oio::Write for WebdavWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
self.write_oneshot(size as u64, AsyncBody::Bytes(bs.copy_to_bytes(size)))
@@ -70,11 +70,11 @@ impl oio::Write for WebdavWriter {
Ok(size)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/webhdfs/writer.rs b/core/src/services/webhdfs/writer.rs
index c3ce0b5be..0c7a9e10b 100644
--- a/core/src/services/webhdfs/writer.rs
+++ b/core/src/services/webhdfs/writer.rs
@@ -38,7 +38,7 @@ impl WebhdfsWriter {
#[async_trait]
impl oio::Write for WebhdfsWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Result<usize> {
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
let size = bs.remaining();
let req = self
@@ -63,11 +63,11 @@ impl oio::Write for WebhdfsWriter {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/types/writer.rs b/core/src/types/writer.rs
index da384729f..77b88899e 100644
--- a/core/src/types/writer.rs
+++ b/core/src/types/writer.rs
@@ -205,7 +205,7 @@ impl Writer {
///
/// Abort should only be called when the writer is not closed or
/// aborted, otherwise an unexpected error could be returned.
- pub fn poll_abort(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ pub fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
if let State::Idle(Some(w)) = &mut self.state {
w.abort().await
} else {
@@ -222,7 +222,7 @@ impl Writer {
///
/// Close should only be called when the writer is not closed or
/// aborted, otherwise an unexpected error could be returned.
- pub fn poll_close(&mut self, cx: &mut Context<'_>) -> Result<()> {
+ pub fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
if let State::Idle(Some(w)) = &mut self.state {
w.close().await
} else {
[incubator-opendal] 22/31: Fix test
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 77f36bba2e3a417abc33daa473629e311c3fce05
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 17:59:47 2023 +0800
Fix test
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/benches/oio/utils.rs | 6 +++---
core/benches/oio/write.rs | 1 -
core/src/layers/oteltrace.rs | 1 -
core/src/layers/throttle.rs | 2 +-
core/src/layers/timeout.rs | 2 ++
core/src/raw/adapters/typed_kv/backend.rs | 2 +-
core/src/raw/oio/write/api.rs | 2 +-
core/src/raw/oio/write/append_object_write.rs | 4 ++--
core/src/raw/oio/write/exact_buf_write.rs | 8 ++++----
core/src/raw/oio/write/multipart_upload_write.rs | 2 +-
core/src/services/azblob/writer.rs | 2 +-
core/src/services/azdfs/writer.rs | 1 -
core/src/services/ftp/writer.rs | 4 ++--
core/src/services/gcs/writer.rs | 6 +++---
core/src/services/gdrive/writer.rs | 1 -
core/src/services/ghac/writer.rs | 2 +-
core/src/services/hdfs/writer.rs | 2 +-
core/src/services/ipmfs/writer.rs | 3 ---
core/src/services/onedrive/writer.rs | 1 -
core/src/services/sftp/writer.rs | 8 +++-----
core/src/services/supabase/writer.rs | 1 -
core/src/services/wasabi/writer.rs | 1 -
core/src/types/writer.rs | 4 ----
23 files changed, 26 insertions(+), 40 deletions(-)
diff --git a/core/benches/oio/utils.rs b/core/benches/oio/utils.rs
index b3ea04027..fe418922e 100644
--- a/core/benches/oio/utils.rs
+++ b/core/benches/oio/utils.rs
@@ -29,17 +29,17 @@ pub struct BlackHoleWriter;
impl oio::Write for BlackHoleWriter {
fn poll_write(
&mut self,
- cx: &mut Context<'_>,
+ _: &mut Context<'_>,
bs: &dyn oio::WriteBuf,
) -> Poll<opendal::Result<usize>> {
Poll::Ready(Ok(bs.remaining()))
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<opendal::Result<()>> {
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<opendal::Result<()>> {
Poll::Ready(Ok(()))
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<opendal::Result<()>> {
+ fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<opendal::Result<()>> {
Poll::Ready(Ok(()))
}
}
diff --git a/core/benches/oio/write.rs b/core/benches/oio/write.rs
index 3ab71cdcf..2ce7d3a2e 100644
--- a/core/benches/oio/write.rs
+++ b/core/benches/oio/write.rs
@@ -18,7 +18,6 @@
use bytes::Buf;
use criterion::Criterion;
use once_cell::sync::Lazy;
-use opendal::raw::oio::Write;
use opendal::raw::oio::{ExactBufWriter, WriteExt};
use rand::thread_rng;
use size::Size;
diff --git a/core/src/layers/oteltrace.rs b/core/src/layers/oteltrace.rs
index 145b997d5..b0d80fe1d 100644
--- a/core/src/layers/oteltrace.rs
+++ b/core/src/layers/oteltrace.rs
@@ -16,7 +16,6 @@
// under the License.
use std::io;
-use std::task;
use std::task::{Context, Poll};
use async_trait::async_trait;
diff --git a/core/src/layers/throttle.rs b/core/src/layers/throttle.rs
index 72e63abe0..708d5980b 100644
--- a/core/src/layers/throttle.rs
+++ b/core/src/layers/throttle.rs
@@ -225,7 +225,7 @@ impl<R: oio::Write> oio::Write for ThrottleWrapper<R> {
Err(negative) => match negative {
// the query is valid but the Decider can not accommodate them.
NegativeMultiDecision::BatchNonConforming(_, not_until) => {
- let wait_time = not_until.wait_time_from(DefaultClock::default().now());
+ let _ = not_until.wait_time_from(DefaultClock::default().now());
// TODO: Should lock the limiter and wait for the wait_time, or should let other small requests go first?
// FIXME: we should sleep here.
diff --git a/core/src/layers/timeout.rs b/core/src/layers/timeout.rs
index d6cd4d24a..2cabf36af 100644
--- a/core/src/layers/timeout.rs
+++ b/core/src/layers/timeout.rs
@@ -206,6 +206,7 @@ pub struct TimeoutWrapper<R> {
inner: R,
timeout: Duration,
+ #[allow(dead_code)]
speed: u64,
start: Option<Instant>,
@@ -221,6 +222,7 @@ impl<R> TimeoutWrapper<R> {
}
}
+ #[allow(dead_code)]
fn io_timeout(&self, size: u64) -> Duration {
let timeout = Duration::from_millis(size * 1000 / self.speed + 1);
diff --git a/core/src/raw/adapters/typed_kv/backend.rs b/core/src/raw/adapters/typed_kv/backend.rs
index ce0f9f073..8ddc0f524 100644
--- a/core/src/raw/adapters/typed_kv/backend.rs
+++ b/core/src/raw/adapters/typed_kv/backend.rs
@@ -411,7 +411,7 @@ impl<S> KvWriter<S> {
#[async_trait]
impl<S: Adapter> oio::Write for KvWriter<S> {
// TODO: we need to support append in the future.
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
+ fn poll_write(&mut self, _: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
if self.future.is_some() {
self.future = None;
return Poll::Ready(Err(Error::new(
diff --git a/core/src/raw/oio/write/api.rs b/core/src/raw/oio/write/api.rs
index c05f061c7..c38221675 100644
--- a/core/src/raw/oio/write/api.rs
+++ b/core/src/raw/oio/write/api.rs
@@ -166,7 +166,7 @@ where
type Output = Result<usize>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<usize>> {
- let mut this = self.project();
+ let this = self.project();
Pin::new(this.writer).poll_write(cx, *this.buf)
}
}
diff --git a/core/src/raw/oio/write/append_object_write.rs b/core/src/raw/oio/write/append_object_write.rs
index 5cbaedb37..c2d0d498b 100644
--- a/core/src/raw/oio/write/append_object_write.rs
+++ b/core/src/raw/oio/write/append_object_write.rs
@@ -119,11 +119,11 @@ where
}
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
Poll::Ready(Ok(()))
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
Poll::Ready(Ok(()))
}
}
diff --git a/core/src/raw/oio/write/exact_buf_write.rs b/core/src/raw/oio/write/exact_buf_write.rs
index 636242280..5c700029b 100644
--- a/core/src/raw/oio/write/exact_buf_write.rs
+++ b/core/src/raw/oio/write/exact_buf_write.rs
@@ -22,7 +22,6 @@ use async_trait::async_trait;
use bytes::{Bytes, BytesMut};
use crate::raw::oio::WriteBuf;
-use crate::raw::oio::WriteExt;
use crate::raw::*;
use crate::*;
@@ -146,6 +145,7 @@ mod tests {
use super::*;
use crate::raw::oio::Write;
+ use crate::raw::oio::WriteExt;
struct MockWriter {
buf: Vec<u8>,
@@ -153,7 +153,7 @@ mod tests {
#[async_trait]
impl Write for MockWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn WriteBuf) -> Poll<Result<usize>> {
+ fn poll_write(&mut self, _: &mut Context<'_>, bs: &dyn WriteBuf) -> Poll<Result<usize>> {
debug!(
"test_fuzz_exact_buf_writer: flush size: {}",
bs.chunk().len()
@@ -163,11 +163,11 @@ mod tests {
Poll::Ready(Ok(bs.chunk().len()))
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
Poll::Ready(Ok(()))
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
Poll::Ready(Ok(()))
}
}
diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs
index af6f825ac..81ee1f1ee 100644
--- a/core/src/raw/oio/write/multipart_upload_write.rs
+++ b/core/src/raw/oio/write/multipart_upload_write.rs
@@ -219,7 +219,7 @@ where
let w = w.take().expect("writer must be valid");
match self.upload_id.clone() {
Some(upload_id) => {
- self.state = State::Close(Box::pin(async move {
+ self.state = State::Abort(Box::pin(async move {
let res = w.abort_part(&upload_id).await;
(w, res)
}));
diff --git a/core/src/services/azblob/writer.rs b/core/src/services/azblob/writer.rs
index 70b023249..15a63caca 100644
--- a/core/src/services/azblob/writer.rs
+++ b/core/src/services/azblob/writer.rs
@@ -161,7 +161,7 @@ impl AzblobWriter {
#[async_trait]
impl oio::Write for AzblobWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
+ fn poll_write(&mut self, _: &mut Context<'_>, _: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
todo!()
}
diff --git a/core/src/services/azdfs/writer.rs b/core/src/services/azdfs/writer.rs
index 1da698dd4..9e2a962f2 100644
--- a/core/src/services/azdfs/writer.rs
+++ b/core/src/services/azdfs/writer.rs
@@ -16,7 +16,6 @@
// under the License.
use std::sync::Arc;
-use std::task::{Context, Poll};
use async_trait::async_trait;
use bytes::Bytes;
diff --git a/core/src/services/ftp/writer.rs b/core/src/services/ftp/writer.rs
index 5a759aa21..f7e8497d6 100644
--- a/core/src/services/ftp/writer.rs
+++ b/core/src/services/ftp/writer.rs
@@ -81,11 +81,11 @@ impl oio::Write for FtpWriter {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
Poll::Ready(Ok(()))
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
Poll::Ready(Ok(()))
}
}
diff --git a/core/src/services/gcs/writer.rs b/core/src/services/gcs/writer.rs
index c21f6b4e5..c61c7df4c 100644
--- a/core/src/services/gcs/writer.rs
+++ b/core/src/services/gcs/writer.rs
@@ -119,7 +119,7 @@ impl GcsWriter {
#[async_trait]
impl oio::Write for GcsWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
+ fn poll_write(&mut self, _: &mut Context<'_>, _: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
// let size = bs.remaining();
//
// let location = match &self.location {
@@ -164,7 +164,7 @@ impl oio::Write for GcsWriter {
todo!()
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
// let location = if let Some(location) = &self.location {
// location
// } else {
@@ -188,7 +188,7 @@ impl oio::Write for GcsWriter {
todo!()
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
// let location = if let Some(location) = &self.location {
// location
// } else {
diff --git a/core/src/services/gdrive/writer.rs b/core/src/services/gdrive/writer.rs
index 2b0ac8c55..e70f1754e 100644
--- a/core/src/services/gdrive/writer.rs
+++ b/core/src/services/gdrive/writer.rs
@@ -24,7 +24,6 @@ use http::StatusCode;
use super::core::GdriveCore;
use super::error::parse_error;
use crate::raw::*;
-use crate::services::gdrive::core::GdriveFile;
use crate::*;
pub struct GdriveWriter {
diff --git a/core/src/services/ghac/writer.rs b/core/src/services/ghac/writer.rs
index 582a150aa..439bbbb2e 100644
--- a/core/src/services/ghac/writer.rs
+++ b/core/src/services/ghac/writer.rs
@@ -102,7 +102,7 @@ impl oio::Write for GhacWriter {
}
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
self.state = State::Idle(None);
Poll::Ready(Ok(()))
diff --git a/core/src/services/hdfs/writer.rs b/core/src/services/hdfs/writer.rs
index ced19efb9..f2725813c 100644
--- a/core/src/services/hdfs/writer.rs
+++ b/core/src/services/hdfs/writer.rs
@@ -20,7 +20,7 @@ use std::pin::Pin;
use std::task::{Context, Poll};
use async_trait::async_trait;
-use futures::{AsyncWrite, AsyncWriteExt};
+use futures::AsyncWrite;
use super::error::parse_io_error;
use crate::raw::*;
diff --git a/core/src/services/ipmfs/writer.rs b/core/src/services/ipmfs/writer.rs
index feb5e5630..e6a32ac1b 100644
--- a/core/src/services/ipmfs/writer.rs
+++ b/core/src/services/ipmfs/writer.rs
@@ -18,11 +18,9 @@
use async_trait::async_trait;
use bytes::Bytes;
use http::StatusCode;
-use std::task::{Context, Poll};
use super::backend::IpmfsBackend;
use super::error::parse_error;
-use crate::raw::oio::WriteBuf;
use crate::raw::*;
use crate::*;
@@ -41,7 +39,6 @@ impl IpmfsWriter {
#[async_trait]
impl oio::OneShotWrite for IpmfsWriter {
async fn write_once(&self, bs: Bytes) -> Result<()> {
- let size = bs.len();
let resp = self.backend.ipmfs_write(&self.path, bs).await?;
let status = resp.status();
diff --git a/core/src/services/onedrive/writer.rs b/core/src/services/onedrive/writer.rs
index 727d8f7d3..5ad92ac63 100644
--- a/core/src/services/onedrive/writer.rs
+++ b/core/src/services/onedrive/writer.rs
@@ -23,7 +23,6 @@ use super::backend::OnedriveBackend;
use super::error::parse_error;
use super::graph_model::OneDriveUploadSessionCreationRequestBody;
use super::graph_model::OneDriveUploadSessionCreationResponseBody;
-use crate::raw::oio::OneShotWriter;
use crate::raw::*;
use crate::*;
diff --git a/core/src/services/sftp/writer.rs b/core/src/services/sftp/writer.rs
index 455e0fdbc..2f07710d0 100644
--- a/core/src/services/sftp/writer.rs
+++ b/core/src/services/sftp/writer.rs
@@ -16,9 +16,7 @@
// under the License.
use async_trait::async_trait;
-use futures::AsyncWrite;
use openssh_sftp_client::file::File;
-use std::pin::Pin;
use std::task::{Context, Poll};
use crate::raw::oio;
@@ -36,7 +34,7 @@ impl SftpWriter {
#[async_trait]
impl oio::Write for SftpWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
+ fn poll_write(&mut self, _: &mut Context<'_>, _: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
// Pin::new(&mut self.file)
// .poll_write(cx, bs.chunk())
// .map_err(Error::from)
@@ -44,11 +42,11 @@ impl oio::Write for SftpWriter {
todo!()
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
Poll::Ready(Ok(()))
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
// Pin::new(&mut self.file).poll_flush(cx).map_err(Error::from)
todo!()
}
diff --git a/core/src/services/supabase/writer.rs b/core/src/services/supabase/writer.rs
index 52292a737..c7fc6c8f8 100644
--- a/core/src/services/supabase/writer.rs
+++ b/core/src/services/supabase/writer.rs
@@ -16,7 +16,6 @@
// under the License.
use std::sync::Arc;
-use std::task::{Context, Poll};
use async_trait::async_trait;
use bytes::Bytes;
diff --git a/core/src/services/wasabi/writer.rs b/core/src/services/wasabi/writer.rs
index 254810aa9..f9e27334e 100644
--- a/core/src/services/wasabi/writer.rs
+++ b/core/src/services/wasabi/writer.rs
@@ -16,7 +16,6 @@
// under the License.
use std::sync::Arc;
-use std::task::Context;
use async_trait::async_trait;
use bytes::Bytes;
diff --git a/core/src/types/writer.rs b/core/src/types/writer.rs
index c90ebcc54..af67b6b0c 100644
--- a/core/src/types/writer.rs
+++ b/core/src/types/writer.rs
@@ -15,17 +15,13 @@
// specific language governing permissions and limitations
// under the License.
-use std::fmt::Display;
use std::io;
use std::pin::Pin;
-use std::task::ready;
use std::task::Context;
use std::task::Poll;
use bytes::Bytes;
-use futures::future::BoxFuture;
use futures::AsyncWrite;
-use futures::FutureExt;
use futures::TryStreamExt;
use crate::raw::oio::WriteBuf;
[incubator-opendal] 08/31: save work
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 34fa1e9b7a2c1e3af15efd88bd998797ce26843f
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 15:39:51 2023 +0800
save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/ftp/writer.rs | 46 +++++++++++++++++++++++++++++------------
1 file changed, 33 insertions(+), 13 deletions(-)
diff --git a/core/src/services/ftp/writer.rs b/core/src/services/ftp/writer.rs
index 9fcfd989d..34ed85b37 100644
--- a/core/src/services/ftp/writer.rs
+++ b/core/src/services/ftp/writer.rs
@@ -16,7 +16,9 @@
// under the License.
use async_trait::async_trait;
+use futures::future::BoxFuture;
use futures::AsyncWriteExt;
+use std::task::{ready, Context, Poll};
use super::backend::FtpBackend;
use crate::raw::*;
@@ -25,41 +27,59 @@ use crate::*;
pub struct FtpWriter {
backend: FtpBackend,
path: String,
+
+ fut: Option<BoxFuture<'static, Result<usize>>>,
}
/// # TODO
///
/// Writer is not implemented correctly.
///
-/// After we can use datastream, we should return it directly.
+/// After we can use data stream, we should return it directly.
impl FtpWriter {
pub fn new(backend: FtpBackend, path: String) -> Self {
- FtpWriter { backend, path }
+ FtpWriter {
+ backend,
+ path,
+ fut: None,
+ }
}
}
#[async_trait]
impl oio::Write for FtpWriter {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
- let bs = bs.copy_to_bytes(size);
+ loop {
+ if let Some(fut) = self.fut.as_mut() {
+ let res = ready!(fut.poll_unpin(cx));
+ self.fut = None;
+ return Poll::Ready(res);
+ }
- let mut ftp_stream = self.backend.ftp_connect(Operation::Write).await?;
- let mut data_stream = ftp_stream.append_with_stream(&self.path).await?;
- data_stream.write_all(&bs).await.map_err(|err| {
- Error::new(ErrorKind::Unexpected, "copy from ftp stream").set_source(err)
- })?;
+ let size = bs.remaining();
+ let bs = bs.copy_to_bytes(size);
- ftp_stream.finalize_put_stream(data_stream).await?;
+ let path = self.path.clone();
+ let backend = self.backend.clone();
+ let fut = async {
+ let mut ftp_stream = backend.ftp_connect(Operation::Write).await?;
+ let mut data_stream = ftp_stream.append_with_stream(&path).await?;
+ data_stream.write_all(&bs).await.map_err(|err| {
+ Error::new(ErrorKind::Unexpected, "copy from ftp stream").set_source(err)
+ })?;
- Ok(size)
+ ftp_stream.finalize_put_stream(data_stream).await?;
+ Ok(size)
+ };
+ self.fut = Some(Box::pin(fut));
+ }
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ Poll::Ready(Ok(()))
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ Poll::Ready(Ok(()))
}
}
[incubator-opendal] 30/31: Fix fs abort
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit ace5c58b07dd5a77226e92ca42d2c46f6ab5f875
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Sep 11 16:30:10 2023 +0800
Fix fs abort
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/fs/writer.rs | 31 +++++++++++++++++++++++++------
1 file changed, 25 insertions(+), 6 deletions(-)
diff --git a/core/src/services/fs/writer.rs b/core/src/services/fs/writer.rs
index 26105115b..6964b4150 100644
--- a/core/src/services/fs/writer.rs
+++ b/core/src/services/fs/writer.rs
@@ -63,12 +63,6 @@ impl oio::Write for FsWriter<tokio::fs::File> {
.map_err(parse_io_error)
}
- fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
- self.f = None;
-
- Poll::Ready(Ok(()))
- }
-
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
loop {
if let Some(fut) = self.fut.as_mut() {
@@ -93,6 +87,31 @@ impl oio::Write for FsWriter<tokio::fs::File> {
}));
}
}
+
+ fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ loop {
+ if let Some(fut) = self.fut.as_mut() {
+ let res = ready!(fut.poll_unpin(cx));
+ self.fut = None;
+ return Poll::Ready(res);
+ }
+
+ let _ = self.f.take().expect("FsWriter must be initialized");
+ let tmp_path = self.tmp_path.clone();
+ self.fut = Some(Box::pin(async move {
+ if let Some(tmp_path) = &tmp_path {
+ tokio::fs::remove_file(tmp_path)
+ .await
+ .map_err(parse_io_error)
+ } else {
+ Err(Error::new(
+ ErrorKind::Unsupported,
+ "Fs doesn't support abort if atomic_write_dir is not set",
+ ))
+ }
+ }));
+ }
+ }
}
impl oio::BlockingWrite for FsWriter<std::fs::File> {
[incubator-opendal] 15/31: Fix build for wasabi
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 25cb7b8afdeca32238c614bb1952f1062c3d61eb
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 16:20:14 2023 +0800
Fix build for wasabi
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/wasabi/writer.rs | 20 ++++++--------------
1 file changed, 6 insertions(+), 14 deletions(-)
diff --git a/core/src/services/wasabi/writer.rs b/core/src/services/wasabi/writer.rs
index 1c7b5ed2e..685848084 100644
--- a/core/src/services/wasabi/writer.rs
+++ b/core/src/services/wasabi/writer.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
use std::task::Context;
use async_trait::async_trait;
+use bytes::Bytes;
use http::StatusCode;
use super::core::*;
@@ -39,10 +40,9 @@ impl WasabiWriter {
}
}
-#[async_trait]
-impl oio::Write for WasabiWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
+impl oio::OneShotWrite for WasabiWriter {
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
+ let size = bs.len();
let resp = self
.core
@@ -52,24 +52,16 @@ impl oio::Write for WasabiWriter {
self.op.content_type(),
self.op.content_disposition(),
self.op.cache_control(),
- AsyncBody::Bytes(bs.copy_to_bytes(size)),
+ AsyncBody::Bytes(bs),
)
.await?;
match resp.status() {
StatusCode::CREATED | StatusCode::OK => {
resp.into_body().consume().await?;
- Ok(size)
+ Ok(())
}
_ => Err(parse_error(resp).await?),
}
}
-
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
-
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
}
[incubator-opendal] 28/31: Fix build
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit db50a60f3c938d0f51683dc2db4e1edf69280fc1
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Sep 11 16:19:01 2023 +0800
Fix build
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/sftp/writer.rs | 27 +++++++++++++++------------
1 file changed, 15 insertions(+), 12 deletions(-)
diff --git a/core/src/services/sftp/writer.rs b/core/src/services/sftp/writer.rs
index 2f07710d0..75db3081e 100644
--- a/core/src/services/sftp/writer.rs
+++ b/core/src/services/sftp/writer.rs
@@ -17,37 +17,40 @@
use async_trait::async_trait;
use openssh_sftp_client::file::File;
+use openssh_sftp_client::file::TokioCompatFile;
+use std::pin::Pin;
use std::task::{Context, Poll};
+use tokio::io::AsyncWrite;
use crate::raw::oio;
use crate::*;
pub struct SftpWriter {
- file: File,
+ file: Pin<Box<TokioCompatFile>>,
}
impl SftpWriter {
pub fn new(file: File) -> Self {
- SftpWriter { file }
+ SftpWriter {
+ file: Box::pin(TokioCompatFile::new(file)),
+ }
}
}
#[async_trait]
impl oio::Write for SftpWriter {
- fn poll_write(&mut self, _: &mut Context<'_>, _: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- // Pin::new(&mut self.file)
- // .poll_write(cx, bs.chunk())
- // .map_err(Error::from)
+ fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
+ self.file
+ .as_mut()
+ .poll_write(cx, bs.chunk())
+ .map_err(Error::from)
+ }
- todo!()
+ fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
+ self.file.as_mut().poll_shutdown(cx).map_err(Error::from)
}
fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
Poll::Ready(Ok(()))
}
-
- fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
- // Pin::new(&mut self.file).poll_flush(cx).map_err(Error::from)
- todo!()
- }
}
[incubator-opendal] 16/31: Save work
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit a5e53461769aa6b357f50bf7c7a078065d674cb6
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 16:30:06 2023 +0800
Save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/dropbox/writer.rs | 20 ++++++-----------
core/src/services/gdrive/writer.rs | 31 ++++++---------------------
core/src/services/onedrive/writer.rs | 19 +++++------------
core/src/services/vercel_artifacts/writer.rs | 24 ++++++---------------
core/src/services/webdav/writer.rs | 32 ++++++++--------------------
core/src/services/webhdfs/writer.rs | 20 ++++++-----------
6 files changed, 39 insertions(+), 107 deletions(-)
diff --git a/core/src/services/dropbox/writer.rs b/core/src/services/dropbox/writer.rs
index c7887a7d7..5e3e19a92 100644
--- a/core/src/services/dropbox/writer.rs
+++ b/core/src/services/dropbox/writer.rs
@@ -18,6 +18,7 @@
use std::sync::Arc;
use async_trait::async_trait;
+use bytes::Bytes;
use http::StatusCode;
use super::core::DropboxCore;
@@ -37,10 +38,9 @@ impl DropboxWriter {
}
}
-#[async_trait]
-impl oio::Write for DropboxWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
+impl oio::OneShotWrite for DropboxWriter {
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
+ let size = bs.len();
let resp = self
.core
@@ -48,24 +48,16 @@ impl oio::Write for DropboxWriter {
&self.path,
Some(size),
self.op.content_type(),
- AsyncBody::Bytes(bs.copy_to_bytes(size)),
+ AsyncBody::Bytes(bs),
)
.await?;
let status = resp.status();
match status {
StatusCode::OK => {
resp.into_body().consume().await?;
- Ok(size)
+ Ok(())
}
_ => Err(parse_error(resp).await?),
}
}
-
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
-
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
}
diff --git a/core/src/services/gdrive/writer.rs b/core/src/services/gdrive/writer.rs
index b733863a8..f2f0b4e1c 100644
--- a/core/src/services/gdrive/writer.rs
+++ b/core/src/services/gdrive/writer.rs
@@ -49,7 +49,7 @@ impl GdriveWriter {
///
/// This is used for small objects.
/// And should overwrite the object if it already exists.
- pub async fn write_create(&mut self, size: u64, body: Bytes) -> Result<()> {
+ pub async fn write_create(&self, size: u64, body: Bytes) -> Result<()> {
let resp = self
.core
.gdrive_upload_simple_request(&self.path, size, body)
@@ -59,13 +59,7 @@ impl GdriveWriter {
match status {
StatusCode::OK | StatusCode::CREATED => {
- let bs = resp.into_body().bytes().await?;
-
- let file = serde_json::from_slice::<GdriveFile>(&bs)
- .map_err(new_json_deserialize_error)?;
-
- self.file_id = Some(file.id);
-
+ resp.into_body().consume().await?;
Ok(())
}
_ => Err(parse_error(resp).await?),
@@ -93,26 +87,15 @@ impl GdriveWriter {
}
}
-#[async_trait]
-impl oio::Write for GdriveWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
+impl oio::OneShotWrite for GdriveWriter {
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
+ let size = bs.len();
if self.file_id.is_none() {
- self.write_create(size as u64, bs.copy_to_bytes(size))
- .await?;
+ self.write_create(size as u64, bs).await?;
} else {
- self.write_overwrite(size as u64, bs.copy_to_bytes(size))
- .await?;
+ self.write_overwrite(size as u64, bs).await?;
}
- Ok(size)
- }
-
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
-
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
diff --git a/core/src/services/onedrive/writer.rs b/core/src/services/onedrive/writer.rs
index 46aa50a21..76fc734d1 100644
--- a/core/src/services/onedrive/writer.rs
+++ b/core/src/services/onedrive/writer.rs
@@ -23,6 +23,7 @@ use super::backend::OnedriveBackend;
use super::error::parse_error;
use super::graph_model::OneDriveUploadSessionCreationRequestBody;
use super::graph_model::OneDriveUploadSessionCreationResponseBody;
+use crate::raw::oio::OneShotWriter;
use crate::raw::*;
use crate::*;
@@ -43,11 +44,9 @@ impl OneDriveWriter {
}
}
-#[async_trait]
-impl oio::Write for OneDriveWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
- let bs = bs.copy_to_bytes(size);
+impl oio::OneShotWrite for OneDriveWriter {
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
+ let size = bs.len();
if size <= Self::MAX_SIMPLE_SIZE {
self.write_simple(bs).await?;
@@ -55,20 +54,12 @@ impl oio::Write for OneDriveWriter {
self.write_chunked(bs).await?;
}
- Ok(size)
- }
-
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
-
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
Ok(())
}
}
impl OneDriveWriter {
- async fn write_simple(&mut self, bs: Bytes) -> Result<()> {
+ async fn write_simple(&self, bs: Bytes) -> Result<()> {
let resp = self
.backend
.onedrive_upload_simple(
diff --git a/core/src/services/vercel_artifacts/writer.rs b/core/src/services/vercel_artifacts/writer.rs
index 6579374a7..596c933cf 100644
--- a/core/src/services/vercel_artifacts/writer.rs
+++ b/core/src/services/vercel_artifacts/writer.rs
@@ -16,6 +16,7 @@
// under the License.
use async_trait::async_trait;
+use bytes::Bytes;
use http::StatusCode;
use super::backend::VercelArtifactsBackend;
@@ -36,18 +37,13 @@ impl VercelArtifactsWriter {
}
}
-#[async_trait]
-impl oio::Write for VercelArtifactsWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
+impl oio::OneShotWrite for VercelArtifactsWriter {
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
+ let size = bs.len();
let resp = self
.backend
- .vercel_artifacts_put(
- self.path.as_str(),
- self.op.content_length().unwrap(),
- AsyncBody::Bytes(bs.copy_to_bytes(size)),
- )
+ .vercel_artifacts_put(self.path.as_str(), size as u64, AsyncBody::Bytes(bs))
.await?;
let status = resp.status();
@@ -55,17 +51,9 @@ impl oio::Write for VercelArtifactsWriter {
match status {
StatusCode::OK | StatusCode::ACCEPTED => {
resp.into_body().consume().await?;
- Ok(size)
+ Ok(())
}
_ => Err(parse_error(resp).await?),
}
}
-
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
-
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
}
diff --git a/core/src/services/webdav/writer.rs b/core/src/services/webdav/writer.rs
index eaff3d8b4..5b6ea319f 100644
--- a/core/src/services/webdav/writer.rs
+++ b/core/src/services/webdav/writer.rs
@@ -16,6 +16,7 @@
// under the License.
use async_trait::async_trait;
+use bytes::Bytes;
use http::StatusCode;
use super::backend::WebdavBackend;
@@ -34,16 +35,21 @@ impl WebdavWriter {
pub fn new(backend: WebdavBackend, op: OpWrite, path: String) -> Self {
WebdavWriter { backend, op, path }
}
+}
+
+#[async_trait]
+impl oio::OneShotWrite for WebdavWriter {
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
+ let size = bs.len();
- async fn write_oneshot(&mut self, size: u64, body: AsyncBody) -> Result<()> {
let resp = self
.backend
.webdav_put(
&self.path,
- Some(size),
+ Some(size as u64),
self.op.content_type(),
self.op.content_disposition(),
- body,
+ AsyncBody::Bytes(bs),
)
.await?;
@@ -58,23 +64,3 @@ impl WebdavWriter {
}
}
}
-
-#[async_trait]
-impl oio::Write for WebdavWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
-
- self.write_oneshot(size as u64, AsyncBody::Bytes(bs.copy_to_bytes(size)))
- .await?;
-
- Ok(size)
- }
-
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
-
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
-}
diff --git a/core/src/services/webhdfs/writer.rs b/core/src/services/webhdfs/writer.rs
index 0c7a9e10b..22901b496 100644
--- a/core/src/services/webhdfs/writer.rs
+++ b/core/src/services/webhdfs/writer.rs
@@ -16,6 +16,7 @@
// under the License.
use async_trait::async_trait;
+use bytes::Bytes;
use http::StatusCode;
use super::backend::WebhdfsBackend;
@@ -36,10 +37,9 @@ impl WebhdfsWriter {
}
}
-#[async_trait]
-impl oio::Write for WebhdfsWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
+impl oio::OneShotWrite for WebhdfsWriter {
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
+ let size = bs.len();
let req = self
.backend
@@ -47,7 +47,7 @@ impl oio::Write for WebhdfsWriter {
&self.path,
Some(size),
self.op.content_type(),
- AsyncBody::Bytes(bs.copy_to_bytes(size)),
+ AsyncBody::Bytes(bs),
)
.await?;
@@ -57,17 +57,9 @@ impl oio::Write for WebhdfsWriter {
match status {
StatusCode::CREATED | StatusCode::OK => {
resp.into_body().consume().await?;
- Ok(size)
+ Ok(())
}
_ => Err(parse_error(resp).await?),
}
}
-
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
-
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
}
[incubator-opendal] 21/31: Improve writer
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 1f60e84104134c034adb3f8b080d666ee9c7b597
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 17:33:01 2023 +0800
Improve writer
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/types/writer.rs | 191 +++++------------------------------------------
1 file changed, 18 insertions(+), 173 deletions(-)
diff --git a/core/src/types/writer.rs b/core/src/types/writer.rs
index 9ab2b69dd..c90ebcc54 100644
--- a/core/src/types/writer.rs
+++ b/core/src/types/writer.rs
@@ -56,14 +56,9 @@ use crate::*;
/// and flush them into storage at needs. And finally, the file will be available
/// after `close` has been called.
pub struct Writer {
- state: State,
+ inner: oio::Writer,
}
-/// # Safety
-///
-/// Writer will only be accessed by `&mut Self`
-unsafe impl Sync for Writer {}
-
impl Writer {
/// Create a new writer.
///
@@ -75,25 +70,14 @@ impl Writer {
pub(crate) async fn create(acc: FusedAccessor, path: &str, op: OpWrite) -> Result<Self> {
let (_, w) = acc.write(path, op).await?;
- Ok(Writer {
- state: State::Idle(Some(w)),
- })
+ Ok(Writer { inner: w })
}
/// Write into inner writer.
pub async fn write(&mut self, bs: impl Into<Bytes>) -> Result<()> {
- let w = if let State::Idle(Some(w)) = &mut self.state {
- w
- } else {
- unreachable!(
- "writer state invalid while write, expect Idle, actual {}",
- self.state
- );
- };
-
let mut bs = bs.into();
while bs.remaining() > 0 {
- let n = w.write(&bs).await?;
+ let n = self.inner.write(&bs).await?;
bs.advance(n);
}
@@ -137,20 +121,11 @@ impl Writer {
S: futures::Stream<Item = Result<T>> + Send + Sync + Unpin + 'static,
T: Into<Bytes>,
{
- let w = if let State::Idle(Some(w)) = &mut self.state {
- w
- } else {
- unreachable!(
- "writer state invalid while sink, expect Idle, actual {}",
- self.state
- );
- };
-
let mut written = 0;
while let Some(bs) = sink_from.try_next().await? {
let mut bs = bs.into();
while bs.remaining() > 0 {
- let n = w.write(&bs).await?;
+ let n = self.inner.write(&bs).await?;
bs.advance(n);
written += n as u64;
}
@@ -206,14 +181,7 @@ impl Writer {
/// Abort should only be called when the writer is not closed or
/// aborted, otherwise an unexpected error could be returned.
pub async fn abort(&mut self) -> Result<()> {
- if let State::Idle(Some(w)) = &mut self.state {
- w.abort().await
- } else {
- unreachable!(
- "writer state invalid while abort, expect Idle, actual {}",
- self.state
- );
- }
+ self.inner.abort().await
}
/// Close the writer and make sure all data have been committed.
@@ -223,30 +191,7 @@ impl Writer {
/// Close should only be called when the writer is not closed or
/// aborted, otherwise an unexpected error could be returned.
pub async fn close(&mut self) -> Result<()> {
- if let State::Idle(Some(w)) = &mut self.state {
- w.close().await
- } else {
- unreachable!(
- "writer state invalid while close, expect Idle, actual {}",
- self.state
- );
- }
- }
-}
-
-enum State {
- Idle(Option<oio::Writer>),
- Write(BoxFuture<'static, Result<(usize, oio::Writer)>>),
- Close(BoxFuture<'static, Result<oio::Writer>>),
-}
-
-impl Display for State {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- match self {
- State::Idle(_) => write!(f, "Idle"),
- State::Write(_) => write!(f, "Write"),
- State::Close(_) => write!(f, "Close"),
- }
+ self.inner.close().await
}
}
@@ -256,35 +201,9 @@ impl AsyncWrite for Writer {
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<io::Result<usize>> {
- loop {
- match &mut self.state {
- State::Idle(w) => {
- let mut w = w
- .take()
- .expect("invalid state of writer: Idle state with empty write");
- // FIXME: This will the buf everytime, we should avoid this.
- let bs = Bytes::copy_from_slice(buf);
- let fut = async move {
- let n = w.write(&bs).await?;
- Ok((n, w))
- };
- self.state = State::Write(Box::pin(fut));
- }
- State::Write(fut) => match ready!(fut.poll_unpin(cx)) {
- Ok((size, w)) => {
- self.state = State::Idle(Some(w));
- return Poll::Ready(Ok(size));
- }
- Err(err) => {
- self.state = State::Idle(None);
- return Poll::Ready(Err(io::Error::new(io::ErrorKind::Other, err)));
- }
- },
- State::Close(_) => {
- unreachable!("invalid state of writer: poll_write with State::Close")
- }
- };
- }
+ self.inner
+ .poll_write(cx, &buf)
+ .map_err(|err| io::Error::new(io::ErrorKind::Other, err))
}
/// Writer makes sure that every write is flushed.
@@ -293,33 +212,9 @@ impl AsyncWrite for Writer {
}
fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
- loop {
- match &mut self.state {
- State::Idle(w) => {
- let mut w = w
- .take()
- .expect("invalid state of writer: Idle state with empty write");
- let fut = async move {
- w.close().await?;
- Ok(w)
- };
- self.state = State::Close(Box::pin(fut));
- }
- State::Write(_) => {
- unreachable!("invalid state of writer: poll_close with State::Write")
- }
- State::Close(fut) => match ready!(fut.poll_unpin(cx)) {
- Ok(w) => {
- self.state = State::Idle(Some(w));
- return Poll::Ready(Ok(()));
- }
- Err(err) => {
- self.state = State::Idle(None);
- return Poll::Ready(Err(io::Error::new(io::ErrorKind::Other, err)));
- }
- },
- }
- }
+ self.inner
+ .poll_close(cx)
+ .map_err(|err| io::Error::new(io::ErrorKind::Other, err))
}
}
@@ -329,35 +224,9 @@ impl tokio::io::AsyncWrite for Writer {
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<io::Result<usize>> {
- loop {
- match &mut self.state {
- State::Idle(w) => {
- let mut w = w
- .take()
- .expect("invalid state of writer: Idle state with empty write");
- // FIXME: This will the buf everytime, we should avoid this.
- let bs = Bytes::copy_from_slice(buf);
- let fut = async move {
- let n = w.write(&bs).await?;
- Ok((n, w))
- };
- self.state = State::Write(Box::pin(fut));
- }
- State::Write(fut) => match ready!(fut.poll_unpin(cx)) {
- Ok((size, w)) => {
- self.state = State::Idle(Some(w));
- return Poll::Ready(Ok(size));
- }
- Err(err) => {
- self.state = State::Idle(None);
- return Poll::Ready(Err(io::Error::new(io::ErrorKind::Other, err)));
- }
- },
- State::Close(_) => {
- unreachable!("invalid state of writer: poll_write with State::Close")
- }
- };
- }
+ self.inner
+ .poll_write(cx, &buf)
+ .map_err(|err| io::Error::new(io::ErrorKind::Other, err))
}
fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<io::Result<()>> {
@@ -365,33 +234,9 @@ impl tokio::io::AsyncWrite for Writer {
}
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
- loop {
- match &mut self.state {
- State::Idle(w) => {
- let mut w = w
- .take()
- .expect("invalid state of writer: Idle state with empty write");
- let fut = async move {
- w.close().await?;
- Ok(w)
- };
- self.state = State::Close(Box::pin(fut));
- }
- State::Write(_) => {
- unreachable!("invalid state of writer: poll_close with State::Write")
- }
- State::Close(fut) => match ready!(fut.poll_unpin(cx)) {
- Ok(w) => {
- self.state = State::Idle(Some(w));
- return Poll::Ready(Ok(()));
- }
- Err(err) => {
- self.state = State::Idle(None);
- return Poll::Ready(Err(io::Error::new(io::ErrorKind::Other, err)));
- }
- },
- }
- }
+ self.inner
+ .poll_close(cx)
+ .map_err(|err| io::Error::new(io::ErrorKind::Other, err))
}
}
[incubator-opendal] 07/31: Fix build
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit a20afd7b06dfc08755559e970e4fd40354e4ca5b
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 15:31:10 2023 +0800
Fix build
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/azdfs/writer.rs | 5 +--
core/src/services/fs/writer.rs | 65 +++++++++++++++++++++++++++------------
2 files changed, 49 insertions(+), 21 deletions(-)
diff --git a/core/src/services/azdfs/writer.rs b/core/src/services/azdfs/writer.rs
index 100065a7e..774c9ee4b 100644
--- a/core/src/services/azdfs/writer.rs
+++ b/core/src/services/azdfs/writer.rs
@@ -16,6 +16,7 @@
// under the License.
use std::sync::Arc;
+use std::task::{Context, Poll};
use async_trait::async_trait;
use http::StatusCode;
@@ -90,10 +91,10 @@ impl oio::Write for AzdfsWriter {
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ Poll::Ready(Ok(()))
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ Poll::Ready(Ok(()))
}
}
diff --git a/core/src/services/fs/writer.rs b/core/src/services/fs/writer.rs
index 3d94a8888..a71de17d5 100644
--- a/core/src/services/fs/writer.rs
+++ b/core/src/services/fs/writer.rs
@@ -17,9 +17,13 @@
use std::io::Write;
use std::path::PathBuf;
+use std::pin::Pin;
+use std::task::{ready, Context, Poll};
use async_trait::async_trait;
-use tokio::io::AsyncWriteExt;
+use futures::future::BoxFuture;
+use futures::FutureExt;
+use tokio::io::AsyncWrite;
use super::error::parse_io_error;
use crate::raw::*;
@@ -28,7 +32,9 @@ use crate::*;
pub struct FsWriter<F> {
target_path: PathBuf,
tmp_path: Option<PathBuf>,
- f: F,
+
+ f: Option<F>,
+ fut: Option<BoxFuture<'static, Result<()>>>,
}
impl<F> FsWriter<F> {
@@ -37,6 +43,7 @@ impl<F> FsWriter<F> {
target_path,
tmp_path,
f,
+ fut: None,
}
}
}
@@ -44,39 +51,59 @@ impl<F> FsWriter<F> {
#[async_trait]
impl oio::Write for FsWriter<tokio::fs::File> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- self.f.write(bs.chunk()).await.map_err(parse_io_error)
+ let f = self.f.as_mut().expect("FsWriter must be initialized");
+
+ Pin::new(f)
+ .poll_write(cx, bs.chunk())
+ .map_err(parse_io_error)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Err(Error::new(
- ErrorKind::Unsupported,
- "output writer doesn't support abort",
- ))
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
+ self.f = None;
+
+ Poll::Ready(Ok(()))
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.f.sync_all().await.map_err(parse_io_error)?;
+ loop {
+ if let Some(fut) = self.fut.as_mut() {
+ let res = ready!(fut.poll_unpin(cx));
+ self.fut = None;
+ return Poll::Ready(res);
+ }
- if let Some(tmp_path) = &self.tmp_path {
- tokio::fs::rename(tmp_path, &self.target_path)
- .await
- .map_err(parse_io_error)?;
- }
+ let f = self.f.take().expect("FsWriter must be initialized");
+ let tmp_path = self.tmp_path.clone();
+ let target_path = self.target_path.clone();
+ self.fut = Some(Box::pin(async {
+ f.sync_all().await.map_err(parse_io_error)?;
- Ok(())
+ if let Some(tmp_path) = &tmp_path {
+ tokio::fs::rename(tmp_path, &target_path)
+ .await
+ .map_err(parse_io_error)?;
+ }
+
+ Ok(())
+ }));
+ }
}
}
impl oio::BlockingWrite for FsWriter<std::fs::File> {
fn write(&mut self, bs: &dyn oio::WriteBuf) -> Result<usize> {
- self.f.write(bs.chunk()).map_err(parse_io_error)
+ let f = self.f.as_mut().expect("FsWriter must be initialized");
+
+ f.write(bs.chunk()).map_err(parse_io_error)
}
fn close(&mut self) -> Result<()> {
- self.f.sync_all().map_err(parse_io_error)?;
+ if let Some(f) = self.f.take() {
+ f.sync_all().map_err(parse_io_error)?;
- if let Some(tmp_path) = &self.tmp_path {
- std::fs::rename(tmp_path, &self.target_path).map_err(parse_io_error)?;
+ if let Some(tmp_path) = &self.tmp_path {
+ std::fs::rename(tmp_path, &self.target_path).map_err(parse_io_error)?;
+ }
}
Ok(())
[incubator-opendal] 09/31: cleanup gcs
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 977440d1a1f1b3a1d2b3c07cdab27cc9c9606512
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 15:43:06 2023 +0800
cleanup gcs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/gcs/writer.rs | 173 +++++++++++++++++++++-------------------
1 file changed, 90 insertions(+), 83 deletions(-)
diff --git a/core/src/services/gcs/writer.rs b/core/src/services/gcs/writer.rs
index 2436165c4..c21f6b4e5 100644
--- a/core/src/services/gcs/writer.rs
+++ b/core/src/services/gcs/writer.rs
@@ -16,6 +16,7 @@
// under the License.
use std::sync::Arc;
+use std::task::{Context, Poll};
use async_trait::async_trait;
use bytes::Bytes;
@@ -119,94 +120,100 @@ impl GcsWriter {
#[async_trait]
impl oio::Write for GcsWriter {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
-
- let location = match &self.location {
- Some(location) => location,
- None => {
- if self.op.content_length().unwrap_or_default() == size as u64 && self.written == 0
- {
- self.write_oneshot(size as u64, AsyncBody::Bytes(bs.copy_to_bytes(size)))
- .await?;
-
- return Ok(size);
- } else {
- let location = self.initiate_upload().await?;
- self.location = Some(location);
- self.location.as_deref().unwrap()
- }
- }
- };
-
- self.buffer.push(bs.copy_to_bytes(size));
- // Return directly if the buffer is not full
- if self.buffer.len() <= self.write_fixed_size {
- return Ok(size);
- }
-
- let bs = self.buffer.peak_exact(self.write_fixed_size);
-
- match self.write_part(location, bs).await {
- Ok(_) => {
- self.buffer.take(self.write_fixed_size);
- self.written += self.write_fixed_size as u64;
- Ok(size)
- }
- Err(e) => {
- // If the upload fails, we should pop the given bs to make sure
- // write is re-enter safe.
- self.buffer.pop();
- Err(e)
- }
- }
+ // let size = bs.remaining();
+ //
+ // let location = match &self.location {
+ // Some(location) => location,
+ // None => {
+ // if self.op.content_length().unwrap_or_default() == size as u64 && self.written == 0
+ // {
+ // self.write_oneshot(size as u64, AsyncBody::Bytes(bs.copy_to_bytes(size)))
+ // .await?;
+ //
+ // return Ok(size);
+ // } else {
+ // let location = self.initiate_upload().await?;
+ // self.location = Some(location);
+ // self.location.as_deref().unwrap()
+ // }
+ // }
+ // };
+ //
+ // self.buffer.push(bs.copy_to_bytes(size));
+ // // Return directly if the buffer is not full
+ // if self.buffer.len() <= self.write_fixed_size {
+ // return Ok(size);
+ // }
+ //
+ // let bs = self.buffer.peak_exact(self.write_fixed_size);
+ //
+ // match self.write_part(location, bs).await {
+ // Ok(_) => {
+ // self.buffer.take(self.write_fixed_size);
+ // self.written += self.write_fixed_size as u64;
+ // Ok(size)
+ // }
+ // Err(e) => {
+ // // If the upload fails, we should pop the given bs to make sure
+ // // write is re-enter safe.
+ // self.buffer.pop();
+ // Err(e)
+ // }
+ // }
+
+ todo!()
}
fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- let location = if let Some(location) = &self.location {
- location
- } else {
- return Ok(());
- };
-
- let resp = self.core.gcs_abort_resumable_upload(location).await?;
-
- match resp.status().as_u16() {
- // gcs returns 499 if the upload aborted successfully
- // reference: https://cloud.google.com/storage/docs/performing-resumable-uploads#cancel-upload-json
- 499 => {
- resp.into_body().consume().await?;
- self.location = None;
- self.buffer.clear();
- Ok(())
- }
- _ => Err(parse_error(resp).await?),
- }
+ // let location = if let Some(location) = &self.location {
+ // location
+ // } else {
+ // return Ok(());
+ // };
+ //
+ // let resp = self.core.gcs_abort_resumable_upload(location).await?;
+ //
+ // match resp.status().as_u16() {
+ // // gcs returns 499 if the upload aborted successfully
+ // // reference: https://cloud.google.com/storage/docs/performing-resumable-uploads#cancel-upload-json
+ // 499 => {
+ // resp.into_body().consume().await?;
+ // self.location = None;
+ // self.buffer.clear();
+ // Ok(())
+ // }
+ // _ => Err(parse_error(resp).await?),
+ // }
+
+ todo!()
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- let location = if let Some(location) = &self.location {
- location
- } else {
- return Ok(());
- };
-
- let bs = self.buffer.peak_exact(self.buffer.len());
-
- let resp = self
- .core
- .gcs_complete_resumable_upload(location, self.written, bs)
- .await?;
-
- let status = resp.status();
- match status {
- StatusCode::OK => {
- resp.into_body().consume().await?;
-
- self.location = None;
- self.buffer.clear();
- Ok(())
- }
- _ => Err(parse_error(resp).await?),
- }
+ // let location = if let Some(location) = &self.location {
+ // location
+ // } else {
+ // return Ok(());
+ // };
+ //
+ // let bs = self.buffer.peak_exact(self.buffer.len());
+ //
+ // let resp = self
+ // .core
+ // .gcs_complete_resumable_upload(location, self.written, bs)
+ // .await?;
+ //
+ // let status = resp.status();
+ // match status {
+ // StatusCode::OK => {
+ // resp.into_body().consume().await?;
+ //
+ // self.location = None;
+ // self.buffer.clear();
+ // Ok(())
+ // }
+ // _ => Err(parse_error(resp).await?),
+ // }
+
+ todo!()
}
}
[incubator-opendal] 12/31: Fix ipmfs
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 2b75cf72f79145ca37c03da890d1d186a58f7dee
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 16:10:34 2023 +0800
Fix ipmfs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/ipmfs/backend.rs | 4 ++--
core/src/services/ipmfs/writer.rs | 17 +++++------------
2 files changed, 7 insertions(+), 14 deletions(-)
diff --git a/core/src/services/ipmfs/backend.rs b/core/src/services/ipmfs/backend.rs
index aebab57e5..eda6d8895 100644
--- a/core/src/services/ipmfs/backend.rs
+++ b/core/src/services/ipmfs/backend.rs
@@ -65,7 +65,7 @@ impl IpmfsBackend {
impl Accessor for IpmfsBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = IpmfsWriter;
+ type Writer = oio::OneShotWriter<IpmfsWriter>;
type BlockingWriter = ();
type Pager = IpmfsPager;
type BlockingPager = ();
@@ -131,7 +131,7 @@ impl Accessor for IpmfsBackend {
Ok((
RpWrite::default(),
- IpmfsWriter::new(self.clone(), path.to_string()),
+ oio::OneShotWriter::new(IpmfsWriter::new(self.clone(), path.to_string())),
))
}
diff --git a/core/src/services/ipmfs/writer.rs b/core/src/services/ipmfs/writer.rs
index dcecb66d3..e3395dfaf 100644
--- a/core/src/services/ipmfs/writer.rs
+++ b/core/src/services/ipmfs/writer.rs
@@ -17,9 +17,11 @@
use async_trait::async_trait;
use http::StatusCode;
+use std::task::{Context, Poll};
use super::backend::IpmfsBackend;
use super::error::parse_error;
+use crate::raw::oio::WriteBuf;
use crate::raw::*;
use crate::*;
@@ -35,9 +37,8 @@ impl IpmfsWriter {
}
}
-#[async_trait]
-impl oio::Write for IpmfsWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
+impl oio::OneShotWrite for IpmfsWriter {
+ async fn write_once(&self, bs: &dyn WriteBuf) -> Result<()> {
let size = bs.remaining();
let resp = self
.backend
@@ -49,17 +50,9 @@ impl oio::Write for IpmfsWriter {
match status {
StatusCode::CREATED | StatusCode::OK => {
resp.into_body().consume().await?;
- Ok(size)
+ Ok(())
}
_ => Err(parse_error(resp).await?),
}
}
-
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
-
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
}
[incubator-opendal] 27/31: refactor
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 19aff6cda8d93f952201fd8482e89d885e795a05
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Sep 11 15:49:15 2023 +0800
refactor
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/oio/write/multipart_upload_write.rs | 6 +++--
core/src/raw/oio/write/range_write.rs | 34 ++++++++++++++++++++++++
2 files changed, 38 insertions(+), 2 deletions(-)
diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs
index 1b2148cb4..ec23b210c 100644
--- a/core/src/raw/oio/write/multipart_upload_write.rs
+++ b/core/src/raw/oio/write/multipart_upload_write.rs
@@ -25,9 +25,11 @@ use crate::*;
/// MultipartUploadWrite is used to implement [`Write`] based on multipart
/// uploads. By implementing MultipartUploadWrite, services don't need to
-/// care about the details of buffering and uploading parts.
+/// care about the details of uploading parts.
///
-/// The layout after adopting [`MultipartUploadWrite`]:
+/// # Architecture
+///
+/// The architecture after adopting [`MultipartUploadWrite`]:
///
/// - Services impl `MultipartUploadWrite`
/// - `MultipartUploadWriter` impl `Write`
diff --git a/core/src/raw/oio/write/range_write.rs b/core/src/raw/oio/write/range_write.rs
index 56276327a..32f832630 100644
--- a/core/src/raw/oio/write/range_write.rs
+++ b/core/src/raw/oio/write/range_write.rs
@@ -24,10 +24,33 @@ use crate::raw::oio::WriteBuf;
use crate::raw::*;
use crate::*;
+/// RangeWrite is used to implement [`Write`] based on range write.
+///
+/// # Services
+///
+/// Services like gcs support range write via [GCS Resumable Upload](https://cloud.google.com/storage/docs/resumable-uploads).
+///
+/// GCS will support upload content by specifying the range of the file in `CONTENT-RANGE`.
+///
+/// Most range based services will have the following limitations:
+///
+/// - The size of chunk per upload must be aligned to a certain size. For example, GCS requires
+/// to align with 256KiB.
+/// - Some services requires to complete the write at the last chunk with the total size.
+///
+/// # Architecture
+///
+/// The architecture after adopting [`RangeWrite`]:
+///
+/// - Services impl `RangeWrite`
+/// - `RangeWriter` impl `Write`
+/// - Expose `RangeWriter` as `Accessor::Writer`
#[async_trait]
pub trait RangeWrite: Send + Sync + Unpin + 'static {
+ /// Initiate range the range write, the returning value is the location.
async fn initiate_range(&self) -> Result<String>;
+ /// write_range will write a range of data.
async fn write_range(
&self,
location: &str,
@@ -36,6 +59,7 @@ pub trait RangeWrite: Send + Sync + Unpin + 'static {
body: AsyncBody,
) -> Result<()>;
+ /// complete_range will complete the range write by uploading the last chunk.
async fn complete_range(
&self,
location: &str,
@@ -44,9 +68,11 @@ pub trait RangeWrite: Send + Sync + Unpin + 'static {
body: AsyncBody,
) -> Result<()>;
+ /// abort_range will abort the range write by abort all already uploaded data.
async fn abort_range(&self, location: &str) -> Result<()>;
}
+/// RangeWriter will implements [`Write`] based on range write.
pub struct RangeWriter<W: RangeWrite> {
location: Option<String>,
written: u64,
@@ -83,6 +109,14 @@ impl<W: RangeWrite> RangeWriter<W> {
}
}
+ /// Set the align size.
+ ///
+ /// The size is default to 256 KiB.
+ ///
+ /// # Note
+ ///
+ /// Please don't mix this with the buffer size. Align size is usually the hard
+ /// limit for the service to accept the chunk.
pub fn with_align_size(mut self, size: usize) -> Self {
self.align_size = size;
self
[incubator-opendal] 25/31: Make clippy happy
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit f46df2490c98fd3eb97387ec0ba7ad81475ff900
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Sep 11 14:12:41 2023 +0800
Make clippy happy
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/oio/write/range_write.rs | 18 ++----------------
core/src/services/gcs/core.rs | 1 -
core/src/services/gcs/writer.rs | 4 ----
core/src/services/ghac/writer.rs | 4 ++--
4 files changed, 4 insertions(+), 23 deletions(-)
diff --git a/core/src/raw/oio/write/range_write.rs b/core/src/raw/oio/write/range_write.rs
index a96af8d3d..56276327a 100644
--- a/core/src/raw/oio/write/range_write.rs
+++ b/core/src/raw/oio/write/range_write.rs
@@ -89,20 +89,6 @@ impl<W: RangeWrite> RangeWriter<W> {
}
}
-impl<W: RangeWrite> RangeWriter<W> {
- fn align(&mut self, bs: &dyn WriteBuf) -> usize {
- let remaining = bs.remaining();
- let current_size = self.align_buffer.len();
-
- let total_size = current_size + remaining;
- if total_size <= self.align_size {
- return remaining;
- }
-
- total_size - total_size % self.align_size - current_size
- }
-}
-
impl<W: RangeWrite> oio::Write for RangeWriter<W> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn WriteBuf) -> Poll<Result<usize>> {
loop {
@@ -191,7 +177,7 @@ impl<W: RangeWrite> oio::Write for RangeWriter<W> {
self.align_buffer.len() > 0,
"RangeWriter requires to have last chunk"
);
- let mut align_buffer = self.align_buffer.clone();
+ let align_buffer = self.align_buffer.clone();
let written = self.written;
let fut = async move {
@@ -243,7 +229,7 @@ impl<W: RangeWrite> oio::Write for RangeWriter<W> {
(w, res)
};
- self.state = State::Complete(Box::pin(fut));
+ self.state = State::Abort(Box::pin(fut));
}
None => return Poll::Ready(Ok(())),
}
diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs
index 82c6d9054..06d772e06 100644
--- a/core/src/services/gcs/core.rs
+++ b/core/src/services/gcs/core.rs
@@ -23,7 +23,6 @@ use std::time::Duration;
use backon::ExponentialBuilder;
use backon::Retryable;
-use bytes::Bytes;
use http::header::CONTENT_LENGTH;
use http::header::CONTENT_RANGE;
use http::header::CONTENT_TYPE;
diff --git a/core/src/services/gcs/writer.rs b/core/src/services/gcs/writer.rs
index 6b66240f7..1b22acd7b 100644
--- a/core/src/services/gcs/writer.rs
+++ b/core/src/services/gcs/writer.rs
@@ -15,14 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use std::fmt::{Debug, Formatter};
use std::sync::Arc;
-use std::task::{ready, Context, Poll};
use async_trait::async_trait;
use bytes::Bytes;
-use futures::future::BoxFuture;
-use futures::FutureExt;
use http::StatusCode;
use super::core::GcsCore;
diff --git a/core/src/services/ghac/writer.rs b/core/src/services/ghac/writer.rs
index 439bbbb2e..09eb82d8f 100644
--- a/core/src/services/ghac/writer.rs
+++ b/core/src/services/ghac/writer.rs
@@ -124,7 +124,7 @@ impl oio::Write for GhacWriter {
if resp.status().is_success() {
resp.into_body().consume().await?;
- Ok(size as usize)
+ Ok(())
} else {
Err(parse_error(resp)
.await
@@ -135,7 +135,7 @@ impl oio::Write for GhacWriter {
(backend, res)
};
- self.state = State::Upload(Box::pin(fut));
+ self.state = State::Commit(Box::pin(fut));
}
State::Upload(_) => {
unreachable!("GhacWriter must not go into State:Upload during poll_close")
[incubator-opendal] 06/31: Fix writer
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit cd167a5b0cb7a79a0405606e168e281921d9d5d9
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 14:51:09 2023 +0800
Fix writer
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/azblob/writer.rs | 28 ++++++----------------------
1 file changed, 6 insertions(+), 22 deletions(-)
diff --git a/core/src/services/azblob/writer.rs b/core/src/services/azblob/writer.rs
index 1a228b647..70b023249 100644
--- a/core/src/services/azblob/writer.rs
+++ b/core/src/services/azblob/writer.rs
@@ -16,6 +16,7 @@
// under the License.
use std::sync::Arc;
+use std::task::{Context, Poll};
use async_trait::async_trait;
use http::StatusCode;
@@ -161,31 +162,14 @@ impl AzblobWriter {
#[async_trait]
impl oio::Write for AzblobWriter {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
-
- if self.op.append() {
- self.append_oneshot(size as u64, AsyncBody::Bytes(bs.copy_to_bytes(size)))
- .await?;
- } else {
- if self.op.content_length().is_none() {
- return Err(Error::new(
- ErrorKind::Unsupported,
- "write without content length is not supported",
- ));
- }
-
- self.write_oneshot(size as u64, AsyncBody::Bytes(bs.copy_to_bytes(size)))
- .await?;
- }
-
- Ok(size)
+ todo!()
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
+ Poll::Ready(Ok(()))
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
+ Poll::Ready(Ok(()))
}
}
[incubator-opendal] 14/31: Fix build
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit a2575f825c8394fa2e089b880c14c50ac7bcccdc
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 16:19:01 2023 +0800
Fix build
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/oio/write/one_shot_write.rs | 12 ++++++++++--
core/src/services/supabase/writer.rs | 29 ++++++-----------------------
core/src/services/wasabi/writer.rs | 1 +
3 files changed, 17 insertions(+), 25 deletions(-)
diff --git a/core/src/raw/oio/write/one_shot_write.rs b/core/src/raw/oio/write/one_shot_write.rs
index ee15c438f..a0a743915 100644
--- a/core/src/raw/oio/write/one_shot_write.rs
+++ b/core/src/raw/oio/write/one_shot_write.rs
@@ -16,6 +16,7 @@
// under the License.
use async_trait::async_trait;
+use bytes::Bytes;
use futures::future::BoxFuture;
use std::task::{ready, Context, Poll};
@@ -33,7 +34,7 @@ pub trait OneShotWrite: Send + Sync + Unpin {
/// write_once write all data at once.
///
/// Implementations should make sure that the data is written correctly at once.
- async fn write_once(&self, body: &dyn oio::WriteBuf) -> Result<()>;
+ async fn write_once(&self, bs: Bytes) -> Result<()>;
}
/// OneShotWrite is used to implement [`Write`] based on one shot.
@@ -62,7 +63,14 @@ impl<W: OneShotWrite> oio::Write for OneShotWriter<W> {
match &mut self.state {
State::Idle(w) => {
let w = w.take().expect("writer must be valid");
- let fut = w.write_once(bs);
+
+ let size = bs.remaining();
+ let bs = bs.copy_to_bytes(size);
+ let fut = async {
+ let res = w.write_once(bs).await;
+
+ (w, res.map(|_| size))
+ };
self.state = State::Write(Box::pin(fut));
}
diff --git a/core/src/services/supabase/writer.rs b/core/src/services/supabase/writer.rs
index d8ba4cc16..3b692fd08 100644
--- a/core/src/services/supabase/writer.rs
+++ b/core/src/services/supabase/writer.rs
@@ -16,6 +16,7 @@
// under the License.
use std::sync::Arc;
+use std::task::{Context, Poll};
use async_trait::async_trait;
use bytes::Bytes;
@@ -41,14 +42,16 @@ impl SupabaseWriter {
path: path.to_string(),
}
}
+}
- pub async fn upload(&self, bytes: Bytes) -> Result<()> {
- let size = bytes.len();
+impl oio::OneShotWrite for SupabaseWriter {
+ async fn write_once(&self, bs: Bytes) -> Result<()> {
+ let size = bs.len();
let mut req = self.core.supabase_upload_object_request(
&self.path,
Some(size),
self.op.content_type(),
- AsyncBody::Bytes(bytes),
+ AsyncBody::Bytes(bs),
)?;
self.core.sign(&mut req)?;
@@ -64,23 +67,3 @@ impl SupabaseWriter {
}
}
}
-
-#[async_trait]
-impl oio::Write for SupabaseWriter {
- fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
- self.upload(bs.copy_to_bytes(size)).await?;
- Ok(size)
- }
-
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Err(Error::new(
- ErrorKind::Unsupported,
- "The abort operation is not yet supported for Supabase backend",
- ))
- }
-
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
- }
-}
diff --git a/core/src/services/wasabi/writer.rs b/core/src/services/wasabi/writer.rs
index 3eeaab12d..1c7b5ed2e 100644
--- a/core/src/services/wasabi/writer.rs
+++ b/core/src/services/wasabi/writer.rs
@@ -16,6 +16,7 @@
// under the License.
use std::sync::Arc;
+use std::task::Context;
use async_trait::async_trait;
use http::StatusCode;
[incubator-opendal] 05/31: save work
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 5d78f4f653845ee9d2a4aa85ae820b2d5ace9795
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 14:48:12 2023 +0800
save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/oio/write/one_shot_write.rs | 39 ++++++++++++++++++++++++--------
1 file changed, 30 insertions(+), 9 deletions(-)
diff --git a/core/src/raw/oio/write/one_shot_write.rs b/core/src/raw/oio/write/one_shot_write.rs
index 85ef499d5..ee15c438f 100644
--- a/core/src/raw/oio/write/one_shot_write.rs
+++ b/core/src/raw/oio/write/one_shot_write.rs
@@ -16,6 +16,8 @@
// under the License.
use async_trait::async_trait;
+use futures::future::BoxFuture;
+use std::task::{ready, Context, Poll};
use crate::raw::*;
use crate::*;
@@ -36,29 +38,48 @@ pub trait OneShotWrite: Send + Sync + Unpin {
/// OneShotWrite is used to implement [`Write`] based on one shot.
pub struct OneShotWriter<W: OneShotWrite> {
- inner: W,
+ state: State<W>,
+}
+
+enum State<W> {
+ Idle(Option<W>),
+ Write(BoxFuture<'static, (W, Result<usize>)>),
}
impl<W: OneShotWrite> OneShotWriter<W> {
/// Create a new one shot writer.
pub fn new(inner: W) -> Self {
- Self { inner }
+ Self {
+ state: State::Idle(Some(inner)),
+ }
}
}
#[async_trait]
impl<W: OneShotWrite> oio::Write for OneShotWriter<W> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- let size = bs.remaining();
- self.inner.write_once(bs).await?;
- Ok(size)
+ loop {
+ match &mut self.state {
+ State::Idle(w) => {
+ let w = w.take().expect("writer must be valid");
+ let fut = w.write_once(bs);
+
+ self.state = State::Write(Box::pin(fut));
+ }
+ State::Write(fut) => {
+ let (w, size) = ready!(fut.as_mut().poll(cx));
+ self.state = State::Idle(Some(w));
+ return Poll::Ready(size);
+ }
+ }
+ }
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
+ Poll::Ready(Ok(()))
}
- fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Ok(())
+ fn poll_close(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
+ Poll::Ready(Ok(()))
}
}
[incubator-opendal] 11/31: Fix build for hdfs
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit c7c4819c2fdddc7cec9fc15f76ad5b2cfa39d4f9
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 8 16:08:43 2023 +0800
Fix build for hdfs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/hdfs/writer.rs | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git a/core/src/services/hdfs/writer.rs b/core/src/services/hdfs/writer.rs
index 1b4f549ce..ced19efb9 100644
--- a/core/src/services/hdfs/writer.rs
+++ b/core/src/services/hdfs/writer.rs
@@ -16,10 +16,11 @@
// under the License.
use std::io::Write;
-use std::task::Context;
+use std::pin::Pin;
+use std::task::{Context, Poll};
use async_trait::async_trait;
-use futures::AsyncWriteExt;
+use futures::{AsyncWrite, AsyncWriteExt};
use super::error::parse_io_error;
use crate::raw::*;
@@ -38,20 +39,17 @@ impl<F> HdfsWriter<F> {
#[async_trait]
impl oio::Write for HdfsWriter<hdrs::AsyncFile> {
fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
- self.f.write(bs.chunk()).await.map_err(parse_io_error)
+ Pin::new(&mut self.f)
+ .poll_write(cx, bs.chunk())
+ .map_err(parse_io_error)
}
- fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- Err(Error::new(
- ErrorKind::Unsupported,
- "output writer doesn't support abort",
- ))
+ fn poll_abort(&mut self, _: &mut Context<'_>) -> Poll<Result<()>> {
+ Poll::Ready(Ok(()))
}
fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
- self.f.close().await.map_err(parse_io_error)?;
-
- Ok(())
+ Pin::new(&mut self.f).poll_close(cx).map_err(parse_io_error)
}
}
[incubator-opendal] 29/31: Cleanup
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch poll-write
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 42b2a9c668192f33db123c72fb91027a4ec1c3e8
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Sep 11 16:20:58 2023 +0800
Cleanup
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/oio/write/range_write.rs | 4 ----
1 file changed, 4 deletions(-)
diff --git a/core/src/raw/oio/write/range_write.rs b/core/src/raw/oio/write/range_write.rs
index 32f832630..ae4558a4c 100644
--- a/core/src/raw/oio/write/range_write.rs
+++ b/core/src/raw/oio/write/range_write.rs
@@ -207,10 +207,6 @@ impl<W: RangeWrite> oio::Write for RangeWriter<W> {
let w = w.take().unwrap();
match self.location.clone() {
Some(location) => {
- debug_assert!(
- self.align_buffer.len() > 0,
- "RangeWriter requires to have last chunk"
- );
let align_buffer = self.align_buffer.clone();
let written = self.written;