You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/08/27 07:55:37 UTC
(opendal) branch xuanwo/fix-behavior-main-ci created (now cf952ede68)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a change to branch xuanwo/fix-behavior-main-ci
in repository https://gitbox.apache.org/repos/asf/opendal.git
at cf952ede68 fix(services/azfile): split range writes at 4 MiB
This branch includes the following new commits:
new c8973b3c04 fix(core): respect stat condition capabilities
new 5db45b0e63 fix(services/b2): preserve range in presigned reads
new cf952ede68 fix(services/azfile): split range writes at 4 MiB
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opendal) 03/03: fix(services/azfile): split range writes at 4 MiB
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/fix-behavior-main-ci
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit cf952ede68b1bc44fca4965c448c6928427178e2
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Thu Aug 27 15:55:07 2026 +0800
fix(services/azfile): split range writes at 4 MiB
---
core/services/azfile/src/writer.rs | 71 +++++++++++++++++++++++++-------------
1 file changed, 47 insertions(+), 24 deletions(-)
diff --git a/core/services/azfile/src/writer.rs b/core/services/azfile/src/writer.rs
index 33e9ae9dd7..f367465e81 100644
--- a/core/services/azfile/src/writer.rs
+++ b/core/services/azfile/src/writer.rs
@@ -24,6 +24,10 @@ use super::core::parse_error;
use opendal_core::raw::*;
use opendal_core::*;
+// Azure Files limits each Put Range update to 4 MiB.
+// https://learn.microsoft.com/en-us/rest/api/storageservices/put-range
+const AZFILE_MAX_WRITE_SIZE: usize = 4 * 1024 * 1024;
+
pub type AzfileWriters = TwoWays<oio::OneShotWriter<AzfileWriter>, oio::AppendWriter<AzfileWriter>>;
pub struct AzfileWriter {
@@ -49,6 +53,46 @@ impl AzfileWriter {
.await
}
+ async fn write_ranges(&self, offset: u64, bs: Buffer) -> Result<Metadata> {
+ let size = bs.len();
+ let mut position = 0;
+ let mut metadata;
+
+ loop {
+ let chunk_size = (size - position).min(AZFILE_MAX_WRITE_SIZE);
+ let end = position + chunk_size;
+ let body = bs.slice(position..end);
+
+ let resp = self
+ .core
+ .azfile_update(
+ &self.ctx,
+ &self.path,
+ chunk_size as u64,
+ offset + position as u64,
+ body,
+ )
+ .await?;
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::CREATED => {
+ metadata = AzfileWriter::parse_metadata(resp.headers())?;
+ }
+ _ => {
+ return Err(parse_error(resp).with_operation("Backend::azfile_update"));
+ }
+ }
+
+ position = end;
+ if position == size {
+ break;
+ }
+ }
+
+ metadata.set_content_length(offset + size as u64);
+ Ok(metadata)
+ }
+
fn parse_metadata(headers: &http::HeaderMap) -> Result<Metadata> {
let mut metadata = Metadata::default();
@@ -82,17 +126,7 @@ impl oio::OneShotWrite for AzfileWriter {
}
}
- let resp = self
- .core
- .azfile_update(&self.ctx, &self.path, size as u64, 0, bs)
- .await?;
- let status = resp.status();
- let mut meta = AzfileWriter::parse_metadata(resp.headers())?;
- meta.set_content_length(size as u64);
- match status {
- StatusCode::OK | StatusCode::CREATED => Ok(meta),
- _ => Err(parse_error(resp).with_operation("Backend::azfile_update")),
- }
+ self.write_ranges(0, bs).await
}
}
@@ -113,18 +147,7 @@ impl oio::AppendWrite for AzfileWriter {
}
}
- async fn append(&self, offset: u64, size: u64, body: Buffer) -> Result<Metadata> {
- let resp = self
- .core
- .azfile_update(&self.ctx, &self.path, size, offset, body)
- .await?;
-
- let status = resp.status();
- let mut meta = AzfileWriter::parse_metadata(resp.headers())?;
- meta.set_content_length(offset + size);
- match status {
- StatusCode::OK | StatusCode::CREATED => Ok(meta),
- _ => Err(parse_error(resp).with_operation("Backend::azfile_update")),
- }
+ async fn append(&self, offset: u64, _size: u64, body: Buffer) -> Result<Metadata> {
+ self.write_ranges(offset, body).await
}
}
(opendal) 02/03: fix(services/b2): preserve range in presigned reads
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/fix-behavior-main-ci
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 5db45b0e63eca96cb12f9cf3d3c84fce7cd3ca3f
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Thu Aug 27 15:55:02 2026 +0800
fix(services/b2): preserve range in presigned reads
---
core/services/b2/src/backend.rs | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/core/services/b2/src/backend.rs b/core/services/b2/src/backend.rs
index 883c0b0c09..bf3047e062 100644
--- a/core/services/b2/src/backend.rs
+++ b/core/services/b2/src/backend.rs
@@ -393,7 +393,7 @@ impl Service for B2Backend {
parts.headers,
)))
}
- PresignOperation::Read(_, _) => {
+ PresignOperation::Read(range, _) => {
let resp = self
.core
.get_download_authorization(ctx, path, args.expire())
@@ -410,7 +410,10 @@ impl Service for B2Backend {
resp.authorization_token
);
- let req = Request::get(url);
+ let mut req = Request::get(url);
+ if !range.is_full() {
+ req = req.header(http::header::RANGE, range.to_header());
+ }
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
(opendal) 01/03: fix(core): respect stat condition capabilities
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/fix-behavior-main-ci
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit c8973b3c04dd46f9e890ac912d92698b1addb3e2
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Thu Aug 27 15:54:58 2026 +0800
fix(core): respect stat condition capabilities
---
core/core/src/types/context/read.rs | 46 +++++++++++++++++++++++++++++++++----
1 file changed, 42 insertions(+), 4 deletions(-)
diff --git a/core/core/src/types/context/read.rs b/core/core/src/types/context/read.rs
index 3f0a017887..cbbfb90df0 100644
--- a/core/core/src/types/context/read.rs
+++ b/core/core/src/types/context/read.rs
@@ -123,20 +123,29 @@ impl ReadContext {
}
let mut op_stat = OpStat::new();
+ let capability = self.srv.capability();
if let Some(v) = self.args().version() {
op_stat = op_stat.with_version(v);
}
- if let Some(v) = self.args().if_match() {
+ if capability.stat_with_if_match
+ && let Some(v) = self.args().if_match()
+ {
op_stat = op_stat.with_if_match(v);
}
- if let Some(v) = self.args().if_none_match() {
+ if capability.stat_with_if_none_match
+ && let Some(v) = self.args().if_none_match()
+ {
op_stat = op_stat.with_if_none_match(v);
}
- if let Some(v) = self.args().if_modified_since() {
+ if capability.stat_with_if_modified_since
+ && let Some(v) = self.args().if_modified_since()
+ {
op_stat = op_stat.with_if_modified_since(v);
}
- if let Some(v) = self.args().if_unmodified_since() {
+ if capability.stat_with_if_unmodified_since
+ && let Some(v) = self.args().if_unmodified_since()
+ {
op_stat = op_stat.with_if_unmodified_since(v);
}
@@ -271,6 +280,8 @@ mod tests {
use bytes::Bytes;
use super::*;
+ use crate::layers::CapabilityOverrideLayer;
+ use crate::layers::CorrectnessCheckLayer;
fn new_read_context(
ctx: OperationContext,
@@ -410,4 +421,31 @@ mod tests {
pretty_assertions::assert_eq!(range, 32..42);
Ok(())
}
+
+ #[tokio::test]
+ async fn test_parse_into_range_skips_unsupported_stat_conditions() -> Result<()> {
+ let op = Operator::new(services::Memory::default())?;
+ op.write("test", Buffer::from("HelloWorld")).await?;
+
+ let srv = CapabilityOverrideLayer::new(|mut capability| {
+ capability.read_with_if_match = true;
+ capability.stat_with_if_match = false;
+ capability
+ })
+ .apply_service(op.base_service().clone());
+ let srv = CorrectnessCheckLayer.apply_service(srv);
+
+ let (_, args, options) = options::ReadOptions {
+ if_match: Some("invalid-etag".to_string()),
+ ..Default::default()
+ }
+ .into();
+ let ctx =
+ new_read_context_with_args(op.base_context().clone(), srv, "test", args, options)?;
+
+ let range = ctx.parse_into_range(..).await?;
+
+ pretty_assertions::assert_eq!(range, 0..10);
+ Ok(())
+ }
}