You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/05/12 05:45:57 UTC
(opendal) 01/03: feat(core): add capability override layer
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/capability-override-tests
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 2602ab816d5484a8184d012fcf89dd3b30c42ec0
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue May 12 13:26:34 2026 +0800
feat(core): add capability override layer
---
core/Cargo.lock | 1 +
core/core/src/layers/capability_override.rs | 110 +++++++++++++++++++
core/core/src/layers/mod.rs | 3 +
core/core/src/types/capability.rs | 6 +-
core/testkit/Cargo.toml | 1 +
core/testkit/src/utils.rs | 161 +++++++++++++++++++++++++++-
core/tests/behavior/README.md | 10 ++
core/tests/behavior/async_delete.rs | 6 +-
8 files changed, 293 insertions(+), 5 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index 829f24a0e..8fcbb4ea0 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -7708,6 +7708,7 @@ dependencies = [
"opendal-layer-retry",
"opendal-layer-timeout",
"rand 0.8.6",
+ "serde_json",
"sha2 0.11.0",
"tokio",
"uuid",
diff --git a/core/core/src/layers/capability_override.rs b/core/core/src/layers/capability_override.rs
new file mode 100644
index 000000000..65cb2e0a1
--- /dev/null
+++ b/core/core/src/layers/capability_override.rs
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt;
+use std::sync::Arc;
+
+use crate::raw::*;
+use crate::*;
+
+/// Layer for overriding an accessor's full capability.
+///
+/// This layer updates [`Capability`] exposed by
+/// [`OperatorInfo::full_capability`][crate::OperatorInfo::full_capability]
+/// without changing the accessor's native capability. It is useful when the
+/// backend implementation supports a capability, but the specific endpoint or
+/// test setup needs to disable or tune it.
+///
+/// # Examples
+///
+/// ```no_run
+/// use opendal_core::layers::CapabilityOverrideLayer;
+/// use opendal_core::services;
+/// use opendal_core::Operator;
+/// use opendal_core::Result;
+///
+/// # fn main() -> Result<()> {
+/// let op = Operator::new(services::Memory::default())?
+/// .layer(CapabilityOverrideLayer::new(|mut cap| {
+/// cap.write_with_if_match = false;
+/// cap.delete_max_size = Some(700);
+/// cap
+/// }))
+/// .finish();
+/// # Ok(())
+/// # }
+/// ```
+#[derive(Clone)]
+pub struct CapabilityOverrideLayer {
+ apply: Arc<dyn Fn(Capability) -> Capability + Send + Sync>,
+}
+
+impl fmt::Debug for CapabilityOverrideLayer {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("CapabilityOverrideLayer")
+ .finish_non_exhaustive()
+ }
+}
+
+impl CapabilityOverrideLayer {
+ /// Create a new [`CapabilityOverrideLayer`].
+ pub fn new(apply: impl Fn(Capability) -> Capability + Send + Sync + 'static) -> Self {
+ Self {
+ apply: Arc::new(apply),
+ }
+ }
+}
+
+impl<A: Access> Layer<A> for CapabilityOverrideLayer {
+ type LayeredAccess = A;
+
+ fn layer(&self, inner: A) -> Self::LayeredAccess {
+ let info = inner.info();
+ let apply = self.apply.clone();
+ info.update_full_capability(|cap| apply(cap));
+ inner
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::Operator;
+ use crate::services;
+
+ #[test]
+ fn capability_override_updates_full_capability_only() -> Result<()> {
+ let op = Operator::new(services::Memory::default())?
+ .layer(CapabilityOverrideLayer::new(|mut cap| {
+ cap.read = false;
+ cap.delete_max_size = Some(7);
+ cap
+ }))
+ .finish();
+
+ assert!(!op.info().full_capability().read);
+ assert_eq!(op.info().full_capability().delete_max_size, Some(7));
+
+ assert!(op.info().native_capability().read);
+ assert_ne!(
+ op.info().native_capability().delete_max_size,
+ op.info().full_capability().delete_max_size
+ );
+
+ Ok(())
+ }
+}
diff --git a/core/core/src/layers/mod.rs b/core/core/src/layers/mod.rs
index 3a1f45bf7..f81b8c837 100644
--- a/core/core/src/layers/mod.rs
+++ b/core/core/src/layers/mod.rs
@@ -29,6 +29,9 @@ pub(crate) use complete::CompleteLayer;
mod simulate;
pub use simulate::SimulateLayer;
+mod capability_override;
+pub use capability_override::CapabilityOverrideLayer;
+
mod correctness_check;
pub(crate) use correctness_check::CorrectnessCheckLayer;
diff --git a/core/core/src/types/capability.rs b/core/core/src/types/capability.rs
index 7c49eee18..e883d316b 100644
--- a/core/core/src/types/capability.rs
+++ b/core/core/src/types/capability.rs
@@ -17,6 +17,9 @@
use std::fmt::Debug;
+use serde::Deserialize;
+use serde::Serialize;
+
/// Capability defines the supported operations and their constraints for a storage Operator.
///
/// # Overview
@@ -62,7 +65,8 @@ use std::fmt::Debug;
/// - Metadata Results: Returning metadata capabilities (e.g., `stat_has_content_length`)
///
/// All capability fields are public and can be accessed directly.
-#[derive(Copy, Clone, Default)]
+#[derive(Copy, Clone, Default, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
pub struct Capability {
/// Indicates if the operator supports metadata retrieval operations.
pub stat: bool,
diff --git a/core/testkit/Cargo.toml b/core/testkit/Cargo.toml
index 537964a84..a6c2396d7 100644
--- a/core/testkit/Cargo.toml
+++ b/core/testkit/Cargo.toml
@@ -38,6 +38,7 @@ opendal-layer-logging = { path = "../layers/logging", version = "0.56.0", defaul
opendal-layer-retry = { path = "../layers/retry", version = "0.56.0", default-features = false }
opendal-layer-timeout = { path = "../layers/timeout", version = "0.56.0", default-features = false }
rand = { workspace = true }
+serde_json = { workspace = true }
sha2 = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread"] }
uuid = { workspace = true }
diff --git a/core/testkit/src/utils.rs b/core/testkit/src/utils.rs
index e356ba038..01c219ebe 100644
--- a/core/testkit/src/utils.rs
+++ b/core/testkit/src/utils.rs
@@ -19,14 +19,23 @@ use std::collections::HashMap;
use std::env;
use std::sync::LazyLock;
+use opendal_core::Capability;
+use opendal_core::Error;
+use opendal_core::ErrorKind;
use opendal_core::Operator;
use opendal_core::Result;
+use opendal_core::layers::CapabilityOverrideLayer;
use opendal_layer_logging::LoggingLayer;
use opendal_layer_retry::RetryLayer;
use opendal_layer_timeout::TimeoutLayer;
+use serde_json::Map;
+use serde_json::Number;
+use serde_json::Value;
use sha2::Digest;
use sha2::Sha256;
+const OPENDAL_TEST_CAPABILITY_OVERRIDES: &str = "OPENDAL_TEST_CAPABILITY_OVERRIDES";
+
pub(crate) fn sha256_digest(data: impl AsRef<[u8]>) -> String {
use std::fmt::Write;
@@ -86,7 +95,14 @@ pub fn init_test_service() -> Result<Option<Operator>> {
// string-based scheme uses a hyphen ('-') as the connector
let scheme = scheme.replace('_', "-");
- let op = Operator::via_iter(scheme, cfg).expect("must succeed");
+ let mut op = Operator::via_iter(scheme, cfg).expect("must succeed");
+
+ if let Ok(overrides) = env::var(OPENDAL_TEST_CAPABILITY_OVERRIDES) {
+ let overrides = CapabilityOverrides::parse(&overrides)?;
+ op = op.layer(CapabilityOverrideLayer::new(move |cap| {
+ overrides.apply(cap)
+ }));
+ }
let op = op
.layer(LoggingLayer::default())
@@ -95,3 +111,146 @@ pub fn init_test_service() -> Result<Option<Operator>> {
Ok(Some(op))
}
+
+#[derive(Clone, Debug, Default)]
+struct CapabilityOverrides {
+ values: Map<String, Value>,
+}
+
+impl CapabilityOverrides {
+ fn parse(input: &str) -> Result<Self> {
+ let mut overrides = Self::default();
+
+ for token in input.split(',').map(str::trim).filter(|v| !v.is_empty()) {
+ let (name, value) = parse_capability_override(token)?;
+ overrides.values.insert(name.to_string(), value);
+ overrides.try_apply(Capability::default()).map_err(|err| {
+ invalid_capability_override(token, &format!("failed to apply override: {err}"))
+ })?;
+ }
+
+ Ok(overrides)
+ }
+
+ fn apply(&self, cap: Capability) -> Capability {
+ self.try_apply(cap)
+ .expect("capability overrides must be validated before applying")
+ }
+
+ fn try_apply(&self, cap: Capability) -> Result<Capability> {
+ let mut value = serde_json::to_value(cap).map_err(|err| {
+ Error::new(
+ ErrorKind::Unexpected,
+ format!("failed to serialize capability: {err}"),
+ )
+ })?;
+ let object = value.as_object_mut().ok_or_else(|| {
+ Error::new(
+ ErrorKind::Unexpected,
+ "serialized capability must be a JSON object",
+ )
+ })?;
+ object.extend(self.values.clone());
+
+ serde_json::from_value(value).map_err(|err| {
+ Error::new(
+ ErrorKind::ConfigInvalid,
+ format!("failed to deserialize capability overrides: {err}"),
+ )
+ })
+ }
+}
+
+fn parse_capability_override(token: &str) -> Result<(&str, Value)> {
+ if let Some(name) = token.strip_prefix('+') {
+ return Ok((name.trim(), Value::Bool(true)));
+ }
+
+ if let Some(name) = token.strip_prefix('-') {
+ return Ok((name.trim(), Value::Bool(false)));
+ }
+
+ let Some((name, value)) = token.split_once('=') else {
+ return Err(invalid_capability_override(
+ token,
+ "expected `+capability`, `-capability`, or `capability=value`",
+ ));
+ };
+
+ Ok((
+ name.trim(),
+ parse_capability_value(value.trim())
+ .map_err(|err| invalid_capability_override(token, &err.to_string()))?,
+ ))
+}
+
+fn invalid_capability_override(token: &str, reason: &str) -> Error {
+ Error::new(
+ ErrorKind::ConfigInvalid,
+ format!("invalid {OPENDAL_TEST_CAPABILITY_OVERRIDES} entry `{token}`: {reason}"),
+ )
+}
+
+fn parse_capability_value(value: &str) -> Result<Value> {
+ match value {
+ "true" | "on" | "yes" => Ok(Value::Bool(true)),
+ "false" | "off" | "no" => Ok(Value::Bool(false)),
+ "none" | "null" | "unset" => Ok(Value::Null),
+ _ => value
+ .parse::<usize>()
+ .map(|v| Value::Number(Number::from(v)))
+ .map_err(|_| {
+ Error::new(
+ ErrorKind::ConfigInvalid,
+ "expected a boolean, non-negative integer, or `none`",
+ )
+ }),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn parse_capability_overrides() {
+ let overrides = CapabilityOverrides::parse("-read,+write_can_append,delete_max_size=7")
+ .expect("override must parse");
+ let cap = overrides.apply(Capability {
+ read: true,
+ delete_max_size: Some(1000),
+ ..Default::default()
+ });
+
+ assert!(!cap.read);
+ assert!(cap.write_can_append);
+ assert_eq!(cap.delete_max_size, Some(7));
+ }
+
+ #[test]
+ fn parse_bool_assignments_and_unset_sizes() {
+ let overrides = CapabilityOverrides::parse("read=false,write=true,delete_max_size=none")
+ .expect("override must parse");
+ let cap = overrides.apply(Capability {
+ read: true,
+ delete_max_size: Some(1000),
+ ..Default::default()
+ });
+
+ assert!(!cap.read);
+ assert!(cap.write);
+ assert_eq!(cap.delete_max_size, None);
+ }
+
+ #[test]
+ fn reject_unknown_capability() {
+ let err = CapabilityOverrides::parse("-not_a_capability").unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::ConfigInvalid);
+ }
+
+ #[test]
+ fn reject_invalid_capability_type() {
+ let err = CapabilityOverrides::parse("read=1").unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::ConfigInvalid);
+ }
+}
diff --git a/core/tests/behavior/README.md b/core/tests/behavior/README.md
index 186c6c030..84bf71454 100644
--- a/core/tests/behavior/README.md
+++ b/core/tests/behavior/README.md
@@ -32,6 +32,16 @@ OPENDAL_FS_ROOT=/tmp/
Notice: If the env variables are not set, all behavior tests will be skipped by default.
+### Override Capability
+
+Behavior tests are selected by the operator's full capability. Test setups can override capability with `OPENDAL_TEST_CAPABILITY_OVERRIDES`:
+
+```shell
+OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,delete_max_size=700
+```
+
+Use `-capability` to disable a boolean capability, `+capability` to enable one, and `capability=value` to set boolean or numeric capability values.
+
## Run
Use `OPENDAL_TEST` to control which service to test:
diff --git a/core/tests/behavior/async_delete.rs b/core/tests/behavior/async_delete.rs
index 90b5bdecf..3d1893a7c 100644
--- a/core/tests/behavior/async_delete.rs
+++ b/core/tests/behavior/async_delete.rs
@@ -17,7 +17,7 @@
use anyhow::Result;
use futures::TryStreamExt;
-use opendal::raw::Access;
+use opendal::layers::CapabilityOverrideLayer;
use opendal::raw::OpDelete;
use crate::*;
@@ -351,7 +351,7 @@ pub async fn test_batch_delete(op: Operator) -> Result<()> {
}
cap.delete_max_size = Some(2);
- op.inner().info().update_full_capability(|_| cap);
+ let op = op.layer(CapabilityOverrideLayer::new(move |_| cap));
let mut files = Vec::new();
for _ in 0..5 {
@@ -385,7 +385,7 @@ pub async fn test_batch_delete_with_version(op: Operator) -> Result<()> {
}
cap.delete_max_size = Some(2);
- op.inner().info().update_full_capability(|_| cap);
+ let op = op.layer(CapabilityOverrideLayer::new(move |_| cap));
let mut files = Vec::new();
for _ in 0..5 {