You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/05/16 07:03:06 UTC
(opendal) branch main updated: feat(services/azdls): Add user defined metadata support (#7528)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new e190ac642 feat(services/azdls): Add user defined metadata support (#7528)
e190ac642 is described below
commit e190ac64209a624998a9987f1ddf7766286f38ad
Author: Yuang Gao <91...@users.noreply.github.com>
AuthorDate: Sat May 16 00:03:00 2026 -0700
feat(services/azdls): Add user defined metadata support (#7528)
* feat(services/azdls): Add user defined metadata support
* silently skip malformed metadata entries
---
core/Cargo.lock | 5 ++-
core/services/azdls/Cargo.toml | 1 +
core/services/azdls/src/backend.rs | 1 +
core/services/azdls/src/core.rs | 83 ++++++++++++++++++++++++++++++++++++++
4 files changed, 88 insertions(+), 2 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index a065f0312..bf782413c 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -3307,7 +3307,7 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0f3f231ff5b8465063b2bf8af3b975222b4fa3cc8c07b43d01fe8e0ae15c9dd"
dependencies = [
- "bindgen 0.72.1",
+ "bindgen",
"libc",
]
@@ -5224,7 +5224,7 @@ version = "0.17.3+10.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9"
dependencies = [
- "bindgen 0.72.1",
+ "bindgen",
"bzip2-sys",
"cc",
"libc",
@@ -6720,6 +6720,7 @@ dependencies = [
name = "opendal-service-azdls"
version = "0.56.0"
dependencies = [
+ "base64 0.22.1",
"bytes",
"http 1.4.0",
"log",
diff --git a/core/services/azdls/Cargo.toml b/core/services/azdls/Cargo.toml
index 468ad1573..1b0c1fa9b 100644
--- a/core/services/azdls/Cargo.toml
+++ b/core/services/azdls/Cargo.toml
@@ -31,6 +31,7 @@ version = { workspace = true }
all-features = true
[dependencies]
+base64 = { workspace = true }
bytes = { workspace = true }
http = { workspace = true }
log = { workspace = true }
diff --git a/core/services/azdls/src/backend.rs b/core/services/azdls/src/backend.rs
index 8409dcb04..8241f2813 100644
--- a/core/services/azdls/src/backend.rs
+++ b/core/services/azdls/src/backend.rs
@@ -333,6 +333,7 @@ impl Builder for AzdlsBuilder {
write_can_multi: true,
write_with_if_none_match: true,
write_with_if_not_exists: true,
+ write_with_user_metadata: true,
create_dir: true,
diff --git a/core/services/azdls/src/core.rs b/core/services/azdls/src/core.rs
index 4fd393f0f..104970b63 100644
--- a/core/services/azdls/src/core.rs
+++ b/core/services/azdls/src/core.rs
@@ -15,9 +15,12 @@
// specific language governing permissions and limitations
// under the License.
+use std::collections::HashMap;
use std::fmt::Debug;
use std::sync::Arc;
+use base64::Engine;
+use base64::prelude::BASE64_STANDARD;
use http::HeaderName;
use http::HeaderValue;
use http::Request;
@@ -40,6 +43,9 @@ pub const X_MS_VERSION_ID: &str = "x-ms-version-id";
const X_MS_CONTINUATION: &str = "x-ms-continuation";
pub const DIRECTORY: &str = "directory";
pub const FILE: &str = "file";
+// Format: `n1=base64(v1),n2=base64(v2),...`.
+// See https://learn.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create
+const X_MS_PROPERTIES: &str = "x-ms-properties";
pub struct AzdlsCore {
pub info: Arc<AccessorInfo>,
@@ -159,6 +165,12 @@ impl AzdlsCore {
req = req.header(IF_NONE_MATCH, v)
}
+ if let Some(user_metadata) = args.user_metadata() {
+ if !user_metadata.is_empty() {
+ req = req.header(X_MS_PROPERTIES, encode_user_metadata(user_metadata));
+ }
+ }
+
let operation = if resource == DIRECTORY {
Operation::CreateDir
} else {
@@ -318,6 +330,13 @@ impl AzdlsCore {
meta.set_version(version_id);
}
+ if let Some(value) = parse_header_to_str(headers, X_MS_PROPERTIES)? {
+ let user_meta = decode_user_metadata(value);
+ if !user_meta.is_empty() {
+ meta = meta.with_user_metadata(user_meta);
+ }
+ }
+
let resource = resp
.headers()
.get("x-ms-resource-type")
@@ -481,3 +500,67 @@ impl AzdlsCore {
}
}
}
+
+fn encode_user_metadata(metadata: &HashMap<String, String>) -> String {
+ metadata
+ .iter()
+ .map(|(k, v)| format!("{k}={}", BASE64_STANDARD.encode(v)))
+ .collect::<Vec<_>>()
+ .join(",")
+}
+
+fn decode_user_metadata(value: &str) -> HashMap<String, String> {
+ value
+ .split(',')
+ .filter_map(|pair| {
+ let (key, encoded) = pair.split_once('=')?;
+ let bytes = BASE64_STANDARD.decode(encoded.trim()).ok()?;
+ let decoded = String::from_utf8(bytes).ok()?;
+ Some((key.trim().to_string(), decoded))
+ })
+ .collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn encode_then_decode_roundtrips() {
+ let mut input = HashMap::new();
+ input.insert("location".to_string(), "everywhere".to_string());
+ input.insert("owner".to_string(), "opendal".to_string());
+
+ let decoded = decode_user_metadata(&encode_user_metadata(&input));
+ assert_eq!(decoded, input);
+ }
+
+ #[test]
+ fn encoded_value_is_base64_so_special_characters_survive() {
+ let mut input = HashMap::new();
+ input.insert("k".to_string(), "a=b,c".to_string());
+
+ let decoded = decode_user_metadata(&encode_user_metadata(&input));
+ assert_eq!(decoded.get("k"), Some(&"a=b,c".to_string()));
+ }
+
+ #[test]
+ fn decode_preserves_utf8_multibyte_values() {
+ let mut input = HashMap::new();
+ input.insert("greeting".to_string(), "你好".to_string());
+
+ let decoded = decode_user_metadata(&encode_user_metadata(&input));
+ assert_eq!(decoded, input);
+ }
+
+ #[test]
+ fn decode_silently_skips_malformed_entries_and_keeps_valid_ones() {
+ let header = format!(
+ "good=Z29vZA==,no_equals,bad_b64=!!!,k={}",
+ BASE64_STANDARD.encode([0xFF, 0xFE])
+ );
+ let decoded = decode_user_metadata(&header);
+ assert_eq!(decoded.len(), 1);
+ assert_eq!(decoded.get("good"), Some(&"good".to_string()));
+ }
+}