You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2025/09/09 02:39:12 UTC
(opendal) branch main updated: refactor: Migrate cloudflare_kv from adapter::kv to Access instead (#6340)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 16bc05e75 refactor: Migrate cloudflare_kv from adapter::kv to Access instead (#6340)
16bc05e75 is described below
commit 16bc05e75c3cbcbcbdfa86add6b48a3c7e074e6a
Author: Kingsword <ki...@gmail.com>
AuthorDate: Tue Sep 9 10:39:07 2025 +0800
refactor: Migrate cloudflare_kv from adapter::kv to Access instead (#6340)
* refactor: Migrate cloudflare_kv from adapter::typed_kv to Access instead
* fix: test
* refactor: the read logic to use the single-key instead of batch_get
* chore: format
* fix(cloudflare_kv): handle path normalization and metadata properly
* refactor: create_dir
* fix: stat
* fix: list page
* fix: get sublist only
* fix: add recursive
* fix: lister
* fix: reader
* chore: cleanup
* chore: cleanup
* fix: clippy warn
* chore: cleanup
---
.env.example | 5 +
core/src/services/cloudflare_kv/backend.rs | 561 +++++++++++++++++++----------
core/src/services/cloudflare_kv/config.rs | 9 +-
core/src/services/cloudflare_kv/core.rs | 168 +++++++++
core/src/services/cloudflare_kv/delete.rs | 119 ++++++
core/src/services/cloudflare_kv/docs.md | 4 +-
core/src/services/cloudflare_kv/error.rs | 4 +-
core/src/services/cloudflare_kv/lister.rs | 170 +++++++++
core/src/services/cloudflare_kv/mod.rs | 10 +
core/src/services/cloudflare_kv/model.rs | 83 +++++
core/src/services/cloudflare_kv/writer.rs | 68 ++++
core/src/types/operator/builder.rs | 2 +
core/src/types/scheme.rs | 2 +
13 files changed, 998 insertions(+), 207 deletions(-)
diff --git a/.env.example b/.env.example
index bafde86de..d6155521d 100644
--- a/.env.example
+++ b/.env.example
@@ -207,3 +207,8 @@ OPENDAL_ALIYUN_DRIVE_ROOT=/path/to/dir
OPENDAL_ALIYUN_DRIVE_REFRESH_TOKEN=<refresh_token>
OPENDAL_ALIYUN_DRIVE_CLIENT_ID=<client_id>
OPENDAL_ALIYUN_DRIVE_CLIENT_SECRET=<client_secret>
+# cloudflare
+OPENDAL_CLOUDFLARE_KV_ROOT=/path/to/dir
+OPENDAL_CLOUDFLARE_KV_API_TOKEN=<api_token>
+OPENDAL_CLOUDFLARE_KV_ACCOUNT_ID=<account_id>
+OPENDAL_CLOUDFLARE_KV_NAMESPACE_ID=<namespace_id>
diff --git a/core/src/services/cloudflare_kv/backend.rs b/core/src/services/cloudflare_kv/backend.rs
index d2cbda4d0..7752f1e06 100644
--- a/core/src/services/cloudflare_kv/backend.rs
+++ b/core/src/services/cloudflare_kv/backend.rs
@@ -17,16 +17,19 @@
use std::fmt::Debug;
use std::fmt::Formatter;
+use std::sync::Arc;
+use std::time::Duration;
use bytes::Buf;
-use http::header;
-use http::Request;
use http::StatusCode;
-use serde::Deserialize;
-use super::error::parse_error;
-use crate::raw::adapters::kv;
use crate::raw::*;
+use crate::services::cloudflare_kv::core::CloudflareKvCore;
+use crate::services::cloudflare_kv::delete::CloudflareKvDeleter;
+use crate::services::cloudflare_kv::error::parse_error;
+use crate::services::cloudflare_kv::lister::CloudflareKvLister;
+use crate::services::cloudflare_kv::model::*;
+use crate::services::cloudflare_kv::writer::CloudflareWriter;
use crate::services::CloudflareKvConfig;
use crate::ErrorKind;
use crate::*;
@@ -60,9 +63,9 @@ impl Debug for CloudflareKvBuilder {
impl CloudflareKvBuilder {
/// Set the token used to authenticate with CloudFlare.
- pub fn token(mut self, token: &str) -> Self {
- if !token.is_empty() {
- self.config.token = Some(token.to_string())
+ pub fn api_token(mut self, api_token: &str) -> Self {
+ if !api_token.is_empty() {
+ self.config.api_token = Some(api_token.to_string())
}
self
}
@@ -83,6 +86,14 @@ impl CloudflareKvBuilder {
self
}
+ /// Set the default ttl for cloudflare_kv services.
+ ///
+ /// If set, we will specify `EX` for write operations.
+ pub fn default_ttl(mut self, ttl: Duration) -> Self {
+ self.config.default_ttl = Some(ttl);
+ self
+ }
+
/// Set the root within this backend.
pub fn root(mut self, root: &str) -> Self {
self.config.root = if root.is_empty() {
@@ -99,9 +110,14 @@ impl Builder for CloudflareKvBuilder {
type Config = CloudflareKvConfig;
fn build(self) -> Result<impl Access> {
- let authorization = match &self.config.token {
- Some(token) => format_authorization_by_bearer(token)?,
- None => return Err(Error::new(ErrorKind::ConfigInvalid, "token is required")),
+ let api_token = match &self.config.api_token {
+ Some(api_token) => format_authorization_by_bearer(api_token)?,
+ None => {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "api_token is required",
+ ))
+ }
};
let Some(account_id) = self.config.account_id.clone() else {
@@ -118,14 +134,15 @@ impl Builder for CloudflareKvBuilder {
));
};
- let client = if let Some(client) = self.http_client {
- client
- } else {
- HttpClient::new().map_err(|err| {
- err.with_operation("Builder::build")
- .with_context("service", Scheme::CloudflareKv)
- })?
- };
+ // Validate default TTL is at least 60 seconds if specified
+ if let Some(ttl) = self.config.default_ttl {
+ if ttl < Duration::from_secs(60) {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "Default TTL must be at least 60 seconds",
+ ));
+ }
+ }
let root = normalize_root(
self.config
@@ -135,217 +152,361 @@ impl Builder for CloudflareKvBuilder {
.as_str(),
);
- let url_prefix = format!(
- r"https://api.cloudflare.com/client/v4/accounts/{account_id}/storage/kv/namespaces/{namespace_id}"
- );
-
- Ok(CloudflareKvBackend::new(Adapter {
- authorization,
- account_id,
- namespace_id,
- client,
- url_prefix,
+ Ok(CloudflareKvAccessor {
+ core: Arc::new(CloudflareKvCore {
+ api_token,
+ account_id,
+ namespace_id,
+ expiration_ttl: self.config.default_ttl,
+ info: {
+ let am = AccessorInfo::default();
+ am.set_scheme(Scheme::CloudflareKv)
+ .set_root(&root)
+ .set_native_capability(Capability {
+ create_dir: true,
+
+ stat: true,
+ stat_with_if_match: true,
+ stat_with_if_none_match: true,
+ stat_with_if_modified_since: true,
+ stat_with_if_unmodified_since: true,
+
+ read: true,
+ read_with_if_match: true,
+ read_with_if_none_match: true,
+ read_with_if_modified_since: true,
+ read_with_if_unmodified_since: true,
+
+ write: true,
+ write_can_empty: true,
+ write_total_max_size: Some(25 * 1024 * 1024),
+
+ list: true,
+ list_with_limit: true,
+ list_with_recursive: true,
+
+ delete: true,
+ delete_max_size: Some(10000),
+
+ shared: false,
+
+ ..Default::default()
+ });
+
+ // allow deprecated api here for compatibility
+ #[allow(deprecated)]
+ if let Some(client) = self.http_client {
+ am.update_http_client(|_| client);
+ }
+
+ am.into()
+ },
+ }),
})
- .with_normalized_root(root))
}
}
-pub type CloudflareKvBackend = kv::Backend<Adapter>;
-
-#[derive(Clone)]
-pub struct Adapter {
- authorization: String,
- account_id: String,
- namespace_id: String,
- client: HttpClient,
- url_prefix: String,
+#[derive(Debug, Clone)]
+pub struct CloudflareKvAccessor {
+ core: std::sync::Arc<CloudflareKvCore>,
}
-impl Debug for Adapter {
- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("Adapter")
- .field("account_id", &self.account_id)
- .field("namespace_id", &self.namespace_id)
- .finish()
- }
-}
+impl Access for CloudflareKvAccessor {
+ type Reader = Buffer;
+ type Writer = oio::OneShotWriter<CloudflareWriter>;
+ type Lister = oio::PageLister<CloudflareKvLister>;
+ type Deleter = oio::BatchDeleter<CloudflareKvDeleter>;
-impl Adapter {
- fn sign<T>(&self, mut req: Request<T>) -> Result<Request<T>> {
- req.headers_mut()
- .insert(header::AUTHORIZATION, self.authorization.parse().unwrap());
- Ok(req)
+ fn info(&self) -> std::sync::Arc<AccessorInfo> {
+ self.core.info.clone()
}
-}
-impl kv::Adapter for Adapter {
- type Scanner = kv::Scanner;
-
- fn info(&self) -> kv::Info {
- kv::Info::new(
- Scheme::CloudflareKv,
- &self.namespace_id,
- Capability {
- read: true,
- write: true,
- list: true,
- shared: true,
-
- ..Default::default()
- },
- )
- }
+ async fn create_dir(&self, path: &str, _args: OpCreateDir) -> Result<RpCreateDir> {
+ let path = build_abs_path(&self.core.info.root(), path);
- async fn get(&self, path: &str) -> Result<Option<Buffer>> {
- let url = format!("{}/values/{}", self.url_prefix, path);
- let mut req = Request::get(&url);
- req = req.header(header::CONTENT_TYPE, "application/json");
- let mut req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- req = self.sign(req)?;
- let resp = self.client.send(req).await?;
- let status = resp.status();
- match status {
- StatusCode::OK => Ok(Some(resp.into_body())),
- _ => Err(parse_error(resp)),
+ if path == build_abs_path(&self.core.info.root(), "") {
+ return Ok(RpCreateDir::default());
}
- }
- async fn set(&self, path: &str, value: Buffer) -> Result<()> {
- let url = format!("{}/values/{}", self.url_prefix, path);
- let req = Request::put(&url);
- let multipart = Multipart::new();
- let multipart = multipart
- .part(FormDataPart::new("metadata").content(serde_json::Value::Null.to_string()))
- .part(FormDataPart::new("value").content(value.to_vec()));
- let mut req = multipart.apply(req)?;
- req = self.sign(req)?;
- let resp = self.client.send(req).await?;
- let status = resp.status();
- match status {
- StatusCode::OK => Ok(()),
- _ => Err(parse_error(resp)),
+ // Split path into segments and create directories for each level
+ let segments: Vec<&str> = path
+ .trim_start_matches('/')
+ .trim_end_matches('/')
+ .split('/')
+ .collect();
+
+ // Create each directory level
+ let mut current_path = String::from("/");
+ for segment in segments {
+ // Build the current directory path
+ if !current_path.ends_with('/') {
+ current_path.push('/');
+ }
+ current_path.push_str(segment);
+ current_path.push('/');
+
+ // Create metadata for current directory
+ let cf_kv_metadata = CfKvMetadata {
+ etag: build_tmp_path_of(¤t_path),
+ last_modified: chrono::Local::now().to_rfc3339(),
+ content_length: 0,
+ is_dir: true,
+ };
+
+ // Set the directory entry
+ self.core
+ .set(¤t_path, Buffer::new(), cf_kv_metadata)
+ .await?;
}
+
+ Ok(RpCreateDir::default())
}
- async fn delete(&self, path: &str) -> Result<()> {
- let url = format!("{}/values/{}", self.url_prefix, path);
- let mut req = Request::delete(&url);
- req = req.header(header::CONTENT_TYPE, "application/json");
- let mut req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- req = self.sign(req)?;
- let resp = self.client.send(req).await?;
- let status = resp.status();
- match status {
- StatusCode::OK => Ok(()),
- _ => Err(parse_error(resp)),
+ async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
+ let path = build_abs_path(&self.core.info.root(), path);
+ let new_path = path.trim_end_matches('/');
+
+ let resp = self.core.metadata(new_path).await?;
+
+ // Handle non-OK response
+ if resp.status() != StatusCode::OK {
+ // Special handling for potential directory paths
+ if path.ends_with('/') && resp.status() == StatusCode::NOT_FOUND {
+ // Try listing the path to check if it's a directory
+ let list_resp = self.core.list(&path, None, None).await?;
+
+ if list_resp.status() == StatusCode::OK {
+ let list_body = list_resp.into_body();
+ let list_result: CfKvListResponse = serde_json::from_reader(list_body.reader())
+ .map_err(new_json_deserialize_error)?;
+
+ // If listing returns results, treat as directory
+ if let Some(entries) = list_result.result {
+ if !entries.is_empty() {
+ return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
+ }
+ }
+
+ // Empty or no results means not found
+ return Err(Error::new(
+ ErrorKind::NotFound,
+ "key not found in CloudFlare KV",
+ ));
+ }
+ }
+
+ // For all other error cases, parse the error response
+ return Err(parse_error(resp));
}
- }
- async fn scan(&self, path: &str) -> Result<Self::Scanner> {
- let mut url = format!("{}/keys", self.url_prefix);
- if !path.is_empty() {
- url = format!("{url}?prefix={path}");
+ let resp_body = resp.into_body();
+ let cf_response: CfKvStatResponse =
+ serde_json::from_reader(resp_body.reader()).map_err(new_json_deserialize_error)?;
+
+ if !cf_response.success {
+ return Err(Error::new(
+ ErrorKind::Unexpected,
+ "cloudflare_kv stat this key failed for reason we don't know",
+ ));
}
- let mut req = Request::get(&url);
- req = req.header(header::CONTENT_TYPE, "application/json");
- let mut req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- req = self.sign(req)?;
- let resp = self.client.send(req).await?;
- let status = resp.status();
- match status {
- StatusCode::OK => {
- let body = resp.into_body();
- let response: CfKvScanResponse =
- serde_json::from_reader(body.reader()).map_err(|e| {
- Error::new(
- ErrorKind::Unexpected,
- format!("failed to parse error response: {e}"),
- )
- })?;
- Ok(Box::new(kv::ScanStdIter::new(
- response.result.into_iter().map(|r| Ok(r.name)),
- )))
+
+ let metadata = match cf_response.result {
+ Some(metadata) => {
+ if path.ends_with('/') && !metadata.is_dir {
+ return Err(Error::new(
+ ErrorKind::NotFound,
+ "key not found in CloudFlare KV",
+ ));
+ } else {
+ metadata
+ }
+ }
+ None => {
+ return Err(Error::new(
+ ErrorKind::NotFound,
+ "key not found in CloudFlare KV",
+ ));
+ }
+ };
+
+ // Check if_match condition
+ if let Some(if_match) = &args.if_match() {
+ if if_match != &metadata.etag {
+ return Err(Error::new(ErrorKind::ConditionNotMatch, "etag mismatch"));
+ }
+ }
+
+ // Check if_none_match condition
+ if let Some(if_none_match) = &args.if_none_match() {
+ if if_none_match == &metadata.etag {
+ return Err(Error::new(
+ ErrorKind::ConditionNotMatch,
+ "etag match when expected none match",
+ ));
+ }
+ }
+
+ // Parse since time once for both time-based conditions
+ let last_modified = chrono::DateTime::parse_from_rfc3339(&metadata.last_modified)
+ .map_err(|_| Error::new(ErrorKind::Unsupported, "invalid since format"))?;
+
+ // Check modified_since condition
+ if let Some(modified_since) = &args.if_modified_since() {
+ if !last_modified.gt(modified_since) {
+ return Err(Error::new(
+ ErrorKind::ConditionNotMatch,
+ "not modified since specified time",
+ ));
}
- _ => Err(parse_error(resp)),
}
+
+ // Check unmodified_since condition
+ if let Some(unmodified_since) = &args.if_unmodified_since() {
+ if !last_modified.le(unmodified_since) {
+ return Err(Error::new(
+ ErrorKind::ConditionNotMatch,
+ "modified since specified time",
+ ));
+ }
+ }
+
+ let meta = Metadata::new(if metadata.is_dir {
+ EntryMode::DIR
+ } else {
+ EntryMode::FILE
+ })
+ .with_etag(metadata.etag)
+ .with_content_length(metadata.content_length as u64)
+ .with_last_modified(parse_datetime_from_rfc3339(&metadata.last_modified)?);
+
+ Ok(RpStat::new(meta))
}
-}
-#[derive(Debug, Deserialize)]
-pub(super) struct CfKvResponse {
- pub(super) errors: Vec<CfKvError>,
-}
+ async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+ let path = build_abs_path(&self.core.info.root(), path);
+ let resp = self.core.get(&path).await?;
-#[derive(Debug, Deserialize)]
-pub(super) struct CfKvScanResponse {
- result: Vec<CfKvScanResult>,
- // According to https://developers.cloudflare.com/api/operations/workers-kv-namespace-list-a-namespace'-s-keys, result_info is used to determine if there are more keys to be listed
- // result_info: Option<CfKvResultInfo>,
-}
+ let status = resp.status();
-#[derive(Debug, Deserialize)]
-struct CfKvScanResult {
- name: String,
-}
+ if status != StatusCode::OK {
+ return Err(parse_error(resp));
+ }
-// #[derive(Debug, Deserialize)]
-// struct CfKvResultInfo {
-// count: i64,
-// cursor: String,
-// }
+ let resp_body = resp.into_body();
-#[derive(Debug, Deserialize)]
-pub(super) struct CfKvError {
- pub(super) code: i32,
-}
+ if args.if_match().is_some()
+ || args.if_none_match().is_some()
+ || args.if_modified_since().is_some()
+ || args.if_unmodified_since().is_some()
+ {
+ let meta_resp = self.core.metadata(&path).await?;
+
+ if meta_resp.status() != StatusCode::OK {
+ return Err(parse_error(meta_resp));
+ }
+
+ let cf_response: CfKvStatResponse =
+ serde_json::from_reader(meta_resp.into_body().reader())
+ .map_err(new_json_deserialize_error)?;
+
+ if !cf_response.success && cf_response.result.is_some() {
+ return Err(Error::new(
+ ErrorKind::Unexpected,
+ "cloudflare_kv read this key failed for reason we don't know",
+ ));
+ }
-#[cfg(test)]
-mod test {
- use super::*;
-
- #[test]
- fn test_deserialize_scan_json_response() {
- let json_str = r#"{
- "errors": [],
- "messages": [],
- "result": [
- {
- "expiration": 1577836800,
- "metadata": {
- "someMetadataKey": "someMetadataValue"
- },
- "name": "My-Key"
- }
- ],
- "success": true,
- "result_info": {
- "count": 1,
- "cursor": "6Ck1la0VxJ0djhidm1MdX2FyDGxLKVeeHZZmORS_8XeSuhz9SjIJRaSa2lnsF01tQOHrfTGAP3R5X1Kv5iVUuMbNKhWNAXHOl6ePB0TUL8nw"
- }
- }"#;
-
- let response: CfKvScanResponse = serde_json::from_slice(json_str.as_bytes()).unwrap();
-
- assert_eq!(response.result.len(), 1);
- assert_eq!(response.result[0].name, "My-Key");
- // assert!(response.result_info.is_some());
- // if let Some(result_info) = response.result_info {
- // assert_eq!(result_info.count, 1);
- // assert_eq!(result_info.cursor, "6Ck1la0VxJ0djhidm1MdX2FyDGxLKVeeHZZmORS_8XeSuhz9SjIJRaSa2lnsF01tQOHrfTGAP3R5X1Kv5iVUuMbNKhWNAXHOl6ePB0TUL8nw");
- // }
+ let metadata = cf_response.result.unwrap();
+
+ // Check if_match condition
+ if let Some(if_match) = &args.if_match() {
+ if if_match != &metadata.etag {
+ return Err(Error::new(ErrorKind::ConditionNotMatch, "etag mismatch"));
+ }
+ }
+
+ // Check if_none_match condition
+ if let Some(if_none_match) = &args.if_none_match() {
+ if if_none_match == &metadata.etag {
+ return Err(Error::new(
+ ErrorKind::ConditionNotMatch,
+ "etag match when expected none match",
+ ));
+ }
+ }
+
+ // Parse since time once for both time-based conditions
+ let last_modified = chrono::DateTime::parse_from_rfc3339(&metadata.last_modified)
+ .map_err(|_| Error::new(ErrorKind::Unsupported, "invalid since format"))?;
+
+ // Check modified_since condition
+ if let Some(modified_since) = &args.if_modified_since() {
+ if !last_modified.gt(modified_since) {
+ return Err(Error::new(
+ ErrorKind::ConditionNotMatch,
+ "not modified since specified time",
+ ));
+ }
+ }
+
+ // Check unmodified_since condition
+ if let Some(unmodified_since) = &args.if_unmodified_since() {
+ if !last_modified.le(unmodified_since) {
+ return Err(Error::new(
+ ErrorKind::ConditionNotMatch,
+ "modified since specified time",
+ ));
+ }
+ }
+ }
+
+ let range = args.range();
+ let buffer = if range.is_full() {
+ resp_body
+ } else {
+ let start = range.offset() as usize;
+ let end = match range.size() {
+ Some(size) => (range.offset() + size) as usize,
+ None => resp_body.len(),
+ };
+ resp_body.slice(start..end.min(resp_body.len()))
+ };
+ Ok((RpRead::new(), buffer))
+ }
+
+ async fn write(&self, path: &str, _: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+ let path = build_abs_path(&self.core.info.root(), path);
+ let writer = CloudflareWriter::new(self.core.clone(), path);
+
+ let w = oio::OneShotWriter::new(writer);
+
+ Ok((RpWrite::default(), w))
+ }
+
+ async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+ Ok((
+ RpDelete::default(),
+ oio::BatchDeleter::new(CloudflareKvDeleter::new(self.core.clone())),
+ ))
}
- #[test]
- fn test_deserialize_json_response() {
- let json_str = r#"{
- "errors": [],
- "messages": [],
- "result": {},
- "success": true
- }"#;
+ async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+ let path = build_abs_path(&self.core.info.root(), path);
+
+ let limit = match args.limit() {
+ Some(limit) => {
+ // The list limit of cloudflare_kv is limited to 10..1000.
+ if !(10..=1000).contains(&limit) {
+ 1000
+ } else {
+ limit
+ }
+ }
+ None => 1000,
+ };
- let response: CfKvResponse = serde_json::from_slice(json_str.as_bytes()).unwrap();
+ let l = CloudflareKvLister::new(self.core.clone(), &path, args.recursive(), Some(limit));
- assert_eq!(response.errors.len(), 0);
+ Ok((RpList::default(), oio::PageLister::new(l)))
}
}
diff --git a/core/src/services/cloudflare_kv/config.rs b/core/src/services/cloudflare_kv/config.rs
index c3781f641..576e84bbf 100644
--- a/core/src/services/cloudflare_kv/config.rs
+++ b/core/src/services/cloudflare_kv/config.rs
@@ -17,6 +17,7 @@
use std::fmt::Debug;
use std::fmt::Formatter;
+use std::time::Duration;
use serde::Deserialize;
use serde::Serialize;
@@ -25,11 +26,13 @@ use serde::Serialize;
#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct CloudflareKvConfig {
/// The token used to authenticate with CloudFlare.
- pub token: Option<String>,
+ pub api_token: Option<String>,
/// The account ID used to authenticate with CloudFlare. Used as URI path parameter.
pub account_id: Option<String>,
/// The namespace ID. Used as URI path parameter.
pub namespace_id: Option<String>,
+ /// The default ttl for write operations.
+ pub default_ttl: Option<Duration>,
/// Root within this backend.
pub root: Option<String>,
@@ -43,8 +46,8 @@ impl Debug for CloudflareKvConfig {
ds.field("account_id", &self.account_id);
ds.field("namespace_id", &self.namespace_id);
- if self.token.is_some() {
- ds.field("token", &"<redacted>");
+ if self.api_token.is_some() {
+ ds.field("api_token", &"<redacted>");
}
ds.finish()
diff --git a/core/src/services/cloudflare_kv/core.rs b/core/src/services/cloudflare_kv/core.rs
new file mode 100644
index 000000000..545f05b28
--- /dev/null
+++ b/core/src/services/cloudflare_kv/core.rs
@@ -0,0 +1,168 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use http::header;
+use http::request;
+use http::Request;
+use http::Response;
+use serde_json::json;
+
+use crate::raw::new_json_serialize_error;
+use crate::raw::percent_encode_path;
+use crate::raw::Operation;
+use crate::raw::QueryPairsWriter;
+use crate::raw::{new_request_build_error, AccessorInfo, FormDataPart, Multipart};
+use crate::services::cloudflare_kv::model::CfKvMetadata;
+use crate::{Buffer, Result};
+
+#[derive(Debug, Clone)]
+pub struct CloudflareKvCore {
+ pub api_token: String,
+ pub account_id: String,
+ pub namespace_id: String,
+ pub expiration_ttl: Option<Duration>,
+ pub info: Arc<AccessorInfo>,
+}
+
+impl CloudflareKvCore {
+ #[inline]
+ async fn send(&self, req: Request<Buffer>) -> Result<Response<Buffer>> {
+ self.info.http_client().send(req).await
+ }
+
+ fn sign(&self, req: request::Builder) -> request::Builder {
+ req.header(header::AUTHORIZATION, &self.api_token)
+ }
+
+ fn url_prefix(&self) -> String {
+ let url = format!(
+ "https://api.cloudflare.com/client/v4/accounts/{}/storage/kv/namespaces/{}",
+ self.account_id, self.namespace_id
+ );
+ url
+ }
+}
+
+impl CloudflareKvCore {
+ pub async fn metadata(&self, path: &str) -> Result<Response<Buffer>> {
+ let url = format!(
+ "{}/metadata/{}",
+ self.url_prefix(),
+ percent_encode_path(path)
+ );
+
+ let req = Request::get(url);
+ let req = self.sign(req);
+
+ let req = req
+ .extension(Operation::Stat)
+ .body(Buffer::new())
+ .map_err(new_request_build_error)?;
+
+ self.send(req).await
+ }
+
+ pub async fn get(&self, path: &str) -> Result<Response<Buffer>> {
+ let url = format!("{}/values/{}", self.url_prefix(), percent_encode_path(path));
+ let req = Request::get(url);
+
+ let req = self.sign(req);
+
+ let req = req
+ .extension(Operation::Read)
+ .body(Buffer::new())
+ .map_err(new_request_build_error)?;
+
+ self.send(req).await
+ }
+
+ pub async fn set(
+ &self,
+ path: &str,
+ value: Buffer,
+ metadata: CfKvMetadata,
+ ) -> Result<Response<Buffer>> {
+ let url = format!("{}/values/{}", self.url_prefix(), percent_encode_path(path));
+
+ let req = Request::put(url);
+ let req = self.sign(req);
+ let req = req.extension(Operation::Write);
+
+ let mut multipart = Multipart::new()
+ .part(FormDataPart::new("value").content(value))
+ .part(
+ FormDataPart::new("metadata")
+ .content(serde_json::to_string(&metadata).map_err(new_json_serialize_error)?),
+ );
+
+ if let Some(expiration_ttl) = self.expiration_ttl {
+ multipart = multipart.part(
+ FormDataPart::new("expiration_ttl").content(expiration_ttl.as_secs().to_string()),
+ );
+ }
+
+ let req = multipart.apply(req)?;
+
+ self.send(req).await
+ }
+
+ pub async fn delete(&self, paths: &[String]) -> Result<Response<Buffer>> {
+ let url = format!("{}/bulk/delete", self.url_prefix());
+
+ let req = Request::post(&url);
+
+ let req = self.sign(req);
+ let req_body = &json!(paths);
+ let req = req
+ .extension(Operation::Delete)
+ .header(header::CONTENT_TYPE, "application/json")
+ .body(Buffer::from(req_body.to_string()))
+ .map_err(new_request_build_error)?;
+
+ self.send(req).await
+ }
+
+ pub async fn list(
+ &self,
+ prefix: &str,
+ limit: Option<usize>,
+ cursor: Option<String>,
+ ) -> Result<Response<Buffer>> {
+ let url = format!("{}/keys", self.url_prefix());
+ let mut url = QueryPairsWriter::new(&url);
+ if let Some(cursor) = cursor {
+ if !cursor.is_empty() {
+ url = url.push("cursor", &cursor);
+ }
+ }
+ url = url.push("limit", &limit.unwrap_or(1000).to_string());
+ url = url.push("prefix", &percent_encode_path(prefix));
+
+ let req = Request::get(url.finish());
+
+ let req = self.sign(req);
+ let req = req
+ .extension(Operation::List)
+ .body(Buffer::new())
+ .map_err(new_request_build_error)?;
+
+ self.send(req).await
+ }
+}
diff --git a/core/src/services/cloudflare_kv/delete.rs b/core/src/services/cloudflare_kv/delete.rs
new file mode 100644
index 000000000..5f71d6400
--- /dev/null
+++ b/core/src/services/cloudflare_kv/delete.rs
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::oio::BatchDeleteResult;
+use crate::raw::*;
+use crate::services::cloudflare_kv::model::CfKvDeleteResponse;
+use crate::*;
+
+pub struct CloudflareKvDeleter {
+ core: Arc<CloudflareKvCore>,
+}
+
+impl CloudflareKvDeleter {
+ pub fn new(core: Arc<CloudflareKvCore>) -> Self {
+ Self { core }
+ }
+}
+
+impl oio::BatchDelete for CloudflareKvDeleter {
+ async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+ let path = build_abs_path(&self.core.info.root(), &path);
+ let resp = self
+ .core
+ .delete(&[path.trim_end_matches('/').to_string()])
+ .await?;
+
+ let status = resp.status();
+
+ if status != StatusCode::OK {
+ return Err(parse_error(resp.clone()));
+ }
+
+ let bs = resp.clone().into_body();
+ let res: CfKvDeleteResponse =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ if !res.success {
+ return Err(parse_error(resp.clone()));
+ }
+
+ Ok(())
+ }
+
+ async fn delete_batch(&self, batch: Vec<(String, OpDelete)>) -> Result<BatchDeleteResult> {
+ let keys = batch
+ .iter()
+ .map(|path| {
+ let path = build_abs_path(&self.core.info.root(), &path.0);
+ path.trim_end_matches('/').to_string()
+ })
+ .collect::<Vec<String>>();
+
+ let resp = self.core.delete(&keys).await?;
+
+ let status = resp.status();
+
+ if status != StatusCode::OK {
+ return Err(parse_error(resp));
+ }
+
+ let bs = resp.into_body();
+
+ let res: CfKvDeleteResponse =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ let result = match (res.success, res.result) {
+ (true, Some(result)) => result,
+ _ => {
+ return Err(Error::new(
+ ErrorKind::Unexpected,
+ "cloudflare_kv delete this key failed for reason we don't know",
+ ))
+ }
+ };
+
+ let mut batched_result = BatchDeleteResult {
+ succeeded: Vec::with_capacity(result.successful_key_count),
+ failed: Vec::with_capacity(result.unsuccessful_keys.len()),
+ };
+
+ for item in batch {
+ if result.unsuccessful_keys.contains(&item.0) {
+ batched_result.failed.push((
+ item.0,
+ item.1,
+ Error::new(
+ ErrorKind::Unexpected,
+ "cloudflare_kv delete this key failed for reason we don't know",
+ ),
+ ));
+ } else {
+ batched_result.succeeded.push(item);
+ }
+ }
+
+ Ok(batched_result)
+ }
+}
diff --git a/core/src/services/cloudflare_kv/docs.md b/core/src/services/cloudflare_kv/docs.md
index ff7f254ee..1920849dd 100644
--- a/core/src/services/cloudflare_kv/docs.md
+++ b/core/src/services/cloudflare_kv/docs.md
@@ -9,13 +9,13 @@ This service can be used to:
- [x] delete
- [ ] copy
- [ ] rename
-- [ ] ~~list~~
+- [x] list
- [ ] ~~presign~~
- [ ] blocking
## Configuration
- `root`: Set the working directory of `OpenDAL`
-- `token`: Set the token of cloudflare api
+- `api_token`: Set the token of cloudflare api
- `account_id`: Set the account identifier of cloudflare
- `namespace_id`: Set the namespace identifier of d1
diff --git a/core/src/services/cloudflare_kv/error.rs b/core/src/services/cloudflare_kv/error.rs
index b04eb3e57..a86f3bf0f 100644
--- a/core/src/services/cloudflare_kv/error.rs
+++ b/core/src/services/cloudflare_kv/error.rs
@@ -20,8 +20,8 @@ use http::Response;
use http::StatusCode;
use serde_json::de;
-use super::backend::CfKvError;
-use super::backend::CfKvResponse;
+use super::model::CfKvError;
+use super::model::CfKvResponse;
use crate::raw::*;
use crate::*;
diff --git a/core/src/services/cloudflare_kv/lister.rs b/core/src/services/cloudflare_kv/lister.rs
new file mode 100644
index 000000000..bc09b7cda
--- /dev/null
+++ b/core/src/services/cloudflare_kv/lister.rs
@@ -0,0 +1,170 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+
+use super::core::CloudflareKvCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::services::cloudflare_kv::model::{CfKvListKey, CfKvListResponse};
+use crate::*;
+
+pub struct CloudflareKvLister {
+ core: Arc<CloudflareKvCore>,
+
+ path: String,
+ limit: Option<usize>,
+ recursive: bool,
+}
+
+impl CloudflareKvLister {
+ pub fn new(
+ core: Arc<CloudflareKvCore>,
+ path: &str,
+ recursive: bool,
+ limit: Option<usize>,
+ ) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ limit,
+ recursive,
+ }
+ }
+
+ fn build_entry_for_item(&self, item: &CfKvListKey, root: &str) -> Result<oio::Entry> {
+ let metadata = item.metadata.clone();
+ let mut name = item.name.clone();
+
+ if metadata.is_dir && !name.ends_with('/') {
+ name += "/";
+ }
+
+ let mut name = name.replace(root.trim_start_matches('/'), "");
+
+ // If it is the root directory, it needs to be processed as /
+ if name.is_empty() {
+ name = "/".to_string();
+ }
+
+ let entry_metadata = if name.ends_with('/') {
+ Metadata::new(EntryMode::DIR)
+ .with_etag(build_tmp_path_of(&name))
+ .with_content_length(0)
+ } else {
+ Metadata::new(EntryMode::FILE)
+ .with_etag(metadata.etag)
+ .with_content_length(metadata.content_length as u64)
+ .with_last_modified(parse_datetime_from_rfc3339(&metadata.last_modified)?)
+ };
+
+ Ok(oio::Entry::new(&name, entry_metadata))
+ }
+
+ fn handle_non_recursive_file_list(
+ &self,
+ ctx: &mut oio::PageContext,
+ result: &[CfKvListKey],
+ root: &str,
+ ) -> Result<()> {
+ if let Some(item) = result.iter().find(|item| item.name == self.path) {
+ let entry = self.build_entry_for_item(item, root)?;
+ ctx.entries.push_back(entry);
+ } else if !result.is_empty() {
+ let path_name = self.path.replace(root.trim_start_matches('/'), "");
+ let entry = oio::Entry::new(
+ &format!("{path_name}/"),
+ Metadata::new(EntryMode::DIR)
+ .with_etag(build_tmp_path_of(&path_name))
+ .with_content_length(0),
+ );
+ ctx.entries.push_back(entry);
+ }
+ ctx.done = true;
+ Ok(())
+ }
+}
+
+impl oio::PageList for CloudflareKvLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let new_path = self.path.trim_end_matches('/');
+ let resp = self
+ .core
+ .list(new_path, self.limit, Some(ctx.token.clone()))
+ .await?;
+
+ if resp.status() != http::StatusCode::OK {
+ return Err(parse_error(resp));
+ }
+
+ let bs = resp.into_body();
+ let res: CfKvListResponse =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ if !res.success {
+ return Err(Error::new(
+ ErrorKind::Unexpected,
+ "oss list this key failed for reason we don't know",
+ ));
+ }
+
+ let (token, done) = res
+ .result_info
+ .and_then(|info| info.cursor)
+ .map_or((String::new(), true), |cursor| {
+ (cursor.clone(), cursor.is_empty())
+ });
+
+ ctx.token = token;
+ ctx.done = done;
+
+ if let Some(result) = res.result {
+ let root = self.core.info.root().to_string();
+
+ if !self.path.ends_with('/') && !self.recursive {
+ self.handle_non_recursive_file_list(ctx, &result, &root)?;
+ return Ok(());
+ }
+
+ for item in result {
+ let mut name = item.name.clone();
+ if item.metadata.is_dir && !name.ends_with('/') {
+ name += "/";
+ }
+
+ // For non-recursive listing, filter out entries not in the current directory.
+ if !self.recursive {
+ if let Some(relative_path) = name.strip_prefix(&self.path) {
+ if relative_path.trim_end_matches('/').contains('/') {
+ continue;
+ }
+ } else if self.path != name {
+ continue;
+ }
+ }
+
+ let entry = self.build_entry_for_item(&item, &root)?;
+ ctx.entries.push_back(entry);
+ }
+ }
+
+ Ok(())
+ }
+}
diff --git a/core/src/services/cloudflare_kv/mod.rs b/core/src/services/cloudflare_kv/mod.rs
index 390bbd291..3b1172ba3 100644
--- a/core/src/services/cloudflare_kv/mod.rs
+++ b/core/src/services/cloudflare_kv/mod.rs
@@ -21,6 +21,16 @@ mod error;
#[cfg(feature = "services-cloudflare-kv")]
mod backend;
#[cfg(feature = "services-cloudflare-kv")]
+mod core;
+#[cfg(feature = "services-cloudflare-kv")]
+mod delete;
+#[cfg(feature = "services-cloudflare-kv")]
+mod lister;
+#[cfg(feature = "services-cloudflare-kv")]
+mod model;
+#[cfg(feature = "services-cloudflare-kv")]
+mod writer;
+#[cfg(feature = "services-cloudflare-kv")]
pub use backend::CloudflareKvBuilder as CloudflareKv;
mod config;
diff --git a/core/src/services/cloudflare_kv/model.rs b/core/src/services/cloudflare_kv/model.rs
new file mode 100644
index 000000000..2ae6c9d9d
--- /dev/null
+++ b/core/src/services/cloudflare_kv/model.rs
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvResponse {
+ pub errors: Vec<CfKvError>,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct CfKvMetadata {
+ pub etag: String,
+ pub last_modified: String,
+ pub content_length: usize,
+ pub is_dir: bool,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvError {
+ pub code: i32,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct CfKvSetData {
+ pub key: String,
+ pub value: String,
+ pub metadata: CfKvMetadata,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvDeleteResult {
+ pub successful_key_count: usize,
+ pub unsuccessful_keys: Vec<String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvDeleteResponse {
+ pub success: bool,
+ pub result: Option<CfKvDeleteResult>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvListKey {
+ pub name: String,
+ pub metadata: CfKvMetadata,
+ // expiration: Option<u32>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvListResultInfo {
+ // pub count: Option<usize>,
+ pub cursor: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvListResponse {
+ pub success: bool,
+ pub result_info: Option<CfKvListResultInfo>,
+ pub result: Option<Vec<CfKvListKey>>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvStatResponse {
+ pub success: bool,
+ pub result: Option<CfKvMetadata>,
+}
diff --git a/core/src/services/cloudflare_kv/writer.rs b/core/src/services/cloudflare_kv/writer.rs
new file mode 100644
index 000000000..b21b7152d
--- /dev/null
+++ b/core/src/services/cloudflare_kv/writer.rs
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::CloudflareKvCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::services::cloudflare_kv::model::CfKvMetadata;
+use crate::*;
+
+pub struct CloudflareWriter {
+ core: Arc<CloudflareKvCore>,
+ path: String,
+}
+
+impl CloudflareWriter {
+ pub fn new(core: Arc<CloudflareKvCore>, path: String) -> Self {
+ CloudflareWriter { core, path }
+ }
+}
+
+impl oio::OneShotWrite for CloudflareWriter {
+ async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
+ let cf_kv_metadata = CfKvMetadata {
+ etag: build_tmp_path_of(&self.path),
+ last_modified: chrono::Local::now().to_rfc3339(),
+ content_length: bs.len(),
+ is_dir: self.path.ends_with('/'),
+ };
+
+ let resp = self
+ .core
+ .set(&self.path, bs, cf_kv_metadata.clone())
+ .await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let mut metadata = Metadata::default();
+ metadata.set_etag(&cf_kv_metadata.etag);
+ metadata
+ .set_last_modified(parse_datetime_from_rfc3339(&cf_kv_metadata.last_modified)?);
+ metadata.set_content_length(cf_kv_metadata.content_length as u64);
+
+ Ok(metadata)
+ }
+ _ => Err(parse_error(resp)),
+ }
+ }
+}
diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs
index c9c168d64..7cbc273a3 100644
--- a/core/src/types/operator/builder.rs
+++ b/core/src/types/operator/builder.rs
@@ -170,6 +170,8 @@ impl Operator {
Scheme::AliyunDrive => Self::from_iter::<services::AliyunDrive>(iter)?.finish(),
#[cfg(feature = "services-alluxio")]
Scheme::Alluxio => Self::from_iter::<services::Alluxio>(iter)?.finish(),
+ #[cfg(feature = "services-cloudflare-kv")]
+ Scheme::CloudflareKv => Self::from_iter::<services::CloudflareKv>(iter)?.finish(),
#[cfg(feature = "services-compfs")]
Scheme::Compfs => Self::from_iter::<services::Compfs>(iter)?.finish(),
#[cfg(feature = "services-upyun")]
diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs
index 6d9c06b56..a055dde85 100644
--- a/core/src/types/scheme.rs
+++ b/core/src/types/scheme.rs
@@ -303,6 +303,8 @@ impl Scheme {
Scheme::Surrealdb,
#[cfg(feature = "services-lakefs")]
Scheme::Lakefs,
+ #[cfg(feature = "services-cloudflare-kv")]
+ Scheme::CloudflareKv,
])
}
}