You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by ko...@apache.org on 2025/12/17 04:33:42 UTC
(opendal) branch extract-lakefs-service created (now 3137ba72f)
This is an automated email from the ASF dual-hosted git repository.
koushiro pushed a change to branch extract-lakefs-service
in repository https://gitbox.apache.org/repos/asf/opendal.git
at 3137ba72f refactor: Split lakefs service to new crate
This branch includes the following new commits:
new 3137ba72f refactor: Split lakefs service to new crate
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opendal) 01/01: refactor: Split lakefs service to new crate
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
koushiro pushed a commit to branch extract-lakefs-service
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 3137ba72f5f2d12f98fbf539081aec904929fbfc
Author: koushiro <ko...@gmail.com>
AuthorDate: Wed Dec 17 12:32:58 2025 +0800
refactor: Split lakefs service to new crate
---
core/Cargo.lock | 15 ++++++++
core/Cargo.toml | 3 +-
core/core/Cargo.toml | 1 -
core/core/src/services/mod.rs | 5 ---
core/services/lakefs/Cargo.toml | 26 ++++++++++++++
.../lakefs => services/lakefs/src}/backend.rs | 4 +--
.../lakefs => services/lakefs/src}/config.rs | 40 ++++++++++------------
.../lakefs => services/lakefs/src}/core.rs | 6 ++--
.../lakefs => services/lakefs/src}/deleter.rs | 5 +--
.../lakefs => services/lakefs/src}/docs.md | 4 +--
.../lakefs => services/lakefs/src}/error.rs | 8 ++---
.../lakefs/mod.rs => services/lakefs/src/lib.rs} | 11 +++---
.../lakefs => services/lakefs/src}/lister.rs | 8 +++--
.../lakefs => services/lakefs/src}/writer.rs | 5 +--
core/src/lib.rs | 4 +++
15 files changed, 93 insertions(+), 52 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index e03ff0760..e02df9797 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -5569,6 +5569,7 @@ dependencies = [
"opendal-service-github",
"opendal-service-hdfs-native",
"opendal-service-ipfs",
+ "opendal-service-lakefs",
"opendal-service-memcached",
"opendal-service-mini-moka",
"opendal-service-moka",
@@ -6153,6 +6154,20 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "opendal-service-lakefs"
+version = "0.55.0"
+dependencies = [
+ "bytes",
+ "ctor",
+ "http 1.4.0",
+ "log",
+ "opendal-core",
+ "serde",
+ "serde_json",
+ "tokio",
+]
+
[[package]]
name = "opendal-service-memcached"
version = "0.55.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index e6231e25f..08d23ec6f 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -131,7 +131,7 @@ services-huggingface = ["opendal-core/services-huggingface"]
services-ipfs = ["dep:opendal-service-ipfs"]
services-ipmfs = ["opendal-core/services-ipmfs"]
services-koofr = ["opendal-core/services-koofr"]
-services-lakefs = ["opendal-core/services-lakefs"]
+services-lakefs = ["dep:opendal-service-lakefs"]
services-memcached = ["dep:opendal-service-memcached"]
services-memory = ["opendal-core/services-memory"]
services-mini-moka = ["dep:opendal-service-mini-moka"]
@@ -221,6 +221,7 @@ opendal-service-ghac = { path = "services/ghac", version = "0.55.0", optional =
opendal-service-github = { path = "services/github", version = "0.55.0", optional = true, default-features = false }
opendal-service-hdfs-native = { path = "services/hdfs-native", version = "0.55.0", optional = true, default-features = false }
opendal-service-ipfs = { path = "services/ipfs", version = "0.55.0", optional = true, default-features = false }
+opendal-service-lakefs = { path = "services/lakefs", version = "0.55.0", optional = true, default-features = false }
opendal-service-memcached = { path = "services/memcached", version = "0.55.0", optional = true, default-features = false }
opendal-service-mini-moka = { path = "services/mini_moka", version = "0.55.0", optional = true, default-features = false }
opendal-service-moka = { path = "services/moka", version = "0.55.0", optional = true, default-features = false }
diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml
index edfaefc7d..e7f0af46e 100644
--- a/core/core/Cargo.toml
+++ b/core/core/Cargo.toml
@@ -69,7 +69,6 @@ services-http = []
services-huggingface = []
services-ipmfs = []
services-koofr = []
-services-lakefs = []
services-memory = []
services-mongodb = ["dep:mongodb", "dep:mongodb-internal-macros"]
services-monoiofs = ["dep:monoio", "dep:flume"]
diff --git a/core/core/src/services/mod.rs b/core/core/src/services/mod.rs
index e1553e71d..3292db0a4 100644
--- a/core/core/src/services/mod.rs
+++ b/core/core/src/services/mod.rs
@@ -79,11 +79,6 @@ mod koofr;
#[cfg(feature = "services-koofr")]
pub use koofr::*;
-#[cfg(feature = "services-lakefs")]
-mod lakefs;
-#[cfg(feature = "services-lakefs")]
-pub use lakefs::*;
-
#[cfg(feature = "services-memory")]
mod memory;
#[cfg(feature = "services-memory")]
diff --git a/core/services/lakefs/Cargo.toml b/core/services/lakefs/Cargo.toml
new file mode 100644
index 000000000..d89c3f785
--- /dev/null
+++ b/core/services/lakefs/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+description = "Apache OpenDAL lakeFS service implementation"
+name = "opendal-service-lakefs"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+bytes = { workspace = true }
+ctor = { workspace = true }
+http = { workspace = true }
+log = { workspace = true }
+opendal-core = { path = "../../core", version = "0.55.0", default-features = false }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/core/core/src/services/lakefs/backend.rs b/core/services/lakefs/src/backend.rs
similarity index 99%
rename from core/core/src/services/lakefs/backend.rs
rename to core/services/lakefs/src/backend.rs
index c4ffc6b6e..e789cc33c 100644
--- a/core/core/src/services/lakefs/backend.rs
+++ b/core/services/lakefs/src/backend.rs
@@ -21,6 +21,8 @@ use bytes::Buf;
use http::Response;
use http::StatusCode;
use log::debug;
+use opendal_core::raw::*;
+use opendal_core::*;
use super::LAKEFS_SCHEME;
use super::config::LakefsConfig;
@@ -30,8 +32,6 @@ use super::deleter::LakefsDeleter;
use super::error::parse_error;
use super::lister::LakefsLister;
use super::writer::LakefsWriter;
-use crate::raw::*;
-use crate::*;
/// [Lakefs](https://docs.lakefs.io/reference/api.html#/)'s API support.
#[doc = include_str!("docs.md")]
diff --git a/core/core/src/services/lakefs/config.rs b/core/services/lakefs/src/config.rs
similarity index 83%
rename from core/core/src/services/lakefs/config.rs
rename to core/services/lakefs/src/config.rs
index dff70ec74..04a338808 100644
--- a/core/core/src/services/lakefs/config.rs
+++ b/core/services/lakefs/src/config.rs
@@ -17,6 +17,7 @@
use std::fmt::Debug;
+use opendal_core::*;
use serde::Deserialize;
use serde::Serialize;
@@ -66,16 +67,13 @@ impl Debug for LakefsConfig {
}
}
-impl crate::Configurator for LakefsConfig {
+impl Configurator for LakefsConfig {
type Builder = LakefsBuilder;
- fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> {
+ fn from_uri(uri: &OperatorUri) -> Result<Self> {
let raw_path = uri.root().ok_or_else(|| {
- crate::Error::new(
- crate::ErrorKind::ConfigInvalid,
- "uri path must contain repository",
- )
- .with_context("service", LAKEFS_SCHEME)
+ Error::new(ErrorKind::ConfigInvalid, "uri path must contain repository")
+ .with_context("service", LAKEFS_SCHEME)
})?;
let (repository, remainder) = match raw_path.split_once('/') {
@@ -89,8 +87,8 @@ impl crate::Configurator for LakefsConfig {
Some(repository)
}
.ok_or_else(|| {
- crate::Error::new(
- crate::ErrorKind::ConfigInvalid,
+ Error::new(
+ ErrorKind::ConfigInvalid,
"repository is required in uri path",
)
.with_context("service", LAKEFS_SCHEME)
@@ -136,42 +134,40 @@ impl crate::Configurator for LakefsConfig {
#[cfg(test)]
mod tests {
use super::*;
- use crate::Configurator;
- use crate::types::OperatorUri;
#[test]
- fn from_uri_sets_endpoint_repository_branch_and_root() {
+ fn from_uri_sets_endpoint_repository_branch_and_root() -> Result<()> {
let uri = OperatorUri::new(
"lakefs://api.example.com/sample/main/data/dir",
Vec::<(String, String)>::new(),
- )
- .unwrap();
+ )?;
- let cfg = LakefsConfig::from_uri(&uri).unwrap();
+ let cfg = LakefsConfig::from_uri(&uri)?;
assert_eq!(cfg.endpoint.as_deref(), Some("https://api.example.com"));
assert_eq!(cfg.repository.as_deref(), Some("sample"));
assert_eq!(cfg.branch.as_deref(), Some("main"));
assert_eq!(cfg.root.as_deref(), Some("data/dir"));
+ Ok(())
}
#[test]
- fn from_uri_requires_repository() {
- let uri =
- OperatorUri::new("lakefs://api.example.com", Vec::<(String, String)>::new()).unwrap();
+ fn from_uri_requires_repository() -> Result<()> {
+ let uri = OperatorUri::new("lakefs://api.example.com", Vec::<(String, String)>::new())?;
assert!(LakefsConfig::from_uri(&uri).is_err());
+ Ok(())
}
#[test]
- fn from_uri_respects_branch_override_and_sets_root() {
+ fn from_uri_respects_branch_override_and_sets_root() -> Result<()> {
let uri = OperatorUri::new(
"lakefs://api.example.com/sample/content",
vec![("branch".to_string(), "develop".to_string())],
- )
- .unwrap();
+ )?;
- let cfg = LakefsConfig::from_uri(&uri).unwrap();
+ let cfg = LakefsConfig::from_uri(&uri)?;
assert_eq!(cfg.branch.as_deref(), Some("develop"));
assert_eq!(cfg.root.as_deref(), Some("content"));
+ Ok(())
}
}
diff --git a/core/core/src/services/lakefs/core.rs b/core/services/lakefs/src/core.rs
similarity index 99%
rename from core/core/src/services/lakefs/core.rs
rename to core/services/lakefs/src/core.rs
index 86051acd4..425184516 100644
--- a/core/core/src/services/lakefs/core.rs
+++ b/core/services/lakefs/src/core.rs
@@ -22,11 +22,11 @@ use std::sync::Arc;
use http::Request;
use http::Response;
use http::header;
+use opendal_core::raw::*;
+use opendal_core::*;
use serde::Deserialize;
-use crate::raw::*;
-use crate::*;
-
+/// Core shared state for lakefs service.
pub struct LakefsCore {
pub info: Arc<AccessorInfo>,
pub endpoint: String,
diff --git a/core/core/src/services/lakefs/deleter.rs b/core/services/lakefs/src/deleter.rs
similarity index 94%
rename from core/core/src/services/lakefs/deleter.rs
rename to core/services/lakefs/src/deleter.rs
index a75820919..ae0c7a6c5 100644
--- a/core/core/src/services/lakefs/deleter.rs
+++ b/core/services/lakefs/src/deleter.rs
@@ -18,12 +18,13 @@
use std::sync::Arc;
use http::StatusCode;
+use opendal_core::raw::*;
+use opendal_core::*;
use super::core::*;
use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
+/// Deleter for lakefs service operations.
pub struct LakefsDeleter {
core: Arc<LakefsCore>,
}
diff --git a/core/core/src/services/lakefs/docs.md b/core/services/lakefs/src/docs.md
similarity index 96%
rename from core/core/src/services/lakefs/docs.md
rename to core/services/lakefs/src/docs.md
index 20348c7fd..cb23d4f85 100644
--- a/core/core/src/services/lakefs/docs.md
+++ b/core/services/lakefs/src/docs.md
@@ -34,8 +34,8 @@ Refer to [`LakefsBuilder`]'s public API docs for more information.
```rust,no_run
use opendal_core::Operator;
-use opendal_core::services::Lakefs;
-use anyhow::Result;
+use opendal_core::Result;
+use opendal_service_lakefs::Lakefs;
#[tokio::main]
async fn main() -> Result<()> {
diff --git a/core/core/src/services/lakefs/error.rs b/core/services/lakefs/src/error.rs
similarity index 96%
rename from core/core/src/services/lakefs/error.rs
rename to core/services/lakefs/src/error.rs
index 3e64b0eea..f8bd43f57 100644
--- a/core/core/src/services/lakefs/error.rs
+++ b/core/services/lakefs/src/error.rs
@@ -19,11 +19,10 @@ use std::fmt::Debug;
use http::Response;
use http::StatusCode;
+use opendal_core::raw::*;
+use opendal_core::*;
use serde::Deserialize;
-use crate::raw::*;
-use crate::*;
-
/// LakefsError is the error returned by Lakefs File System.
#[derive(Default, Deserialize)]
struct LakefsError {
@@ -72,8 +71,7 @@ pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
#[cfg(test)]
mod test {
use super::*;
- use crate::raw::new_json_deserialize_error;
- use crate::types::Result;
+ use opendal_core::raw::new_json_deserialize_error;
#[test]
fn test_parse_error() -> Result<()> {
diff --git a/core/core/src/services/lakefs/mod.rs b/core/services/lakefs/src/lib.rs
similarity index 84%
rename from core/core/src/services/lakefs/mod.rs
rename to core/services/lakefs/src/lib.rs
index cc55ed939..b9e0076f8 100644
--- a/core/core/src/services/lakefs/mod.rs
+++ b/core/services/lakefs/src/lib.rs
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-/// Default scheme for lakefs service.
-pub const LAKEFS_SCHEME: &str = "lakefs";
+//! lakeFS service implementation for Apache OpenDAL.
-use crate::types::DEFAULT_OPERATOR_REGISTRY;
+#![cfg_attr(docsrs, feature(doc_cfg))]
+#![deny(missing_docs)]
mod backend;
mod config;
@@ -31,7 +31,10 @@ mod writer;
pub use backend::LakefsBuilder as Lakefs;
pub use config::LakefsConfig;
+/// Default scheme for lakefs service.
+pub const LAKEFS_SCHEME: &str = "lakefs";
+
#[ctor::ctor]
fn register_lakefs_service() {
- DEFAULT_OPERATOR_REGISTRY.register::<Lakefs>(LAKEFS_SCHEME);
+ opendal_core::DEFAULT_OPERATOR_REGISTRY.register::<Lakefs>(LAKEFS_SCHEME);
}
diff --git a/core/core/src/services/lakefs/lister.rs b/core/services/lakefs/src/lister.rs
similarity index 97%
rename from core/core/src/services/lakefs/lister.rs
rename to core/services/lakefs/src/lister.rs
index fe49f1529..76e207160 100644
--- a/core/core/src/services/lakefs/lister.rs
+++ b/core/services/lakefs/src/lister.rs
@@ -17,13 +17,15 @@
use std::sync::Arc;
+use bytes::Buf;
+use opendal_core::raw::*;
+use opendal_core::*;
+
use super::core::LakefsCore;
use super::core::LakefsListResponse;
use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-use bytes::Buf;
+/// Lister for lakefs service operations.
pub struct LakefsLister {
core: Arc<LakefsCore>,
path: String,
diff --git a/core/core/src/services/lakefs/writer.rs b/core/services/lakefs/src/writer.rs
similarity index 97%
rename from core/core/src/services/lakefs/writer.rs
rename to core/services/lakefs/src/writer.rs
index 8699cc07e..d736eebbb 100644
--- a/core/core/src/services/lakefs/writer.rs
+++ b/core/services/lakefs/src/writer.rs
@@ -19,13 +19,14 @@ use std::sync::Arc;
use bytes::Buf;
use http::StatusCode;
+use opendal_core::raw::*;
+use opendal_core::*;
use super::core::LakefsCore;
use super::core::LakefsStatus;
use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
+/// Writer for lakefs service operations.
pub struct LakefsWriter {
core: Arc<LakefsCore>,
op: OpWrite,
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 055936fe5..66be1e9cc 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -66,6 +66,10 @@ pub mod services {
pub use opendal_service_hdfs_native::*;
#[cfg(feature = "services-ipfs")]
pub use opendal_service_ipfs::*;
+ #[cfg(feature = "services-koofr")]
+ pub use opendal_service_koofr::*;
+ #[cfg(feature = "services-lakefs")]
+ pub use opendal_service_lakefs::*;
#[cfg(feature = "services-memcached")]
pub use opendal_service_memcached::*;
#[cfg(feature = "services-mini-moka")]