You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/04/26 03:01:59 UTC
(opendal) branch main updated: feat(services/compfs): compio runtime and compfs structure (#4534)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 390ae87885 feat(services/compfs): compio runtime and compfs structure (#4534)
390ae87885 is described below
commit 390ae8788572eaab96476d25b1d3653191249afa
Author: Pop <gm...@miao.dev>
AuthorDate: Fri Apr 26 12:01:54 2024 +0900
feat(services/compfs): compio runtime and compfs structure (#4534)
* feat(services/compfs): compio thread
* feat(services/compfs): basic structure for compfs
* feat(services/compfs): scheme impl
* fix naming
---
core/src/services/compfs/backend.rs | 78 ++++++++++++++++++++++
core/src/services/compfs/core.rs | 91 ++++++++++++++++++++++++++
core/src/services/compfs/{mod.rs => lister.rs} | 2 -
core/src/services/compfs/mod.rs | 8 +++
core/src/services/compfs/{mod.rs => reader.rs} | 2 -
core/src/services/compfs/{mod.rs => writer.rs} | 2 -
core/src/services/mod.rs | 2 +
core/src/types/scheme.rs | 6 ++
8 files changed, 185 insertions(+), 6 deletions(-)
diff --git a/core/src/services/compfs/backend.rs b/core/src/services/compfs/backend.rs
new file mode 100644
index 0000000000..846332ba7a
--- /dev/null
+++ b/core/src/services/compfs/backend.rs
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::core::CompioThread;
+use crate::raw::*;
+use crate::*;
+use async_trait::async_trait;
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+/// [`compio`]-based file system support.
+#[derive(Debug, Clone, Default)]
+pub struct CompfsBuilder {
+ root: Option<PathBuf>,
+}
+
+impl CompfsBuilder {
+ /// Set root for Compfs
+ pub fn root(&mut self, root: &str) -> &mut Self {
+ self.root = if root.is_empty() {
+ None
+ } else {
+ Some(PathBuf::from(root))
+ };
+
+ self
+ }
+}
+
+impl Builder for CompfsBuilder {
+ const SCHEME: Scheme = Scheme::Compfs;
+ type Accessor = ();
+
+ fn from_map(map: HashMap<String, String>) -> Self {
+ let mut builder = CompfsBuilder::default();
+
+ map.get("root").map(|v| builder.root(v));
+
+ builder
+ }
+
+ fn build(&mut self) -> Result<Self::Accessor> {
+ todo!()
+ }
+}
+
+#[derive(Debug)]
+pub struct CompfsBackend {
+ rt: CompioThread,
+}
+
+#[async_trait]
+impl Accessor for CompfsBackend {
+ type Reader = ();
+ type Writer = ();
+ type Lister = ();
+ type BlockingReader = ();
+ type BlockingWriter = ();
+ type BlockingLister = ();
+
+ fn info(&self) -> AccessorInfo {
+ todo!()
+ }
+}
diff --git a/core/src/services/compfs/core.rs b/core/src/services/compfs/core.rs
index c229ea4807..b95db7c978 100644
--- a/core/src/services/compfs/core.rs
+++ b/core/src/services/compfs/core.rs
@@ -16,9 +16,100 @@
// under the License.
use compio::buf::IoBuf;
+use compio::runtime::RuntimeBuilder;
+use futures::channel::mpsc::SendError;
+use futures::channel::{mpsc, oneshot};
+use futures::future::LocalBoxFuture;
+use futures::{SinkExt, StreamExt};
+use std::future::Future;
+use std::thread::JoinHandle;
use crate::Buffer;
+/// This is arbitrary, but since all tasks are spawned instantly, we shouldn't need a too big buffer.
+const CHANNEL_SIZE: usize = 4;
+
+fn task<F, Fut, T>(func: F) -> (Task<T>, SpawnTask)
+where
+ F: (FnOnce() -> Fut) + Send + 'static,
+ Fut: Future<Output = T>,
+ T: Send + 'static,
+{
+ let (tx, recv) = oneshot::channel();
+
+ let boxed = Box::new(|| {
+ Box::pin(async move {
+ let res = func().await;
+ tx.send(res).ok();
+ }) as _
+ });
+
+ (Task(recv), SpawnTask(boxed))
+}
+
+/// A task handle that can be used to retrieve result spawned into [`CompioThread`].
+pub struct Task<T>(oneshot::Receiver<T>);
+
+/// Type erased task that can be spawned into a [`CompioThread`].
+struct SpawnTask(Box<dyn (FnOnce() -> LocalBoxFuture<'static, ()>) + Send>);
+
+impl SpawnTask {
+ fn call(self) -> LocalBoxFuture<'static, ()> {
+ (self.0)()
+ }
+}
+
+#[derive(Debug)]
+pub struct CompioThread {
+ thread: JoinHandle<()>,
+ handle: SpawnHandle,
+}
+
+impl CompioThread {
+ pub fn new(builder: RuntimeBuilder) -> Self {
+ let (send, mut recv) = mpsc::channel(CHANNEL_SIZE);
+ let handle = SpawnHandle(send);
+ let thread = std::thread::spawn(move || {
+ let rt = builder.build().expect("failed to create runtime");
+ rt.block_on(async {
+ while let Some(task) = recv.next().await {
+ rt.spawn(task.call()).detach();
+ }
+ });
+ });
+ Self { thread, handle }
+ }
+
+ pub async fn spawn<F, Fut, T>(&self, func: F) -> Result<Task<T>, SendError>
+ where
+ F: (FnOnce() -> Fut) + Send + 'static,
+ Fut: Future<Output = T>,
+ T: Send + 'static,
+ {
+ self.handle.clone().spawn(func).await
+ }
+
+ pub fn handle(&self) -> SpawnHandle {
+ self.handle.clone()
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct SpawnHandle(mpsc::Sender<SpawnTask>);
+
+impl SpawnHandle {
+ pub async fn spawn<F, Fut, T>(&mut self, func: F) -> Result<Task<T>, SendError>
+ where
+ F: (FnOnce() -> Fut) + Send + 'static,
+ Fut: Future<Output = T>,
+ T: Send + 'static,
+ {
+ let (task, spawn) = task(func);
+ self.0.send(spawn).await?;
+ Ok(task)
+ }
+}
+
unsafe impl IoBuf for Buffer {
fn as_buf_ptr(&self) -> *const u8 {
self.current().as_ptr()
diff --git a/core/src/services/compfs/mod.rs b/core/src/services/compfs/lister.rs
similarity index 98%
copy from core/src/services/compfs/mod.rs
copy to core/src/services/compfs/lister.rs
index f0e764508f..b248758bc1 100644
--- a/core/src/services/compfs/mod.rs
+++ b/core/src/services/compfs/lister.rs
@@ -14,5 +14,3 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-
-mod core;
diff --git a/core/src/services/compfs/mod.rs b/core/src/services/compfs/mod.rs
index f0e764508f..678c379176 100644
--- a/core/src/services/compfs/mod.rs
+++ b/core/src/services/compfs/mod.rs
@@ -15,4 +15,12 @@
// specific language governing permissions and limitations
// under the License.
+#![allow(dead_code)] // TODO: Remove this after backend is implemented
+
+mod backend;
mod core;
+mod lister;
+mod reader;
+mod writer;
+
+pub use backend::CompfsBuilder as Compfs;
diff --git a/core/src/services/compfs/mod.rs b/core/src/services/compfs/reader.rs
similarity index 98%
copy from core/src/services/compfs/mod.rs
copy to core/src/services/compfs/reader.rs
index f0e764508f..b248758bc1 100644
--- a/core/src/services/compfs/mod.rs
+++ b/core/src/services/compfs/reader.rs
@@ -14,5 +14,3 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-
-mod core;
diff --git a/core/src/services/compfs/mod.rs b/core/src/services/compfs/writer.rs
similarity index 98%
copy from core/src/services/compfs/mod.rs
copy to core/src/services/compfs/writer.rs
index f0e764508f..b248758bc1 100644
--- a/core/src/services/compfs/mod.rs
+++ b/core/src/services/compfs/writer.rs
@@ -14,5 +14,3 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-
-mod core;
diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs
index 67f01ca94e..1ed7ba482c 100644
--- a/core/src/services/mod.rs
+++ b/core/src/services/mod.rs
@@ -409,3 +409,5 @@ pub use surrealdb::SurrealdbConfig;
#[cfg(feature = "services-compfs")]
mod compfs;
+#[cfg(feature = "services-compfs")]
+pub use compfs::Compfs;
diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs
index 572edd8057..132c150e45 100644
--- a/core/src/types/scheme.rs
+++ b/core/src/types/scheme.rs
@@ -40,6 +40,8 @@ pub enum Scheme {
Azdls,
/// [B2][crate::services::B2]: Backblaze B2 Services.
B2,
+ /// [Compfs][crate::services::Compfs]: Compio fs Services.
+ Compfs,
/// [Seafile][crate::services::Seafile]: Seafile Services.
Seafile,
/// [Upyun][crate::services::Upyun]: Upyun Services.
@@ -205,6 +207,8 @@ impl Scheme {
Scheme::Cacache,
#[cfg(feature = "services-cos")]
Scheme::Cos,
+ #[cfg(feature = "services-compfs")]
+ Scheme::Compfs,
#[cfg(feature = "services-dashmap")]
Scheme::Dashmap,
#[cfg(feature = "services-dropbox")]
@@ -332,6 +336,7 @@ impl FromStr for Scheme {
"b2" => Ok(Scheme::B2),
"chainsafe" => Ok(Scheme::Chainsafe),
"cacache" => Ok(Scheme::Cacache),
+ "compfs" => Ok(Scheme::Compfs),
"cloudflare_kv" => Ok(Scheme::CloudflareKv),
"cos" => Ok(Scheme::Cos),
"d1" => Ok(Scheme::D1),
@@ -402,6 +407,7 @@ impl From<Scheme> for &'static str {
Scheme::Cacache => "cacache",
Scheme::CloudflareKv => "cloudflare_kv",
Scheme::Cos => "cos",
+ Scheme::Compfs => "compfs",
Scheme::D1 => "d1",
Scheme::Dashmap => "dashmap",
Scheme::Etcd => "etcd",