You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by ps...@apache.org on 2025/12/22 10:10:33 UTC
(opendal) branch feature/layer-hotpath created (now be41b5178)
This is an automated email from the ASF dual-hosted git repository.
psiace pushed a change to branch feature/layer-hotpath
in repository https://gitbox.apache.org/repos/asf/opendal.git
at be41b5178 feat(layers/hotpath): add HotpathLayer integration
This branch includes the following new commits:
new be41b5178 feat(layers/hotpath): add HotpathLayer integration
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opendal) 01/01: feat(layers/hotpath): add HotpathLayer integration
Posted by ps...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
psiace pushed a commit to branch feature/layer-hotpath
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit be41b5178cafd1084e8b019bd4338b1874ff1958
Author: Chojan Shang <ps...@apache.org>
AuthorDate: Mon Dec 22 18:10:08 2025 +0800
feat(layers/hotpath): add HotpathLayer integration
Signed-off-by: Chojan Shang <ps...@apache.org>
---
core/Cargo.lock | 133 ++++++++++++++++++++++++++++
core/Cargo.toml | 2 +
core/layers/hotpath/Cargo.toml | 38 ++++++++
core/layers/hotpath/src/lib.rs | 193 +++++++++++++++++++++++++++++++++++++++++
core/src/lib.rs | 2 +
5 files changed, 368 insertions(+)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index c5ca7a3a8..72f4515e7 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -245,6 +245,12 @@ dependencies = [
"stable_deref_trait",
]
+[[package]]
+name = "ascii"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16"
+
[[package]]
name = "ascii-canvas"
version = "3.0.0"
@@ -1645,6 +1651,12 @@ dependencies = [
"windows-link",
]
+[[package]]
+name = "chunked_transfer"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901"
+
[[package]]
name = "ciborium"
version = "0.2.2"
@@ -2767,6 +2779,12 @@ dependencies = [
"log",
]
+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
[[package]]
name = "encoding_rs"
version = "0.8.35"
@@ -2892,6 +2910,16 @@ dependencies = [
"pin-project-lite",
]
+[[package]]
+name = "eyre"
+version = "0.6.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec"
+dependencies = [
+ "indenter",
+ "once_cell",
+]
+
[[package]]
name = "fail"
version = "0.4.0"
@@ -3711,6 +3739,16 @@ dependencies = [
"java-locator",
]
+[[package]]
+name = "hdrhistogram"
+version = "7.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
+dependencies = [
+ "byteorder",
+ "num-traits",
+]
+
[[package]]
name = "hdrs"
version = "0.3.2"
@@ -3851,6 +3889,44 @@ dependencies = [
"windows-sys 0.59.0",
]
+[[package]]
+name = "hotpath"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "daa7e73e5b9dd467b2cbe7cc50ba224cd43bfcc0a618fd75f4adc32cf3a95d66"
+dependencies = [
+ "arc-swap",
+ "base64 0.22.1",
+ "cfg-if",
+ "clap",
+ "colored",
+ "crossbeam-channel",
+ "eyre",
+ "futures-util",
+ "hdrhistogram",
+ "hotpath-macros",
+ "libc",
+ "mach2",
+ "pin-project-lite",
+ "prettytable-rs",
+ "quanta",
+ "regex",
+ "serde",
+ "serde_json",
+ "tiny_http",
+]
+
+[[package]]
+name = "hotpath-macros"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4aca1659ca1e70edc8f96de48530a104bec11a267f663625098137bfd072d601"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.111",
+]
+
[[package]]
name = "html5ever"
version = "0.35.0"
@@ -4215,6 +4291,12 @@ dependencies = [
"icu_properties",
]
+[[package]]
+name = "indenter"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5"
+
[[package]]
name = "indexmap"
version = "1.9.3"
@@ -4341,6 +4423,17 @@ dependencies = [
"serde",
]
+[[package]]
+name = "is-terminal"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "is_terminal_polyfill"
version = "1.70.2"
@@ -4829,6 +4922,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
+[[package]]
+name = "mach2"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b"
+
[[package]]
name = "macro_magic"
version = "0.5.1"
@@ -5542,6 +5641,7 @@ dependencies = [
"opendal-layer-dtrace",
"opendal-layer-fastmetrics",
"opendal-layer-fastrace",
+ "opendal-layer-hotpath",
"opendal-layer-immutable-index",
"opendal-layer-logging",
"opendal-layer-metrics",
@@ -5804,6 +5904,14 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "opendal-layer-hotpath"
+version = "0.55.0"
+dependencies = [
+ "hotpath",
+ "opendal-core",
+]
+
[[package]]
name = "opendal-layer-immutable-index"
version = "0.55.0"
@@ -7517,6 +7625,19 @@ dependencies = [
"syn 2.0.111",
]
+[[package]]
+name = "prettytable-rs"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a"
+dependencies = [
+ "encode_unicode",
+ "is-terminal",
+ "lazy_static",
+ "term",
+ "unicode-width",
+]
+
[[package]]
name = "probe"
version = "0.5.2"
@@ -10250,6 +10371,18 @@ dependencies = [
"crunchy",
]
+[[package]]
+name = "tiny_http"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82"
+dependencies = [
+ "ascii",
+ "chunked_transfer",
+ "httpdate",
+ "log",
+]
+
[[package]]
name = "tinystr"
version = "0.8.2"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 43b061a1e..9a1ab443e 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -99,6 +99,7 @@ layers-concurrent-limit = ["dep:opendal-layer-concurrent-limit"]
layers-dtrace = ["dep:opendal-layer-dtrace"]
layers-fastmetrics = ["dep:opendal-layer-fastmetrics"]
layers-fastrace = ["dep:opendal-layer-fastrace"]
+layers-hotpath = ["dep:opendal-layer-hotpath"]
layers-immutable-index = ["dep:opendal-layer-immutable-index"]
layers-logging = ["dep:opendal-layer-logging"]
layers-metrics = ["dep:opendal-layer-metrics"]
@@ -206,6 +207,7 @@ opendal-layer-concurrent-limit = { path = "layers/concurrent-limit", version = "
opendal-layer-dtrace = { path = "layers/dtrace", version = "0.55.0", optional = true, default-features = false }
opendal-layer-fastmetrics = { path = "layers/fastmetrics", version = "0.55.0", optional = true, default-features = false }
opendal-layer-fastrace = { path = "layers/fastrace", version = "0.55.0", optional = true, default-features = false }
+opendal-layer-hotpath = { path = "layers/hotpath", version = "0.55.0", optional = true, default-features = false }
opendal-layer-immutable-index = { path = "layers/immutable-index", version = "0.55.0", optional = true, default-features = false }
opendal-layer-logging = { path = "layers/logging", version = "0.55.0", optional = true, default-features = false }
opendal-layer-metrics = { path = "layers/metrics", version = "0.55.0", optional = true, default-features = false }
diff --git a/core/layers/hotpath/Cargo.toml b/core/layers/hotpath/Cargo.toml
new file mode 100644
index 000000000..7e1af4738
--- /dev/null
+++ b/core/layers/hotpath/Cargo.toml
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL hotpath layer"
+name = "opendal-layer-hotpath"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+hotpath = { version = "0.9.1", features = ["hotpath"] }
+opendal-core = { path = "../../core", version = "0.55.0", default-features = false }
+
+[dev-dependencies]
+opendal-core = { path = "../../core", version = "0.55.0" }
diff --git a/core/layers/hotpath/src/lib.rs b/core/layers/hotpath/src/lib.rs
new file mode 100644
index 000000000..f0b9c3d8b
--- /dev/null
+++ b/core/layers/hotpath/src/lib.rs
@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use hotpath::MeasurementGuard;
+use opendal_core::raw::*;
+use opendal_core::*;
+
+const LABEL_CREATE_DIR: &str = "opendal.create_dir";
+const LABEL_READ: &str = "opendal.read";
+const LABEL_WRITE: &str = "opendal.write";
+const LABEL_COPY: &str = "opendal.copy";
+const LABEL_RENAME: &str = "opendal.rename";
+const LABEL_STAT: &str = "opendal.stat";
+const LABEL_DELETE: &str = "opendal.delete";
+const LABEL_LIST: &str = "opendal.list";
+const LABEL_PRESIGN: &str = "opendal.presign";
+
+const LABEL_READER_READ: &str = "opendal.reader.read";
+const LABEL_WRITER_WRITE: &str = "opendal.writer.write";
+const LABEL_WRITER_CLOSE: &str = "opendal.writer.close";
+const LABEL_WRITER_ABORT: &str = "opendal.writer.abort";
+const LABEL_LISTER_NEXT: &str = "opendal.lister.next";
+const LABEL_DELETER_DELETE: &str = "opendal.deleter.delete";
+const LABEL_DELETER_CLOSE: &str = "opendal.deleter.close";
+
+/// Add [hotpath](https://docs.rs/hotpath/) profiling for every operation.
+///
+/// # Notes
+///
+/// When `hotpath` profiling is enabled, initialize a guard via
+/// [`hotpath::FunctionsGuardBuilder`] or `#[hotpath::main]` before running
+/// operations. Otherwise, hotpath will panic on the first measurement.
+///
+/// # Examples
+///
+/// ```no_run
+/// # use opendal_core::services;
+/// # use opendal_core::Operator;
+/// # use opendal_core::Result;
+/// # use opendal_layer_hotpath::HotpathLayer;
+/// #
+/// # #[tokio::main]
+/// # async fn main() -> Result<()> {
+/// let _guard = hotpath::FunctionsGuardBuilder::new("opendal").build();
+/// let op = Operator::new(services::Memory::default())?
+/// .layer(HotpathLayer)
+/// .finish();
+/// op.write("test", "hello").await?;
+/// # Ok(())
+/// # }
+/// ```
+pub struct HotpathLayer;
+
+impl<A: Access> Layer<A> for HotpathLayer {
+ type LayeredAccess = HotpathAccessor<A>;
+
+ fn layer(&self, inner: A) -> Self::LayeredAccess {
+ HotpathAccessor { inner }
+ }
+}
+
+#[derive(Debug)]
+pub struct HotpathAccessor<A> {
+ inner: A,
+}
+
+impl<A: Access> LayeredAccess for HotpathAccessor<A> {
+ type Inner = A;
+ type Reader = HotpathWrapper<A::Reader>;
+ type Writer = HotpathWrapper<A::Writer>;
+ type Lister = HotpathWrapper<A::Lister>;
+ type Deleter = HotpathWrapper<A::Deleter>;
+
+ fn inner(&self) -> &Self::Inner {
+ &self.inner
+ }
+
+ async fn create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
+ let _guard = MeasurementGuard::build(LABEL_CREATE_DIR, false, true);
+ self.inner.create_dir(path, args).await
+ }
+
+ async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+ let _guard = MeasurementGuard::build(LABEL_READ, false, true);
+ let (rp, reader) = self.inner.read(path, args).await?;
+ Ok((rp, HotpathWrapper::new(reader)))
+ }
+
+ async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+ let _guard = MeasurementGuard::build(LABEL_WRITE, false, true);
+ let (rp, writer) = self.inner.write(path, args).await?;
+ Ok((rp, HotpathWrapper::new(writer)))
+ }
+
+ async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
+ let _guard = MeasurementGuard::build(LABEL_COPY, false, true);
+ self.inner().copy(from, to, args).await
+ }
+
+ async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
+ let _guard = MeasurementGuard::build(LABEL_RENAME, false, true);
+ self.inner().rename(from, to, args).await
+ }
+
+ async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
+ let _guard = MeasurementGuard::build(LABEL_STAT, false, true);
+ self.inner.stat(path, args).await
+ }
+
+ async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+ let _guard = MeasurementGuard::build(LABEL_DELETE, false, true);
+ let (rp, deleter) = self.inner.delete().await?;
+ Ok((rp, HotpathWrapper::new(deleter)))
+ }
+
+ async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+ let _guard = MeasurementGuard::build(LABEL_LIST, false, true);
+ let (rp, lister) = self.inner.list(path, args).await?;
+ Ok((rp, HotpathWrapper::new(lister)))
+ }
+
+ async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
+ let _guard = MeasurementGuard::build(LABEL_PRESIGN, false, true);
+ self.inner.presign(path, args).await
+ }
+}
+
+pub struct HotpathWrapper<R> {
+ inner: R,
+}
+
+impl<R> HotpathWrapper<R> {
+ fn new(inner: R) -> Self {
+ Self { inner }
+ }
+}
+
+impl<R: oio::Read> oio::Read for HotpathWrapper<R> {
+ async fn read(&mut self) -> Result<Buffer> {
+ let _guard = MeasurementGuard::build(LABEL_READER_READ, false, true);
+ self.inner.read().await
+ }
+}
+
+impl<R: oio::Write> oio::Write for HotpathWrapper<R> {
+ async fn write(&mut self, bs: Buffer) -> Result<()> {
+ let _guard = MeasurementGuard::build(LABEL_WRITER_WRITE, false, true);
+ self.inner.write(bs).await
+ }
+
+ async fn close(&mut self) -> Result<Metadata> {
+ let _guard = MeasurementGuard::build(LABEL_WRITER_CLOSE, false, true);
+ self.inner.close().await
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ let _guard = MeasurementGuard::build(LABEL_WRITER_ABORT, false, true);
+ self.inner.abort().await
+ }
+}
+
+impl<R: oio::List> oio::List for HotpathWrapper<R> {
+ async fn next(&mut self) -> Result<Option<oio::Entry>> {
+ let _guard = MeasurementGuard::build(LABEL_LISTER_NEXT, false, true);
+ self.inner.next().await
+ }
+}
+
+impl<R: oio::Delete> oio::Delete for HotpathWrapper<R> {
+ async fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
+ let _guard = MeasurementGuard::build(LABEL_DELETER_DELETE, false, true);
+ self.inner.delete(path, args).await
+ }
+
+ async fn close(&mut self) -> Result<()> {
+ let _guard = MeasurementGuard::build(LABEL_DELETER_CLOSE, false, true);
+ self.inner.close().await
+ }
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index a17f8f792..b69107c8e 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -167,6 +167,8 @@ pub mod layers {
pub use opendal_layer_fastmetrics::*;
#[cfg(feature = "layers-fastrace")]
pub use opendal_layer_fastrace::*;
+ #[cfg(feature = "layers-hotpath")]
+ pub use opendal_layer_hotpath::*;
#[cfg(feature = "layers-immutable-index")]
pub use opendal_layer_immutable_index::*;
#[cfg(feature = "layers-logging")]