You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by ko...@apache.org on 2025/06/08 11:49:07 UTC

(opendal) 01/01: feat(layers): add fastmetrics layer

This is an automated email from the ASF dual-hosted git repository.

koushiro pushed a commit to branch add-fastmetrics-layer
in repository https://gitbox.apache.org/repos/asf/opendal.git

commit 575710fb7888f2345a427a371a98ad95c2ce9d61
Author: koushiro <ko...@gmail.com>
AuthorDate: Sun Jun 8 19:48:54 2025 +0800

    feat(layers): add fastmetrics layer
---
 core/Cargo.lock                |  15 ++
 core/Cargo.toml                |   4 +
 core/src/layers/fastmetrics.rs | 459 +++++++++++++++++++++++++++++++++++++++++
 core/src/layers/mod.rs         |   7 +
 4 files changed, 485 insertions(+)

diff --git a/core/Cargo.lock b/core/Cargo.lock
index eb7d85be3..f6393c455 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -2908,6 +2908,20 @@ dependencies = [
  "web-time",
 ]
 
+[[package]]
+name = "fastmetrics"
+version = "0.3.0-rc.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "774a5f2be60bec7269d30012dae25a1ead785244a8f38e06e2dd5163b595d083"
+dependencies = [
+ "cfg-if",
+ "dtoa",
+ "foldhash",
+ "itoa",
+ "parking_lot 0.12.3",
+ "paste",
+]
+
 [[package]]
 name = "fastrace"
 version = "0.7.9"
@@ -5356,6 +5370,7 @@ dependencies = [
  "divan",
  "dotenvy",
  "etcd-client",
+ "fastmetrics",
  "fastrace",
  "fastrace-jaeger",
  "flume",
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 74671155f..78baea1ec 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -91,6 +91,8 @@ layers-mime-guess = ["dep:mime_guess"]
 layers-prometheus = ["dep:prometheus"]
 # Enable layers prometheus support, with prometheus-client crate
 layers-prometheus-client = ["dep:prometheus-client"]
+# Enable layers fastmetrics support.
+layers-fastmetrics = ["dep:fastmetrics"]
 # Enable layers fastrace support.
 layers-fastrace = ["dep:fastrace"]
 # Enable layers tracing support.
@@ -397,6 +399,8 @@ opentelemetry = { version = "0.30.0", optional = true }
 prometheus = { version = "0.13", features = ["process"], optional = true }
 # for layers-prometheus-client
 prometheus-client = { version = "0.23.1", optional = true }
+# for fastmetrics
+fastmetrics = { version = "0.3.0-rc.4", optional = true }
 # for layers-tracing
 tracing = { version = "0.1", optional = true }
 # for layers-dtrace
diff --git a/core/src/layers/fastmetrics.rs b/core/src/layers/fastmetrics.rs
new file mode 100644
index 000000000..47199320c
--- /dev/null
+++ b/core/src/layers/fastmetrics.rs
@@ -0,0 +1,459 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::{fmt, time::Duration};
+
+use fastmetrics::encoder::EncodeLabelSet;
+use fastmetrics::encoder::LabelSetEncoder;
+use fastmetrics::metrics::counter::Counter;
+use fastmetrics::metrics::family::Family;
+use fastmetrics::metrics::family::MetricFactory;
+use fastmetrics::metrics::gauge::Gauge;
+use fastmetrics::metrics::histogram::Histogram;
+use fastmetrics::registry::with_global_registry_mut;
+use fastmetrics::registry::Register;
+use fastmetrics::registry::Registry;
+use fastmetrics::registry::RegistryError;
+
+use crate::layers::observe;
+use crate::raw::*;
+use crate::*;
+
+/// Add [fastmetrics](https://docs.rs/fastmetrics/) for every operation.
+///
+/// # Examples
+///
+/// ```no_run
+/// # use opendal::layers::FastmetricsLayer;
+/// # use opendal::services;
+/// # use opendal::Operator;
+/// # use opendal::Result;
+///
+/// # fn main() -> Result<()> {
+/// let mut registry = fastmetrics::registry::Registry::default();
+/// let _ = Operator::new(services::Memory::default())?
+///     .layer(FastmetricsLayer::builder().register(&mut registry)?)
+///     .finish();
+/// # Ok(())
+/// # }
+/// ```
+#[derive(Clone, Debug)]
+pub struct FastmetricsLayer {
+    interceptor: FastmetricsInterceptor,
+}
+
+impl FastmetricsLayer {
+    /// Create a [`FastmetricsLayerBuilder`] to set the configuration of metrics.
+    pub fn builder() -> FastmetricsLayerBuilder {
+        FastmetricsLayerBuilder::default()
+    }
+}
+
+impl<A: Access> Layer<A> for FastmetricsLayer {
+    type LayeredAccess = observe::MetricsAccessor<A, FastmetricsInterceptor>;
+
+    fn layer(&self, inner: A) -> Self::LayeredAccess {
+        observe::MetricsLayer::new(self.interceptor.clone()).layer(inner)
+    }
+}
+
+/// [`FastmetricsLayerBuilder`] is a config builder to build a [`FastmetricsLayer`].
+pub struct FastmetricsLayerBuilder {
+    bytes_buckets: Vec<f64>,
+    bytes_rate_buckets: Vec<f64>,
+    entries_buckets: Vec<f64>,
+    entries_rate_buckets: Vec<f64>,
+    duration_seconds_buckets: Vec<f64>,
+    ttfb_buckets: Vec<f64>,
+    disable_label_root: bool,
+}
+
+impl Default for FastmetricsLayerBuilder {
+    fn default() -> Self {
+        Self {
+            bytes_buckets: observe::DEFAULT_BYTES_BUCKETS.to_vec(),
+            bytes_rate_buckets: observe::DEFAULT_BYTES_RATE_BUCKETS.to_vec(),
+            entries_buckets: observe::DEFAULT_ENTRIES_BUCKETS.to_vec(),
+            entries_rate_buckets: observe::DEFAULT_ENTRIES_RATE_BUCKETS.to_vec(),
+            duration_seconds_buckets: observe::DEFAULT_DURATION_SECONDS_BUCKETS.to_vec(),
+            ttfb_buckets: observe::DEFAULT_TTFB_BUCKETS.to_vec(),
+            disable_label_root: false,
+        }
+    }
+}
+
+impl FastmetricsLayerBuilder {
+    /// Set buckets for bytes related histogram like `operation_bytes`.
+    pub fn bytes_buckets(mut self, buckets: Vec<f64>) -> Self {
+        if !buckets.is_empty() {
+            self.bytes_buckets = buckets;
+        }
+        self
+    }
+
+    /// Set buckets for bytes rate related histogram like `operation_bytes_rate`.
+    pub fn bytes_rate_buckets(mut self, buckets: Vec<f64>) -> Self {
+        if !buckets.is_empty() {
+            self.bytes_rate_buckets = buckets;
+        }
+        self
+    }
+
+    /// Set buckets for entries related histogram like `operation_entries`.
+    pub fn entries_buckets(mut self, buckets: Vec<f64>) -> Self {
+        if !buckets.is_empty() {
+            self.entries_buckets = buckets;
+        }
+        self
+    }
+
+    /// Set buckets for entries rate related histogram like `operation_entries_rate`.
+    pub fn entries_rate_buckets(mut self, buckets: Vec<f64>) -> Self {
+        if !buckets.is_empty() {
+            self.entries_rate_buckets = buckets;
+        }
+        self
+    }
+
+    /// Set buckets for duration seconds related histogram like `operation_duration_seconds`.
+    pub fn duration_seconds_buckets(mut self, buckets: Vec<f64>) -> Self {
+        if !buckets.is_empty() {
+            self.duration_seconds_buckets = buckets;
+        }
+        self
+    }
+
+    /// Set buckets for ttfb related histogram like `operation_ttfb_seconds`.
+    pub fn ttfb_buckets(mut self, buckets: Vec<f64>) -> Self {
+        if !buckets.is_empty() {
+            self.ttfb_buckets = buckets;
+        }
+        self
+    }
+
+    /// The 'root' label might have risks of being high cardinality; users can choose to disable it
+    /// when they found it's not useful for their metrics.
+    pub fn disable_label_root(mut self, disable: bool) -> Self {
+        self.disable_label_root = disable;
+        self
+    }
+
+    /// Register the metrics into the registry and return a [`FastmetricsLayer`].
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// # use log::debug;
+    /// # use opendal::layers::FastmetricsLayer;
+    /// # use opendal::services;
+    /// # use opendal::Operator;
+    /// # use opendal::Result;
+    ///
+    /// # #[tokio::main]
+    /// # async fn main() -> Result<()> {
+    /// let mut registry = fastmetrics::registry::Registry::default();
+    ///
+    /// // Pick a builder and configure it.
+    /// let builder = services::Memory::default();
+    /// let _ = Operator::new(builder)?
+    ///     .layer(FastmetricsLayer::builder().register(&mut registry)?)
+    ///     .finish();
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn register(self, registry: &mut Registry) -> Result<FastmetricsLayer> {
+        let operation_bytes = Family::<OperationLabels, Histogram, _>::new(HistogramFactory {
+            buckets: self.bytes_buckets.clone(),
+        });
+        let operation_bytes_rate = Family::<OperationLabels, Histogram, _>::new(HistogramFactory {
+            buckets: self.bytes_rate_buckets.clone(),
+        });
+        let operation_entries = Family::<OperationLabels, Histogram, _>::new(HistogramFactory {
+            buckets: self.entries_buckets.clone(),
+        });
+        let operation_entries_rate =
+            Family::<OperationLabels, Histogram, _>::new(HistogramFactory {
+                buckets: self.entries_rate_buckets.clone(),
+            });
+        let operation_duration_seconds =
+            Family::<OperationLabels, Histogram, _>::new(HistogramFactory {
+                buckets: self.duration_seconds_buckets.clone(),
+            });
+        let operation_errors_total = Family::<OperationLabels, Counter>::default();
+        let operation_executing = Family::<OperationLabels, Gauge>::default();
+        let operation_ttfb_seconds =
+            Family::<OperationLabels, Histogram, _>::new(HistogramFactory {
+                buckets: self.ttfb_buckets.clone(),
+            });
+
+        let http_executing = Family::<OperationLabels, Gauge>::default();
+        let http_request_bytes = Family::<OperationLabels, Histogram, _>::new(HistogramFactory {
+            buckets: self.bytes_buckets.clone(),
+        });
+        let http_request_bytes_rate =
+            Family::<OperationLabels, Histogram, _>::new(HistogramFactory {
+                buckets: self.bytes_rate_buckets.clone(),
+            });
+        let http_request_duration_seconds =
+            Family::<OperationLabels, Histogram, _>::new(HistogramFactory {
+                buckets: self.duration_seconds_buckets.clone(),
+            });
+        let http_response_bytes = Family::<OperationLabels, Histogram, _>::new(HistogramFactory {
+            buckets: self.bytes_buckets.clone(),
+        });
+        let http_response_bytes_rate =
+            Family::<OperationLabels, Histogram, _>::new(HistogramFactory {
+                buckets: self.bytes_rate_buckets.clone(),
+            });
+        let http_response_duration_seconds =
+            Family::<OperationLabels, Histogram, _>::new(HistogramFactory {
+                buckets: self.duration_seconds_buckets.clone(),
+            });
+        let http_connection_errors_total = Family::<OperationLabels, Counter>::default();
+        let http_status_errors_total = Family::<OperationLabels, Counter>::default();
+
+        let interceptor = FastmetricsInterceptor {
+            operation_bytes,
+            operation_bytes_rate,
+            operation_entries,
+            operation_entries_rate,
+            operation_duration_seconds,
+            operation_errors_total,
+            operation_executing,
+            operation_ttfb_seconds,
+
+            http_executing,
+            http_request_bytes,
+            http_request_bytes_rate,
+            http_request_duration_seconds,
+            http_response_bytes,
+            http_response_bytes_rate,
+            http_response_duration_seconds,
+            http_connection_errors_total,
+            http_status_errors_total,
+
+            disable_label_root: self.disable_label_root,
+        };
+        interceptor
+            .register(registry)
+            .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()).set_source(err))?;
+
+        Ok(FastmetricsLayer { interceptor })
+    }
+
+    /// Register the metrics into the global registry and return a [`FastmetricsLayer`].
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// # use opendal::layers::FastmetricsLayer;
+    /// # use opendal::services;
+    /// # use opendal::Operator;
+    /// # use opendal::Result;
+    ///
+    /// # fn main() -> Result<()> {
+    /// // Pick a builder and configure it.
+    /// let builder = services::Memory::default();
+    /// let _ = Operator::new(builder)?
+    ///     .layer(FastmetricsLayer::builder().register_global()?)
+    ///     .finish();
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn register_global(self) -> Result<FastmetricsLayer> {
+        with_global_registry_mut(|registry| self.register(registry))
+    }
+}
+
+#[derive(Clone)]
+struct HistogramFactory {
+    buckets: Vec<f64>,
+}
+
+impl MetricFactory<Histogram> for HistogramFactory {
+    fn new_metric(&self) -> Histogram {
+        Histogram::new(self.buckets.iter().cloned())
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct FastmetricsInterceptor {
+    operation_bytes: Family<OperationLabels, Histogram, HistogramFactory>,
+    operation_bytes_rate: Family<OperationLabels, Histogram, HistogramFactory>,
+    operation_entries: Family<OperationLabels, Histogram, HistogramFactory>,
+    operation_entries_rate: Family<OperationLabels, Histogram, HistogramFactory>,
+    operation_duration_seconds: Family<OperationLabels, Histogram, HistogramFactory>,
+    operation_errors_total: Family<OperationLabels, Counter>,
+    operation_executing: Family<OperationLabels, Gauge>,
+    operation_ttfb_seconds: Family<OperationLabels, Histogram, HistogramFactory>,
+
+    http_executing: Family<OperationLabels, Gauge>,
+    http_request_bytes: Family<OperationLabels, Histogram, HistogramFactory>,
+    http_request_bytes_rate: Family<OperationLabels, Histogram, HistogramFactory>,
+    http_request_duration_seconds: Family<OperationLabels, Histogram, HistogramFactory>,
+    http_response_bytes: Family<OperationLabels, Histogram, HistogramFactory>,
+    http_response_bytes_rate: Family<OperationLabels, Histogram, HistogramFactory>,
+    http_response_duration_seconds: Family<OperationLabels, Histogram, HistogramFactory>,
+    http_connection_errors_total: Family<OperationLabels, Counter>,
+    http_status_errors_total: Family<OperationLabels, Counter>,
+
+    disable_label_root: bool,
+}
+
+impl Register for FastmetricsInterceptor {
+    fn register(&self, registry: &mut Registry) -> Result<(), RegistryError> {
+        macro_rules! register_metrics {
+            ($($field:ident => $value:expr),* $(,)?) => {
+                $(
+                    {
+                        let ((name, unit), help) = ($value.name_with_unit(), $value.help());
+                        registry.register_metric(name, help, unit, self.$field.clone())?;
+                    }
+                )*
+            };
+        }
+
+        register_metrics! {
+            // Operation metrics
+            operation_bytes => observe::MetricValue::OperationBytes(0),
+            operation_bytes_rate => observe::MetricValue::OperationBytesRate(0.0),
+            operation_entries => observe::MetricValue::OperationEntries(0),
+            operation_entries_rate => observe::MetricValue::OperationEntriesRate(0.0),
+            operation_duration_seconds => observe::MetricValue::OperationDurationSeconds(Duration::default()),
+            operation_errors_total => observe::MetricValue::OperationErrorsTotal,
+            operation_executing => observe::MetricValue::OperationExecuting(0),
+            operation_ttfb_seconds => observe::MetricValue::OperationTtfbSeconds(Duration::default()),
+
+            // HTTP metrics
+            http_executing => observe::MetricValue::HttpExecuting(0),
+            http_request_bytes => observe::MetricValue::HttpRequestBytes(0),
+            http_request_bytes_rate => observe::MetricValue::HttpRequestBytesRate(0.0),
+            http_request_duration_seconds => observe::MetricValue::HttpRequestDurationSeconds(Duration::default()),
+            http_response_bytes => observe::MetricValue::HttpResponseBytes(0),
+            http_response_bytes_rate => observe::MetricValue::HttpResponseBytesRate(0.0),
+            http_response_duration_seconds => observe::MetricValue::HttpResponseDurationSeconds(Duration::default()),
+            http_connection_errors_total => observe::MetricValue::HttpConnectionErrorsTotal,
+            http_status_errors_total => observe::MetricValue::HttpStatusErrorsTotal,
+        }
+
+        Ok(())
+    }
+}
+
+impl observe::MetricsIntercept for FastmetricsInterceptor {
+    fn observe(&self, labels: observe::MetricLabels, value: observe::MetricValue) {
+        let labels = OperationLabels {
+            labels,
+            disable_label_root: self.disable_label_root,
+        };
+        match value {
+            observe::MetricValue::OperationBytes(v) => {
+                self.operation_bytes
+                    .with_or_new(&labels, |hist| hist.observe(v as f64));
+            }
+            observe::MetricValue::OperationBytesRate(v) => {
+                self.operation_bytes_rate
+                    .with_or_new(&labels, |hist| hist.observe(v));
+            }
+            observe::MetricValue::OperationEntries(v) => {
+                self.operation_entries
+                    .with_or_new(&labels, |hist| hist.observe(v as f64));
+            }
+            observe::MetricValue::OperationEntriesRate(v) => {
+                self.operation_entries_rate
+                    .with_or_new(&labels, |hist| hist.observe(v));
+            }
+            observe::MetricValue::OperationDurationSeconds(v) => {
+                self.operation_duration_seconds
+                    .with_or_new(&labels, |hist| hist.observe(v.as_secs_f64()));
+            }
+            observe::MetricValue::OperationErrorsTotal => {
+                self.operation_errors_total
+                    .with_or_new(&labels, |counter| counter.inc());
+            }
+            observe::MetricValue::OperationExecuting(v) => {
+                self.operation_executing
+                    .with_or_new(&labels, |gauge| gauge.inc_by(v as i64));
+            }
+            observe::MetricValue::OperationTtfbSeconds(v) => {
+                self.operation_ttfb_seconds
+                    .with_or_new(&labels, |hist| hist.observe(v.as_secs_f64()));
+            }
+
+            observe::MetricValue::HttpExecuting(v) => {
+                self.http_executing
+                    .with_or_new(&labels, |gauge| gauge.inc_by(v as i64));
+            }
+            observe::MetricValue::HttpRequestBytes(v) => {
+                self.http_request_bytes
+                    .with_or_new(&labels, |hist| hist.observe(v as f64));
+            }
+            observe::MetricValue::HttpRequestBytesRate(v) => {
+                self.http_request_bytes_rate
+                    .with_or_new(&labels, |hist| hist.observe(v));
+            }
+            observe::MetricValue::HttpRequestDurationSeconds(v) => {
+                self.http_request_duration_seconds
+                    .with_or_new(&labels, |hist| hist.observe(v.as_secs_f64()));
+            }
+            observe::MetricValue::HttpResponseBytes(v) => {
+                self.http_response_bytes
+                    .with_or_new(&labels, |hist| hist.observe(v as f64));
+            }
+            observe::MetricValue::HttpResponseBytesRate(v) => {
+                self.http_response_bytes_rate
+                    .with_or_new(&labels, |hist| hist.observe(v));
+            }
+            observe::MetricValue::HttpResponseDurationSeconds(v) => {
+                self.http_response_duration_seconds
+                    .with_or_new(&labels, |hist| hist.observe(v.as_secs_f64()));
+            }
+            observe::MetricValue::HttpConnectionErrorsTotal => {
+                self.http_connection_errors_total
+                    .with_or_new(&labels, |counter| counter.inc());
+            }
+            observe::MetricValue::HttpStatusErrorsTotal => {
+                self.http_status_errors_total
+                    .with_or_new(&labels, |counter| counter.inc());
+            }
+        };
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+struct OperationLabels {
+    labels: observe::MetricLabels,
+    disable_label_root: bool,
+}
+
+impl EncodeLabelSet for OperationLabels {
+    fn encode(&self, encoder: &mut dyn LabelSetEncoder) -> fmt::Result {
+        encoder.encode(&(observe::LABEL_SCHEME, self.labels.scheme.into_static()))?;
+        encoder.encode(&(observe::LABEL_NAMESPACE, self.labels.namespace.as_ref()))?;
+        if !self.disable_label_root {
+            encoder.encode(&(observe::LABEL_ROOT, self.labels.root.as_ref()))?;
+        }
+        encoder.encode(&(observe::LABEL_OPERATION, self.labels.operation))?;
+        if let Some(error) = &self.labels.error {
+            encoder.encode(&(observe::LABEL_ERROR, error.into_static()))?;
+        }
+        if let Some(code) = &self.labels.status_code {
+            encoder.encode(&(observe::LABEL_STATUS_CODE, code.as_str()))?;
+        }
+        Ok(())
+    }
+}
diff --git a/core/src/layers/mod.rs b/core/src/layers/mod.rs
index a4adf9161..dc8d79c29 100644
--- a/core/src/layers/mod.rs
+++ b/core/src/layers/mod.rs
@@ -68,6 +68,13 @@ pub use self::prometheus_client::PrometheusClientLayer;
 #[cfg(feature = "layers-prometheus-client")]
 pub use self::prometheus_client::PrometheusClientLayerBuilder;
 
+#[cfg(feature = "layers-fastmetrics")]
+mod fastmetrics;
+#[cfg(feature = "layers-fastmetrics")]
+pub use self::fastmetrics::FastmetricsLayer;
+#[cfg(feature = "layers-fastmetrics")]
+pub use self::fastmetrics::FastmetricsLayerBuilder;
+
 mod retry;
 pub use self::retry::RetryInterceptor;
 pub use self::retry::RetryLayer;