You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dubbo.apache.org by zh...@apache.org on 2026/04/30 07:39:39 UTC
(dubbo-kubernetes) branch master updated: Added observability and e2e testing (#899)
This is an automated email from the ASF dual-hosted git repository.
mfordjody pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new f7346507 Added observability and e2e testing (#899)
f7346507 is described below
commit f7346507f3a0145f9f8fa7ce8657b3020becbacc
Author: Joe Zhong <11...@qq.com>
AuthorDate: Thu Apr 30 15:39:34 2026 +0800
Added observability and e2e testing (#899)
---
.../charts/dubbod/templates/service.yaml | 7 +
dubbooperator/pkg/render/manifest_test.go | 49 +-
pkg/log/log.go | 10 +
pkg/log/log_test.go | 33 +
pkg/monitoring/monitoring.go | 36 +-
.../monitoring/monitoring_test.go | 37 +-
.../addons/00-namespace.yaml | 39 +-
samples/addons/README.md | 38 +
samples/addons/grafana.yaml | 931 +++++++++++++++++++++
samples/addons/prometheus.yaml | 132 +++
tests/e2e/dubbod-observability.sh | 100 +++
11 files changed, 1347 insertions(+), 65 deletions(-)
diff --git a/dubboinstaller/charts/dubbod/templates/service.yaml b/dubboinstaller/charts/dubbod/templates/service.yaml
index f462d01f..141ec006 100644
--- a/dubboinstaller/charts/dubbod/templates/service.yaml
+++ b/dubboinstaller/charts/dubbod/templates/service.yaml
@@ -20,8 +20,15 @@ metadata:
namespace: dubbo-system
labels:
app: dubbod
+ annotations:
+ prometheus.io/path: /metrics
+ prometheus.io/port: "8080"
+ prometheus.io/scrape: "true"
spec:
ports:
+ - port: 8080
+ name: http-monitoring
+ protocol: TCP
- port: 26010
name: grpc-xds
protocol: TCP
diff --git a/dubbooperator/pkg/render/manifest_test.go b/dubbooperator/pkg/render/manifest_test.go
index 3d5f4bf3..e988e151 100644
--- a/dubbooperator/pkg/render/manifest_test.go
+++ b/dubbooperator/pkg/render/manifest_test.go
@@ -15,7 +15,11 @@
package render
-import "testing"
+import (
+ "testing"
+
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
func TestGenerateManifestUsesEmbeddedInstallerAssets(t *testing.T) {
manifests, _, err := GenerateManifest(nil, nil, nil, nil)
@@ -26,3 +30,46 @@ func TestGenerateManifestUsesEmbeddedInstallerAssets(t *testing.T) {
t.Fatal("GenerateManifest() returned zero manifest sets")
}
}
+
+func TestGenerateManifestExposesDubbodPrometheusScrapeEndpoint(t *testing.T) {
+ manifests, _, err := GenerateManifest(nil, nil, nil, nil)
+ if err != nil {
+ t.Fatalf("GenerateManifest() error = %v", err)
+ }
+
+ for _, set := range manifests {
+ for _, m := range set.Manifests {
+ if m.GetKind() != "Service" || m.GetName() != "dubbod" || m.GetNamespace() != "dubbo-system" {
+ continue
+ }
+ annotations := m.GetAnnotations()
+ if annotations["prometheus.io/scrape"] != "true" {
+ t.Fatalf("prometheus.io/scrape = %q, want true", annotations["prometheus.io/scrape"])
+ }
+ if annotations["prometheus.io/path"] != "/metrics" {
+ t.Fatalf("prometheus.io/path = %q, want /metrics", annotations["prometheus.io/path"])
+ }
+ if annotations["prometheus.io/port"] != "8080" {
+ t.Fatalf("prometheus.io/port = %q, want 8080", annotations["prometheus.io/port"])
+ }
+
+ ports, ok, err := unstructured.NestedSlice(m.Object, "spec", "ports")
+ if err != nil || !ok {
+ t.Fatalf("spec.ports missing: ok=%v err=%v", ok, err)
+ }
+ for _, port := range ports {
+ p, ok := port.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ name, _, _ := unstructured.NestedString(p, "name")
+ number, _, _ := unstructured.NestedInt64(p, "port")
+ if name == "http-monitoring" && number == 8080 {
+ return
+ }
+ }
+ t.Fatal("dubbod Service missing http-monitoring port 8080")
+ }
+ }
+ t.Fatal("dubbod Service not rendered")
+}
diff --git a/pkg/log/log.go b/pkg/log/log.go
index 549874c0..7aa47a9b 100644
--- a/pkg/log/log.go
+++ b/pkg/log/log.go
@@ -24,6 +24,8 @@ import (
"strings"
"sync"
"time"
+
+ "github.com/apache/dubbo-kubernetes/pkg/monitoring"
)
// Level represents the logging level
@@ -147,6 +149,13 @@ var (
defaultScope = "default"
defaultLogger *Logger
defaultLoggerOnce sync.Once
+ levelTag = monitoring.CreateLabel("level")
+ scopeTag = monitoring.CreateLabel("scope")
+ logMessages = monitoring.NewSum(
+ "dubbod_log_messages_total",
+ "Total log messages emitted by level and scope.",
+ monitoring.WithLabels("level", "scope"),
+ )
)
// RegisterScope creates and registers a new logging scope
@@ -257,6 +266,7 @@ func (l *Logger) logMessage(level Level, msg string, preserveNewlines bool) {
levelName := levelNames[level]
scopeName := l.scope.Name()
+ logMessages.With(levelTag.Value(levelName), scopeTag.Value(scopeName)).Increment()
// Check deduplication
dedup := GetDeduplicator()
diff --git a/pkg/log/log_test.go b/pkg/log/log_test.go
index 7c52db0f..470a6ac6 100644
--- a/pkg/log/log_test.go
+++ b/pkg/log/log_test.go
@@ -21,6 +21,8 @@ import (
"encoding/json"
"strings"
"testing"
+
+ "github.com/apache/dubbo-kubernetes/pkg/monitoring"
)
func TestInfoJSONWritesIndentedJSONBlock(t *testing.T) {
@@ -55,3 +57,34 @@ func TestInfoJSONWritesIndentedJSONBlock(t *testing.T) {
t.Fatalf("pushVersion = %#v, want v1", parsed["pushVersion"])
}
}
+
+func TestLogMessagesMetricRecordsLevelAndScope(t *testing.T) {
+ var out bytes.Buffer
+ logger := RegisterScope("log-metric-test", "log metric test")
+ logger.Scope().SetOutput(&out)
+
+ logger.Warn("metric event")
+
+ families, err := monitoring.GetRegistry().Gather()
+ if err != nil {
+ t.Fatalf("gather metrics: %v", err)
+ }
+ for _, family := range families {
+ if family.GetName() != "dubbod_log_messages_total" {
+ continue
+ }
+ for _, metric := range family.GetMetric() {
+ labels := map[string]string{}
+ for _, label := range metric.GetLabel() {
+ labels[label.GetName()] = label.GetValue()
+ }
+ if labels["level"] == "warn" && labels["scope"] == "log-metric-test" {
+ if metric.GetGauge().GetValue() < 1 {
+ t.Fatalf("dubbod_log_messages_total = %v, want >= 1", metric.GetGauge().GetValue())
+ }
+ return
+ }
+ }
+ }
+ t.Fatal("dubbod_log_messages_total with level=warn scope=log-metric-test not found")
+}
diff --git a/pkg/monitoring/monitoring.go b/pkg/monitoring/monitoring.go
index 82b5e0e9..35603243 100644
--- a/pkg/monitoring/monitoring.go
+++ b/pkg/monitoring/monitoring.go
@@ -334,6 +334,11 @@ type distribution struct {
}
func newDistribution(name, description string, bounds []float64, opts ...Options) Metric {
+ o := &options{}
+ for _, opt := range opts {
+ opt(o)
+ }
+
d := &distribution{
name: name,
description: description,
@@ -345,13 +350,19 @@ func newDistribution(name, description string, bounds []float64, opts ...Options
Name: name,
Help: description,
Buckets: bounds,
- }, []string{})
+ }, o.labels)
registryLock.Lock()
registry.MustRegister(d.histogram)
registryLock.Unlock()
})
+ if len(o.labels) > 0 {
+ return &distributionMetric{
+ parent: d,
+ }
+ }
+
return &distributionMetric{
parent: d,
histogram: d.histogram.WithLabelValues(),
@@ -377,6 +388,9 @@ func (dm *distributionMetric) Name() string {
}
func (dm *distributionMetric) Record(value float64) {
+ if dm.histogram == nil {
+ return
+ }
dm.histogram.Observe(value)
}
@@ -389,29 +403,11 @@ func (dm *distributionMetric) With(labelValues ...LabelValue) Metric {
return dm
}
- // Create new histogram vec with labels if needed
- labelNames := make([]string, len(labelValues))
labels := make(prometheus.Labels)
- for i, lv := range labelValues {
- labelNames[i] = lv.Name
+ for _, lv := range labelValues {
labels[lv.Name] = lv.Value
}
- // Check if we need to recreate the histogram with labels
- if dm.labels == nil {
- newHist := prometheus.NewHistogramVec(prometheus.HistogramOpts{
- Name: dm.parent.name,
- Help: dm.parent.description,
- Buckets: dm.parent.bounds,
- }, labelNames)
-
- registryLock.Lock()
- registry.MustRegister(newHist)
- registryLock.Unlock()
-
- dm.parent.histogram = newHist
- }
-
return &distributionMetric{
parent: dm.parent,
histogram: dm.parent.histogram.With(labels),
diff --git a/dubbooperator/pkg/render/manifest_test.go b/pkg/monitoring/monitoring_test.go
similarity index 51%
copy from dubbooperator/pkg/render/manifest_test.go
copy to pkg/monitoring/monitoring_test.go
index 3d5f4bf3..91bf05d2 100644
--- a/dubbooperator/pkg/render/manifest_test.go
+++ b/pkg/monitoring/monitoring_test.go
@@ -1,3 +1,4 @@
+//
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
@@ -13,16 +14,38 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package render
+package monitoring
+
+import (
+ "fmt"
+ "testing"
+ "time"
+)
+
+func TestDistributionWithLabelsRegistersOnce(t *testing.T) {
+ phaseTag := CreateLabel("phase")
+ name := fmt.Sprintf("test_distribution_with_labels_seconds_%d", time.Now().UnixNano())
+ metric := NewDistribution(
+ name,
+ "Test labeled distribution registration.",
+ []float64{0.1, 1},
+ WithLabels("phase"),
+ )
-import "testing"
+ metric.With(phaseTag.Value("push")).Record(0.2)
+ metric.With(phaseTag.Value("send")).Record(0.4)
-func TestGenerateManifestUsesEmbeddedInstallerAssets(t *testing.T) {
- manifests, _, err := GenerateManifest(nil, nil, nil, nil)
+ families, err := GetRegistry().Gather()
if err != nil {
- t.Fatalf("GenerateManifest() error = %v", err)
+ t.Fatalf("gather metrics: %v", err)
}
- if len(manifests) == 0 {
- t.Fatal("GenerateManifest() returned zero manifest sets")
+ for _, family := range families {
+ if family.GetName() == name {
+ if got := len(family.GetMetric()); got != 2 {
+ t.Fatalf("metric series = %d, want 2", got)
+ }
+ return
+ }
}
+ t.Fatalf("%s not gathered", name)
}
diff --git a/dubboinstaller/charts/dubbod/templates/service.yaml b/samples/addons/00-namespace.yaml
similarity index 57%
copy from dubboinstaller/charts/dubbod/templates/service.yaml
copy to samples/addons/00-namespace.yaml
index f462d01f..42e674c6 100644
--- a/dubboinstaller/charts/dubbod/templates/service.yaml
+++ b/samples/addons/00-namespace.yaml
@@ -14,41 +14,6 @@
# limitations under the License.
apiVersion: v1
-kind: Service
+kind: Namespace
metadata:
- name: dubbod
- namespace: dubbo-system
- labels:
- app: dubbod
-spec:
- ports:
- - port: 26010
- name: grpc-xds
- protocol: TCP
- - port: 26012
- name: https-dns
- protocol: TCP
- - port: 443
- name: https-webhook
- targetPort: 26017
- protocol: TCP
- selector:
- app: dubbod
----
-apiVersion: v1
-kind: Service
-metadata:
- name: dubbod-gui
- namespace: dubbo-system
- labels:
- app: dubbod
-spec:
- type: NodePort
- ports:
- - port: 26080
- name: gui
- targetPort: 26080
- nodePort: 30080
- protocol: TCP
- selector:
- app: dubbod
+ name: dubbo-system
diff --git a/samples/addons/README.md b/samples/addons/README.md
new file mode 100644
index 00000000..c1538a72
--- /dev/null
+++ b/samples/addons/README.md
@@ -0,0 +1,38 @@
+# Dubbo 可观测性 Addons
+
+这个目录提供开箱即用的 Prometheus 和 Grafana 部署，用于观察 `dubbo-system/dubbod` 的日志信号、监控指标和控制面追踪耗时。
+
+## 安装
+
+```bash
+kubectl apply -f samples/addons
+kubectl -n dubbo-system rollout status deploy/prometheus --timeout=180s
+kubectl -n dubbo-system rollout status deploy/grafana --timeout=180s
+```
+
+## 打开 Grafana
+
+```bash
+kubectl -n dubbo-system port-forward svc/grafana 3000:3000
+```
+
+浏览器打开 `http://127.0.0.1:3000`，进入 `Dubbo / Dubbo Control Plane Observability` 仪表盘。
+
+## 验证 Prometheus
+
+```bash
+kubectl -n dubbo-system port-forward svc/prometheus 9090:9090
+curl 'http://127.0.0.1:9090/api/v1/query?query=dubbod_uptime_seconds'
+```
+
+## 覆盖内容
+
+- 日志：`dubbod_log_messages_total` 按 `level` 和 `scope` 统计日志速率，原始日志仍通过 `kubectl -n dubbo-system logs deploy/dubbod` 查看。
+- 监控：Prometheus 抓取 `dubbod.dubbo-system.svc:8080/metrics`，Grafana 内置控制面总览面板。
+- 追踪：通过 `dubbod_xds_push_time`、`dubbod_xds_send_time`、`dubbod_proxy_queue_time` 等直方图展示 xDS 推送链路耗时。
+
+## 清理
+
+```bash
+kubectl delete -f samples/addons --ignore-not-found=true
+```
diff --git a/samples/addons/grafana.yaml b/samples/addons/grafana.yaml
new file mode 100644
index 00000000..463c0414
--- /dev/null
+++ b/samples/addons/grafana.yaml
@@ -0,0 +1,931 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: grafana
+ namespace: dubbo-system
+ labels:
+ app.kubernetes.io/name: grafana
+ app.kubernetes.io/part-of: dubbo-observability
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: grafana
+ namespace: dubbo-system
+ labels:
+ app.kubernetes.io/name: grafana
+ app.kubernetes.io/part-of: dubbo-observability
+data:
+ grafana.ini: |
+ [analytics]
+ check_for_updates = false
+ [log]
+ mode = console
+ [paths]
+ data = /var/lib/grafana
+ logs = /var/log/grafana
+ plugins = /var/lib/grafana/plugins
+ provisioning = /etc/grafana/provisioning
+ datasources.yaml: |
+ apiVersion: 1
+ datasources:
+ - name: Prometheus
+ uid: prometheus
+ type: prometheus
+ access: proxy
+ orgId: 1
+ url: http://prometheus.dubbo-system.svc:9090
+ isDefault: true
+ editable: true
+ jsonData:
+ timeInterval: 15s
+ dashboardproviders.yaml: |
+ apiVersion: 1
+ providers:
+ - name: dubbo
+ orgId: 1
+ folder: Dubbo
+ type: file
+ disableDeletion: false
+ editable: true
+ options:
+ path: /var/lib/grafana/dashboards/dubbo
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: dubbod-grafana-dashboards
+ namespace: dubbo-system
+ labels:
+ app.kubernetes.io/name: grafana
+ app.kubernetes.io/part-of: dubbo-observability
+data:
+ dubbod-observability.json: |
+ {
+ "annotations": {
+ "list": []
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 0,
+ "y": 0
+ },
+ "id": 1,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "targets": [
+ {
+ "expr": "max(dubbod_uptime_seconds)",
+ "refId": "A"
+ }
+ ],
+ "title": "Uptime",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 6,
+ "y": 0
+ },
+ "id": 2,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "targets": [
+ {
+ "expr": "sum(dubbod_xds)",
+ "refId": "A"
+ }
+ ],
+ "title": "XDS Clients",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 12,
+ "y": 0
+ },
+ "id": 3,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "targets": [
+ {
+ "expr": "max(dubbod_services)",
+ "refId": "A"
+ }
+ ],
+ "title": "Services",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 3,
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 18,
+ "y": 0
+ },
+ "id": 4,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "targets": [
+ {
+ "expr": "sum(rate(dubbod_log_messages_total{level=~\"error|warn\"}[5m]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Warn/Error Log Rate",
+ "type": "stat"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 4
+ },
+ "id": 5,
+ "panels": [],
+ "title": "Logs",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 5
+ },
+ "id": 6,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "sum by (level) (rate(dubbod_log_messages_total[5m]))",
+ "legendFormat": "{{level}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Log Rate By Level",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 5
+ },
+ "id": 7,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "sum by (scope) (rate(dubbod_log_messages_total{level=~\"warn|error\"}[5m]))",
+ "legendFormat": "{{scope}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Warn/Error Logs By Scope",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 13
+ },
+ "id": 8,
+ "panels": [],
+ "title": "Monitoring",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 14
+ },
+ "id": 9,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "sum by (type) (rate(dubbod_xds_pushes[5m]))",
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }
+ ],
+ "title": "XDS Push Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 14
+ },
+ "id": 10,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "sum by (type) (rate(dubbod_push_triggers[5m]))",
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Push Trigger Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 14
+ },
+ "id": 11,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "sum by (type) (rate(dubbod_inbound_updates[5m]))",
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Inbound Updates",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 22
+ },
+ "id": 12,
+ "panels": [],
+ "title": "Tracing",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 23
+ },
+ "id": 13,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le, type) (rate(dubbod_xds_push_time_bucket[5m])))",
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }
+ ],
+ "title": "XDS Push P99",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 23
+ },
+ "id": 14,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(dubbod_xds_send_time_bucket[5m])))",
+ "legendFormat": "send",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(dubbod_proxy_queue_time_bucket[5m])))",
+ "legendFormat": "queue",
+ "refId": "B"
+ }
+ ],
+ "title": "Send And Queue P99",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 23
+ },
+ "id": 15,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(dubbod_debounce_time_bucket[5m])))",
+ "legendFormat": "debounce",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(dubbod_pushcontext_init_seconds_bucket[5m])))",
+ "legendFormat": "push context",
+ "refId": "B"
+ }
+ ],
+ "title": "Control Plane Stage P99",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 31
+ },
+ "id": 16,
+ "panels": [],
+ "title": "Errors",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 32
+ },
+ "id": 17,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "sum(rate(dubbod_total_xds_internal_errors[5m]))",
+ "legendFormat": "xds internal",
+ "refId": "A"
+ },
+ {
+ "expr": "sum(rate(dubbod_xds_write_timeout[5m]))",
+ "legendFormat": "write timeout",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(dubbod_sds_certificate_errors_total[5m]))",
+ "legendFormat": "sds certificate",
+ "refId": "C"
+ }
+ ],
+ "title": "Error Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineWidth": 1,
+ "showPoints": "never"
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 32
+ },
+ "id": 18,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "expr": "sum by (type) (rate(dubbod_total_xds_rejects[5m]))",
+ "legendFormat": "{{type}}",
+ "refId": "A"
+ }
+ ],
+ "title": "XDS Reject Rate",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "15s",
+ "schemaVersion": 39,
+ "tags": [
+ "dubbo",
+ "dubbod",
+ "observability"
+ ],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-30m",
+ "to": "now"
+ },
+ "timezone": "browser",
+ "title": "Dubbo Control Plane Observability",
+ "uid": "dubbod-observability",
+ "version": 1,
+ "weekStart": ""
+ }
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: grafana
+ namespace: dubbo-system
+ labels:
+ app.kubernetes.io/name: grafana
+ app.kubernetes.io/part-of: dubbo-observability
+spec:
+ type: ClusterIP
+ ports:
+ - name: http
+ port: 3000
+ protocol: TCP
+ targetPort: 3000
+ selector:
+ app.kubernetes.io/name: grafana
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: grafana
+ namespace: dubbo-system
+ labels:
+ app.kubernetes.io/name: grafana
+ app.kubernetes.io/part-of: dubbo-observability
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: grafana
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: grafana
+ app.kubernetes.io/part-of: dubbo-observability
+ spec:
+ serviceAccountName: grafana
+ containers:
+ - name: grafana
+ image: docker.io/grafana/grafana:12.0.1
+ imagePullPolicy: IfNotPresent
+ ports:
+ - name: http
+ containerPort: 3000
+ protocol: TCP
+ env:
+ - name: GF_AUTH_ANONYMOUS_ENABLED
+ value: "true"
+ - name: GF_AUTH_ANONYMOUS_ORG_ROLE
+ value: Admin
+ - name: GF_AUTH_BASIC_ENABLED
+ value: "false"
+ - name: GF_SECURITY_ADMIN_USER
+ value: admin
+ - name: GF_SECURITY_ADMIN_PASSWORD
+ value: admin
+ readinessProbe:
+ httpGet:
+ path: /api/health
+ port: 3000
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ livenessProbe:
+ httpGet:
+ path: /api/health
+ port: 3000
+ initialDelaySeconds: 60
+ periodSeconds: 15
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+ limits:
+ memory: 512Mi
+ volumeMounts:
+ - name: config
+ mountPath: /etc/grafana/grafana.ini
+ subPath: grafana.ini
+ readOnly: true
+ - name: config
+ mountPath: /etc/grafana/provisioning/datasources/datasources.yaml
+ subPath: datasources.yaml
+ readOnly: true
+ - name: config
+ mountPath: /etc/grafana/provisioning/dashboards/dashboardproviders.yaml
+ subPath: dashboardproviders.yaml
+ readOnly: true
+ - name: dashboards
+ mountPath: /var/lib/grafana/dashboards/dubbo
+ readOnly: true
+ - name: storage
+ mountPath: /var/lib/grafana
+ volumes:
+ - name: config
+ configMap:
+ name: grafana
+ - name: dashboards
+ configMap:
+ name: dubbod-grafana-dashboards
+ - name: storage
+ emptyDir: {}
diff --git a/samples/addons/prometheus.yaml b/samples/addons/prometheus.yaml
new file mode 100644
index 00000000..7d46e93f
--- /dev/null
+++ b/samples/addons/prometheus.yaml
@@ -0,0 +1,132 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: prometheus
+ namespace: dubbo-system
+ labels:
+ app.kubernetes.io/name: prometheus
+ app.kubernetes.io/part-of: dubbo-observability
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: prometheus
+ namespace: dubbo-system
+ labels:
+ app.kubernetes.io/name: prometheus
+ app.kubernetes.io/part-of: dubbo-observability
+data:
+ prometheus.yml: |
+ global:
+ scrape_interval: 15s
+ evaluation_interval: 15s
+
+ scrape_configs:
+ - job_name: prometheus
+ static_configs:
+ - targets:
+ - localhost:9090
+
+ - job_name: dubbod
+ metrics_path: /metrics
+ static_configs:
+ - targets:
+ - dubbod.dubbo-system.svc:8080
+ labels:
+ app: dubbod
+ namespace: dubbo-system
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: prometheus
+ namespace: dubbo-system
+ labels:
+ app.kubernetes.io/name: prometheus
+ app.kubernetes.io/part-of: dubbo-observability
+spec:
+ ports:
+ - name: http
+ port: 9090
+ protocol: TCP
+ targetPort: 9090
+ selector:
+ app.kubernetes.io/name: prometheus
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: prometheus
+ namespace: dubbo-system
+ labels:
+ app.kubernetes.io/name: prometheus
+ app.kubernetes.io/part-of: dubbo-observability
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: prometheus
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: prometheus
+ app.kubernetes.io/part-of: dubbo-observability
+ spec:
+ serviceAccountName: prometheus
+ containers:
+ - name: prometheus
+ image: prom/prometheus:v3.5.0
+ imagePullPolicy: IfNotPresent
+ args:
+ - --config.file=/etc/prometheus/prometheus.yml
+ - --storage.tsdb.path=/prometheus
+ - --web.enable-lifecycle
+ ports:
+ - name: http
+ containerPort: 9090
+ protocol: TCP
+ readinessProbe:
+ httpGet:
+ path: /-/ready
+ port: 9090
+ initialDelaySeconds: 5
+ periodSeconds: 10
+ livenessProbe:
+ httpGet:
+ path: /-/healthy
+ port: 9090
+ initialDelaySeconds: 30
+ periodSeconds: 15
+ resources:
+ requests:
+ cpu: 100m
+ memory: 256Mi
+ limits:
+ memory: 512Mi
+ volumeMounts:
+ - name: config
+ mountPath: /etc/prometheus
+ readOnly: true
+ - name: storage
+ mountPath: /prometheus
+ volumes:
+ - name: config
+ configMap:
+ name: prometheus
+ - name: storage
+ emptyDir: {}
diff --git a/tests/e2e/dubbod-observability.sh b/tests/e2e/dubbod-observability.sh
new file mode 100755
index 00000000..d3ef0b1d
--- /dev/null
+++ b/tests/e2e/dubbod-observability.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+KUBECTL="${KUBECTL:-kubectl}"
+DUBBOD_NAMESPACE="${DUBBOD_NAMESPACE:-dubbo-system}"
+DUBBOD_SERVICE="${DUBBOD_SERVICE:-dubbod}"
+DUBBOD_DEPLOYMENT="${DUBBOD_DEPLOYMENT:-dubbod}"
+ADDONS_DIR="${ADDONS_DIR:-samples/addons}"
+PROMETHEUS_SERVICE="${PROMETHEUS_SERVICE:-prometheus}"
+GRAFANA_SERVICE="${GRAFANA_SERVICE:-grafana}"
+PROMETHEUS_LOCAL_PORT="${PROMETHEUS_LOCAL_PORT:-19090}"
+PROMETHEUS_QUERY="${PROMETHEUS_QUERY:-up{job=\"dubbod\"}}"
+DRY_RUN_ONLY="${DUBBO_OBSERVABILITY_DRY_RUN_ONLY:-false}"
+
+if [[ ! -d "${ADDONS_DIR}" ]]; then
+ echo "addons directory not found: ${ADDONS_DIR}" >&2
+ exit 1
+fi
+
+"${KUBECTL}" apply --dry-run=client -f "${ADDONS_DIR}" >/dev/null
+
+if [[ "${DRY_RUN_ONLY}" == "true" ]]; then
+ exit 0
+fi
+
+"${KUBECTL}" -n "${DUBBOD_NAMESPACE}" rollout status "deploy/${DUBBOD_DEPLOYMENT}" --timeout=180s
+"${KUBECTL}" apply -f "${ADDONS_DIR}"
+"${KUBECTL}" -n "${DUBBOD_NAMESPACE}" rollout status "deploy/${PROMETHEUS_SERVICE}" --timeout=180s
+"${KUBECTL}" -n "${DUBBOD_NAMESPACE}" rollout status "deploy/${GRAFANA_SERVICE}" --timeout=180s
+"${KUBECTL}" -n "${DUBBOD_NAMESPACE}" get "svc/${PROMETHEUS_SERVICE}" "svc/${GRAFANA_SERVICE}" >/dev/null
+
+scrape_annotation="$("${KUBECTL}" -n "${DUBBOD_NAMESPACE}" get "svc/${DUBBOD_SERVICE}" -o jsonpath='{.metadata.annotations.prometheus\.io/scrape}')"
+metrics_port="$("${KUBECTL}" -n "${DUBBOD_NAMESPACE}" get "svc/${DUBBOD_SERVICE}" -o jsonpath='{.metadata.annotations.prometheus\.io/port}')"
+if [[ "${scrape_annotation}" != "true" || "${metrics_port}" != "8080" ]]; then
+ echo "dubbod service prometheus annotations invalid: scrape=${scrape_annotation} port=${metrics_port}" >&2
+ exit 1
+fi
+
+pf_log="$(mktemp -t dubbod-observability-prometheus.XXXXXX.log)"
+"${KUBECTL}" -n "${DUBBOD_NAMESPACE}" port-forward "svc/${PROMETHEUS_SERVICE}" \
+ "${PROMETHEUS_LOCAL_PORT}:9090" >"${pf_log}" 2>&1 &
+pf_pid="$!"
+
+cleanup() {
+ if kill -0 "${pf_pid}" >/dev/null 2>&1; then
+ kill "${pf_pid}" >/dev/null 2>&1 || true
+ wait "${pf_pid}" >/dev/null 2>&1 || true
+ fi
+ rm -f "${pf_log}"
+}
+trap cleanup EXIT
+
+for _ in $(seq 1 100); do
+ if nc -z 127.0.0.1 "${PROMETHEUS_LOCAL_PORT}" >/dev/null 2>&1; then
+ break
+ fi
+ if ! kill -0 "${pf_pid}" >/dev/null 2>&1; then
+ cat "${pf_log}" >&2
+ exit 1
+ fi
+ sleep 0.1
+done
+
+if ! nc -z 127.0.0.1 "${PROMETHEUS_LOCAL_PORT}" >/dev/null 2>&1; then
+ cat "${pf_log}" >&2
+ echo "prometheus port-forward did not become ready" >&2
+ exit 1
+fi
+
+python3 - "${PROMETHEUS_LOCAL_PORT}" "${PROMETHEUS_QUERY}" <<'PY'
+import json
+import sys
+import urllib.parse
+import urllib.request
+
+port, query = sys.argv[1], sys.argv[2]
+url = f"http://127.0.0.1:{port}/api/v1/query?" + urllib.parse.urlencode({"query": query})
+with urllib.request.urlopen(url, timeout=10) as response:
+ payload = json.load(response)
+
+if payload.get("status") != "success":
+ raise SystemExit(f"prometheus query failed: {payload}")
+if not payload.get("data", {}).get("result"):
+ raise SystemExit(f"prometheus query returned no series: {query}")
+PY