You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2021/04/07 07:38:58 UTC
[apisix] branch master updated: change(prometheus): redesign the
latency metrics & update grafana (#3993)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new b273ae7 change(prometheus): redesign the latency metrics & update grafana (#3993)
b273ae7 is described below
commit b273ae7002c0f47d506d5d68ad99361c4082e9ae
Author: Shuyang Wu <wo...@gmail.com>
AuthorDate: Wed Apr 7 15:38:49 2021 +0800
change(prometheus): redesign the latency metrics & update grafana (#3993)
Co-authored-by: nic-chen <jo...@163.com>
---
apisix/plugins/prometheus/exporter.lua | 16 +-
docs/assets/images/plugin/grafana-1.png | Bin 260106 -> 172664 bytes
docs/assets/images/plugin/grafana-2.png | Bin 211404 -> 166250 bytes
docs/assets/images/plugin/grafana-3.png | Bin 38360 -> 181975 bytes
docs/assets/images/plugin/grafana-4.png | Bin 0 -> 54301 bytes
docs/assets/images/plugin/prometheus-1.png | Bin 67980 -> 12535 bytes
docs/assets/images/plugin/prometheus-2.png | Bin 237119 -> 9077 bytes
.../other/json/apisix-grafana-dashboard.json | 348 +++++++++++----------
docs/en/latest/plugins/prometheus.md | 25 +-
docs/zh/latest/plugins/prometheus.md | 22 +-
t/plugin/prometheus.t | 8 +-
11 files changed, 210 insertions(+), 209 deletions(-)
diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index 603649d..533415c 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -103,11 +103,6 @@ function _M.init()
"HTTP request latency in milliseconds per service in APISIX",
{"type", "service", "consumer", "node"}, DEFAULT_BUCKETS)
- metrics.overhead = prometheus:histogram("http_overhead",
- "HTTP request overhead added by APISIX in milliseconds per service " ..
- "in APISIX",
- {"type", "service", "consumer", "node"}, DEFAULT_BUCKETS)
-
metrics.bandwidth = prometheus:counter("bandwidth",
"Total bandwidth in bytes consumed per service in APISIX",
{"type", "route", "service", "consumer", "node"})
@@ -146,12 +141,15 @@ function _M.log(conf, ctx)
metrics.latency:observe(latency,
gen_arr("request", service_id, consumer_name, balancer_ip))
- local overhead = latency
+ local apisix_latency = latency
if ctx.var.upstream_response_time then
- overhead = overhead - ctx.var.upstream_response_time * 1000
+ local upstream_latency = ctx.var.upstream_response_time * 1000
+ metrics.latency:observe(upstream_latency,
+ gen_arr("upstream", service_id, consumer_name, balancer_ip))
+ apisix_latency = apisix_latency - upstream_latency
end
- metrics.overhead:observe(overhead,
- gen_arr("request", service_id, consumer_name, balancer_ip))
+ metrics.latency:observe(apisix_latency,
+ gen_arr("apisix", service_id, consumer_name, balancer_ip))
metrics.bandwidth:inc(vars.request_length,
gen_arr("ingress", route_id, service_id, consumer_name, balancer_ip))
diff --git a/docs/assets/images/plugin/grafana-1.png b/docs/assets/images/plugin/grafana-1.png
index 1f65ab5..7c68f70 100644
Binary files a/docs/assets/images/plugin/grafana-1.png and b/docs/assets/images/plugin/grafana-1.png differ
diff --git a/docs/assets/images/plugin/grafana-2.png b/docs/assets/images/plugin/grafana-2.png
index 2db20d1..8446e7a 100644
Binary files a/docs/assets/images/plugin/grafana-2.png and b/docs/assets/images/plugin/grafana-2.png differ
diff --git a/docs/assets/images/plugin/grafana-3.png b/docs/assets/images/plugin/grafana-3.png
index 90304f1..e8673c2 100644
Binary files a/docs/assets/images/plugin/grafana-3.png and b/docs/assets/images/plugin/grafana-3.png differ
diff --git a/docs/assets/images/plugin/grafana-4.png b/docs/assets/images/plugin/grafana-4.png
new file mode 100644
index 0000000..b07c62d
Binary files /dev/null and b/docs/assets/images/plugin/grafana-4.png differ
diff --git a/docs/assets/images/plugin/prometheus-1.png b/docs/assets/images/plugin/prometheus-1.png
index 6e3c8b6..4303e96 100644
Binary files a/docs/assets/images/plugin/prometheus-1.png and b/docs/assets/images/plugin/prometheus-1.png differ
diff --git a/docs/assets/images/plugin/prometheus-2.png b/docs/assets/images/plugin/prometheus-2.png
index f48350b..b99fb95 100644
Binary files a/docs/assets/images/plugin/prometheus-2.png and b/docs/assets/images/plugin/prometheus-2.png differ
diff --git a/docs/assets/other/json/apisix-grafana-dashboard.json b/docs/assets/other/json/apisix-grafana-dashboard.json
index 191a21c..217ab9f 100644
--- a/docs/assets/other/json/apisix-grafana-dashboard.json
+++ b/docs/assets/other/json/apisix-grafana-dashboard.json
@@ -61,7 +61,7 @@
"gnetId": 11719,
"graphTooltip": 0,
"id": null,
- "iteration": 1616655621009,
+ "iteration": 1617695812393,
"links": [],
"panels": [
{
@@ -498,7 +498,9 @@
"min": true,
"rightSide": true,
"show": true,
- "total": false,
+ "sort": "total",
+ "sortDesc": true,
+ "total": true,
"values": true
},
"lines": true,
@@ -582,20 +584,24 @@
"fillGradient": 0,
"gridPos": {
"h": 6,
- "w": 12,
+ "w": 24,
"x": 0,
"y": 19
},
"hiddenSeries": false,
- "id": 19,
+ "id": 21,
"legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
"show": true,
- "total": false,
- "values": false
+ "sort": "total",
+ "sortDesc": true,
+ "total": true,
+ "values": true
},
"lines": true,
"linewidth": 1,
@@ -614,12 +620,12 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(rate(apisix_bandwidth{type=\"egress\", service =~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (service)",
+ "expr": "sum(irate(apisix_bandwidth{type=\"ingress\", service =~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (service)",
"legendFormat": "service:{{service}}",
"refId": "A"
},
{
- "expr": "sum(rate(apisix_bandwidth{type=\"egress\", service =~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (route)",
+ "expr": "sum(irate(apisix_bandwidth{type=\"ingress\", service =~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (route)",
"legendFormat": "route:{{route}}",
"refId": "B"
}
@@ -628,7 +634,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Egress per service/route",
+ "title": "Ingress per service/route",
"tooltip": {
"shared": true,
"sort": 0,
@@ -683,20 +689,24 @@
"fillGradient": 0,
"gridPos": {
"h": 6,
- "w": 12,
- "x": 12,
- "y": 19
+ "w": 24,
+ "x": 0,
+ "y": 25
},
"hiddenSeries": false,
- "id": 21,
+ "id": 19,
"legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
"show": true,
- "total": false,
- "values": false
+ "sort": "total",
+ "sortDesc": true,
+ "total": true,
+ "values": true
},
"lines": true,
"linewidth": 1,
@@ -715,12 +725,13 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(irate(apisix_bandwidth{type=\"ingress\", service =~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (service)",
+ "expr": "sum(rate(apisix_bandwidth{type=\"egress\", service =~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (service)",
+ "interval": "",
"legendFormat": "service:{{service}}",
"refId": "A"
},
{
- "expr": "sum(irate(apisix_bandwidth{type=\"ingress\", service =~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (route)",
+ "expr": "sum(rate(apisix_bandwidth{type=\"egress\", service =~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (route)",
"legendFormat": "route:{{route}}",
"refId": "B"
}
@@ -729,7 +740,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Ingress per service/route",
+ "title": "Egress per service/route",
"tooltip": {
"shared": true,
"sort": 0,
@@ -774,7 +785,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 25
+ "y": 31
},
"id": 15,
"panels": [],
@@ -782,7 +793,10 @@
"type": "row"
},
{
- "aliasColors": {},
+ "aliasColors": {
+ "HTTP Status:200": "green",
+ "HTTP Status:500": "red"
+ },
"bars": false,
"cacheTimeout": null,
"dashLength": 10,
@@ -802,7 +816,7 @@
"h": 6,
"w": 24,
"x": 0,
- "y": 26
+ "y": 32
},
"hiddenSeries": false,
"id": 2,
@@ -839,38 +853,19 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(rate(apisix_http_status{code=~\"2..\",service=~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (status_2xx)",
+ "expr": "sum(rate(apisix_http_status{service=~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (code)",
"instant": false,
"interval": "",
"intervalFactor": 1,
- "legendFormat": "status_2xx",
+ "legendFormat": "HTTP Status:{{code}}",
"refId": "A"
- },
- {
- "expr": "sum(rate(apisix_http_status{code=~\"3..\",service=~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (status_3xx)",
- "interval": "",
- "legendFormat": "status_3xx",
- "refId": "D"
- },
- {
- "expr": "sum(rate(apisix_http_status{code=~\"4..\",service=~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (status_4xx)",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "status_4xx",
- "refId": "B"
- },
- {
- "expr": "sum(rate(apisix_http_status{code=~\"5..\",service=~\"$service\",route=~\"$route\",instance=~\"$instance\"}[1m])) by (status_5xx)",
- "interval": "",
- "legendFormat": "status_5xx",
- "refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Requests per second (RPS)",
+ "title": "Requests per second (RPS) by status code",
"tooltip": {
"shared": true,
"sort": 0,
@@ -929,19 +924,23 @@
"h": 6,
"w": 24,
"x": 0,
- "y": 32
+ "y": 38
},
"hiddenSeries": false,
"id": 32,
"interval": "",
"legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": true,
"show": true,
- "total": false,
- "values": false
+ "sort": "total",
+ "sortDesc": true,
+ "total": true,
+ "values": true
},
"lines": true,
"linewidth": 1,
@@ -1052,9 +1051,9 @@
"fillGradient": 0,
"gridPos": {
"h": 6,
- "w": 24,
+ "w": 8,
"x": 0,
- "y": 38
+ "y": 44
},
"hiddenSeries": false,
"id": 27,
@@ -1084,20 +1083,20 @@
"steppedLine": false,
"targets": [
{
- "expr": "histogram_quantile(0.90, sum(rate(apisix_http_latency_bucket{service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
+ "expr": "histogram_quantile(0.90, sum(rate(apisix_http_latency_bucket{type=~\"request\",service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
"format": "time_series",
"interval": "",
"legendFormat": "P90",
"refId": "A"
},
{
- "expr": "histogram_quantile(0.95, sum(rate(apisix_http_latency_bucket{service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
+ "expr": "histogram_quantile(0.95, sum(rate(apisix_http_latency_bucket{type=~\"request\",service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
"interval": "",
"legendFormat": "P95",
"refId": "B"
},
{
- "expr": "histogram_quantile(0.99, sum(rate(apisix_http_latency_bucket{service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
+ "expr": "histogram_quantile(0.99, sum(rate(apisix_http_latency_bucket{type=~\"request\",service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
"interval": "",
"legendFormat": "P99",
"refId": "C"
@@ -1107,7 +1106,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "HTTP Latency",
+ "title": "Request Latency",
"tooltip": {
"shared": true,
"sort": 0,
@@ -1175,8 +1174,8 @@
"fillGradient": 0,
"gridPos": {
"h": 6,
- "w": 24,
- "x": 0,
+ "w": 8,
+ "x": 8,
"y": 44
},
"hiddenSeries": false,
@@ -1207,7 +1206,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "histogram_quantile(0.90, sum(rate(apisix_http_overhead_bucket{service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
+ "expr": "histogram_quantile(0.90, sum(rate(apisix_http_latency_bucket{type=~\"apisix\",service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
"format": "time_series",
"instant": false,
"interval": "",
@@ -1216,13 +1215,13 @@
"refId": "A"
},
{
- "expr": "histogram_quantile(0.95, sum(rate(apisix_http_overhead_bucket{service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
+ "expr": "histogram_quantile(0.95, sum(rate(apisix_http_latency_bucket{type=~\"apisix\",service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
"interval": "",
"legendFormat": "P95",
"refId": "B"
},
{
- "expr": "histogram_quantile(0.99, sum(rate(apisix_http_overhead_bucket{service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
+ "expr": "histogram_quantile(0.99, sum(rate(apisix_http_latency_bucket{type=~\"apisix\",service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
"interval": "",
"legendFormat": "P99",
"refId": "C"
@@ -1232,7 +1231,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "HTTP Overhead",
+ "title": "APISIX Latency",
"tooltip": {
"shared": true,
"sort": 0,
@@ -1270,20 +1269,10 @@
}
},
{
- "collapsed": false,
- "datasource": "${DS_PROMETHEUS}",
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 50
- },
- "id": 23,
- "panels": [],
- "title": "Misc",
- "type": "row"
- },
- {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
@@ -1298,82 +1287,28 @@
},
{
"color": "red",
- "value": 0
- },
- {
- "color": "green",
- "value": 1
+ "value": 80
}
]
}
},
"overrides": []
},
- "gridPos": {
- "h": 3,
- "w": 4,
- "x": 0,
- "y": 51
- },
- "id": 25,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "reduceOptions": {
- "calcs": [
- "last"
- ],
- "fields": "",
- "values": false
- },
- "textMode": "auto"
- },
- "pluginVersion": "7.3.7",
- "targets": [
- {
- "expr": "sum(apisix_etcd_reachable{instance=~\"$instance\"})",
- "interval": "",
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Etcd reachable",
- "type": "stat"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "custom": {},
- "links": []
- },
- "overrides": []
- },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 6,
- "w": 10,
- "x": 4,
- "y": 51
+ "w": 8,
+ "x": 16,
+ "y": 44
},
"hiddenSeries": false,
- "id": 30,
+ "id": 33,
"legend": {
- "alignAsTable": false,
"avg": false,
"current": false,
"max": false,
"min": false,
- "rightSide": false,
"show": true,
"total": false,
"values": false
@@ -1395,18 +1330,30 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(apisix_etcd_modify_indexes{key=~\"consumers|global_rules|max_modify_index|prev_index|protos|routes|services|ssls|stream_routes|upstreams|x_etcd_index\"}) by (key)",
+ "expr": "histogram_quantile(0.90, sum(rate(apisix_http_latency_bucket{type=~\"upstream\",service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
+ "format": "time_series",
"interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{key}}",
+ "legendFormat": "P90",
"refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.95, sum(rate(apisix_http_latency_bucket{type=~\"upstream\",service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
+ "interval": "",
+ "legendFormat": "P95",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(apisix_http_latency_bucket{type=~\"upstream\",service=~\"$service\",consumer=~\"$consumer\",node=~\"$node\"}[1m])) by (le))",
+ "interval": "",
+ "legendFormat": "P99",
+ "refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Etcd modify indexes",
+ "title": "Upstream Latency",
"tooltip": {
"shared": true,
"sort": 0,
@@ -1422,8 +1369,7 @@
},
"yaxes": [
{
- "decimals": 0,
- "format": "short",
+ "format": "ms",
"label": "",
"logBase": 1,
"max": null,
@@ -1431,9 +1377,8 @@
"show": true
},
{
- "decimals": null,
"format": "short",
- "label": "",
+ "label": null,
"logBase": 1,
"max": null,
"min": null,
@@ -1446,6 +1391,20 @@
}
},
{
+ "collapsed": false,
+ "datasource": "${DS_PROMETHEUS}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 50
+ },
+ "id": 23,
+ "panels": [],
+ "title": "Misc",
+ "type": "row"
+ },
+ {
"aliasColors": {},
"bars": false,
"dashLength": 10,
@@ -1462,12 +1421,12 @@
"fillGradient": 0,
"gridPos": {
"h": 6,
- "w": 10,
- "x": 14,
+ "w": 18,
+ "x": 0,
"y": 51
},
"hiddenSeries": false,
- "id": 31,
+ "id": 30,
"legend": {
"alignAsTable": false,
"avg": false,
@@ -1496,10 +1455,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(apisix_batch_process_entries{name=~\"sys-logger|http-logger|sls-logger|tcp-logger|udp-logger|zipkin\"}) by (name)",
+ "expr": "sum(apisix_etcd_modify_indexes{key=~\"consumers|global_rules|max_modify_index|prev_index|protos|routes|services|ssls|stream_routes|upstreams|x_etcd_index\"}) by (key)",
"interval": "",
"intervalFactor": 1,
- "legendFormat": "{{name}}",
+ "legendFormat": "{{key}}",
"refId": "A"
}
],
@@ -1507,7 +1466,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Batch process entries",
+ "title": "Etcd modify indexes",
"tooltip": {
"shared": true,
"sort": 0,
@@ -1560,6 +1519,67 @@
"value": null
},
{
+ "color": "red",
+ "value": 0
+ },
+ {
+ "color": "green",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 3,
+ "x": 18,
+ "y": 51
+ },
+ "id": 25,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "7.3.7",
+ "targets": [
+ {
+ "expr": "sum(apisix_etcd_reachable{instance=~\"$instance\"})",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Etcd reachable",
+ "type": "stat"
+ },
+ {
+ "datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
"color": "yellow",
"value": 1
}
@@ -1569,10 +1589,10 @@
"overrides": []
},
"gridPos": {
- "h": 3,
- "w": 4,
- "x": 0,
- "y": 54
+ "h": 6,
+ "w": 3,
+ "x": 21,
+ "y": 51
},
"id": 29,
"options": {
@@ -1660,7 +1680,7 @@
"allValue": ".*",
"current": {},
"datasource": "${DS_PROMETHEUS}",
- "definition": "label_values(apisix_http_status,instance)",
+ "definition": "label_values(apisix_nginx_http_current_connections,instance)",
"error": null,
"hide": 0,
"includeAll": true,
@@ -1728,7 +1748,7 @@
]
},
"time": {
- "from": "now-12h",
+ "from": "now-30m",
"to": "now"
},
"timepicker": {
@@ -1748,5 +1768,5 @@
"timezone": "",
"title": "Apache APISIX",
"uid": "bLlNuRLWz",
- "version": 9
+ "version": 13
}
diff --git a/docs/en/latest/plugins/prometheus.md b/docs/en/latest/plugins/prometheus.md
index 2fd198f..a7100bf 100644
--- a/docs/en/latest/plugins/prometheus.md
+++ b/docs/en/latest/plugins/prometheus.md
@@ -143,6 +143,8 @@ Or you can goto [Grafana official](https://grafana.com/grafana/dashboards/11719)
+
+
### Available metrics
* `Status codes`: HTTP status code returned from upstream services. These status code available per service and across all services.
@@ -174,27 +176,17 @@ Or you can goto [Grafana official](https://grafana.com/grafana/dashboards/11719)
* `etcd reachability`: A gauge type with a value of 0 or 1, representing if etcd can be reached by a APISIX or not, where `1` is available, and `0` is unavailable.
* `Connections`: Various Nginx connection metrics like active, reading, writing, and number of accepted connections.
* `Batch process entries`: A gauge type, when we use plugins and the plugin used batch process to send data, such as: sys logger, http logger, sls logger, tcp logger, udp logger and zipkin, then the entries which hasn't been sent in batch process will be counted in the metrics.
-* `Latency`: The per service histogram of request time and the overhead added by APISIX (request time - upstream response time).
+* `Latency`: The per service histogram of request time in different dimensions.
Attributes：
| Name | Description |
| ----------| ------------- |
- | type | Its value is fixed as `request`, which means HTTP request. |
+ | type | The value can be `apisix`, `upstream` or `request`, which means http latency caused by apisix, upstream, or their sum. |
| service | The `service_id` of the route matched by the request. When the route lacks service_id, the default is `$host`. |
| consumer | The `consumer_name` of the consumer that matches the request. If it does not match, the default value is an empty string. |
| node | The `ip` of the upstream node. |
-* `Overhead`: HTTP request overhead (in milliseconds) added per service in APISIX.
-
- Attributes：
- | Name | Description |
- | ---------| ------------- |
- | type | Its value is fixed as `request`, which means HTTP request. |
- | service | The `service_id` of the route matched by the request. When the route lacks service_id, the default is `$host`. |
- | consumer | The `consumer_name` of the consumer that matches the request. If it does not match, the default value is an empty string. |
- | node | The `ip` of the upstream node. |
-
* `Info`: the information of APISIX node.
Here is the original metric data of APISIX:
@@ -252,13 +244,12 @@ apisix_nginx_http_current_connections{state="writing"} 1
apisix_nginx_metric_errors_total 0
# HELP apisix_http_latency HTTP request latency in milliseconds per service in APISIX
# TYPE apisix_http_latency histogram
+apisix_http_latency_bucket{type="apisix",service="",consumer="",node="127.0.0.1",le="1"} 1
+apisix_http_latency_bucket{type="apisix",service="",consumer="",node="127.0.0.1",le="2"} 1
apisix_http_latency_bucket{type="request",service="",consumer="",node="127.0.0.1",le="1"} 1
apisix_http_latency_bucket{type="request",service="",consumer="",node="127.0.0.1",le="2"} 1
-...
-# HELP apisix_http_overhead HTTP request overhead added by APISIX in milliseconds per service in APISIX
-# TYPE apisix_http_overhead histogram
-apisix_http_overhead_bucket{type="request",service="",consumer="",node="127.0.0.1",le="1"} 1
-apisix_http_overhead_bucket{type="request",service="",consumer="",node="127.0.0.1",le="2"} 1
+apisix_http_latency_bucket{type="upstream",service="",consumer="",node="127.0.0.1",le="1"} 1
+apisix_http_latency_bucket{type="upstream",service="",consumer="",node="127.0.0.1",le="2"} 1
...
# HELP apisix_node_info Info of APISIX node
# TYPE apisix_node_info gauge
diff --git a/docs/zh/latest/plugins/prometheus.md b/docs/zh/latest/plugins/prometheus.md
index a29c25e..cef4745 100644
--- a/docs/zh/latest/plugins/prometheus.md
+++ b/docs/zh/latest/plugins/prometheus.md
@@ -141,6 +141,8 @@ plugin_attr:
+
+
### 可有的指标
* `Status codes`: upstream 服务返回的 HTTP 状态码，可以统计到每个服务或所有服务的响应状态码的次数总和。具有的维度：
@@ -172,16 +174,7 @@ plugin_attr:
| 名称 | 描述 |
| -------------| ------------- |
- | type | 它的值固定为 `request`，表示 HTTP 请求。 |
- | service | 与请求匹配的 route 的 `service_id`。当路由缺少 service_id 时，则默认为 `$host`。 |
- | consumer | 与请求匹配的 consumer 的 `consumer_name`。未匹配，则默认为空字符串。 |
- | node | 命中的 upstream 节点 `ip`。 |
-
-* `Overhead`: 每个服务在 APISIX 中的请求开销（以毫秒为单位）。具有的维度：
-
- | 名称 | 描述 |
- | -------------| ------------- |
- | type | 它的值固定为 `request`，表示 HTTP 请求。 |
+ | type | 该值可以为`apisix`, `upstream` 和 `request`，分别表示耗时的来源为 APISIX、上游及其总和。 |
| service | 与请求匹配的 route 的 `service_id`。当路由缺少 service_id 时，则默认为 `$host`。 |
| consumer | 与请求匹配的 consumer 的 `consumer_name`。未匹配，则默认为空字符串。 |
| node | 命中的 upstream 节点 `ip`。 |
@@ -243,13 +236,12 @@ apisix_nginx_http_current_connections{state="writing"} 1
apisix_nginx_metric_errors_total 0
# HELP apisix_http_latency HTTP request latency in milliseconds per service in APISIX
# TYPE apisix_http_latency histogram
+apisix_http_latency_bucket{type="apisix",service="",consumer="",node="127.0.0.1",le="1"} 1
+apisix_http_latency_bucket{type="apisix",service="",consumer="",node="127.0.0.1",le="2"} 1
apisix_http_latency_bucket{type="request",service="",consumer="",node="127.0.0.1",le="1"} 1
apisix_http_latency_bucket{type="request",service="",consumer="",node="127.0.0.1",le="2"} 1
-...
-# HELP apisix_http_overhead HTTP request overhead added by APISIX in milliseconds per service in APISIX
-# TYPE apisix_http_overhead histogram
-apisix_http_overhead_bucket{type="request",service="",consumer="",node="127.0.0.1",le="1"} 1
-apisix_http_overhead_bucket{type="request",service="",consumer="",node="127.0.0.1",le="2"} 1
+apisix_http_latency_bucket{type="upstream",service="",consumer="",node="127.0.0.1",le="1"} 1
+apisix_http_latency_bucket{type="upstream",service="",consumer="",node="127.0.0.1",le="2"} 1
...
# HELP apisix_node_info Info of APISIX node
# TYPE apisix_node_info gauge
diff --git a/t/plugin/prometheus.t b/t/plugin/prometheus.t
index 51d73ee..7df67e4 100644
--- a/t/plugin/prometheus.t
+++ b/t/plugin/prometheus.t
@@ -527,11 +527,11 @@ qr/apisix_http_status\{code="404",route="3",matched_uri="\/hello3",matched_host=
-=== TEST 25: fetch the prometheus metric data with `overhead`
+=== TEST 25: fetch the prometheus metric data with apisix latency
--- request
GET /apisix/prometheus/metrics
--- response_body eval
-qr/.*apisix_http_overhead_bucket.*/
+qr/.*apisix_http_latency_bucket\{type="apisix".*/
--- no_error_log
[error]
@@ -610,11 +610,11 @@ passed
-=== TEST 29: fetch the prometheus metric data with `overhead`(the overhead < 1s)
+=== TEST 29: fetch the prometheus metric data with apisix latency (latency < 1s)
--- request
GET /apisix/prometheus/metrics
--- response_body eval
-qr/apisix_http_overhead_bucket.*service=\"3\".*le=\"500.*/
+qr/apisix_http_latency_bucket\{type="apisix".*service=\"3\".*le=\"500.*/
--- no_error_log
[error]