You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2022/06/01 09:19:16 UTC
[apisix] branch master updated: feat(stream): add prometheus plugin (#7174)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new a8afdf2b8 feat(stream): add prometheus plugin (#7174)
a8afdf2b8 is described below
commit a8afdf2b83be9728fd1c486265c2b6eb299463ce
Author: 罗泽轩 <sp...@gmail.com>
AuthorDate: Wed Jun 1 17:19:08 2022 +0800
feat(stream): add prometheus plugin (#7174)
---
apisix/cli/ngx_tpl.lua | 44 ++++++++-
apisix/cli/ops.lua | 12 +++
apisix/plugin.lua | 7 +-
apisix/plugins/prometheus.lua | 54 +----------
apisix/plugins/prometheus/exporter.lua | 111 +++++++++++++++++++++-
apisix/stream/plugins/prometheus.lua | 48 ++++++++++
conf/config-default.yaml | 5 +
docs/en/latest/plugins/prometheus.md | 59 +++++++++++-
docs/zh/latest/plugins/prometheus.md | 57 +++++++++++-
t/APISIX.pm | 15 ++-
t/cli/common.sh | 1 +
t/cli/test_etcd_mtls.sh | 2 +-
t/cli/test_prometheus_stream.sh | 93 +++++++++++++++++++
t/stream-plugin/prometheus.t | 162 +++++++++++++++++++++++++++++++++
14 files changed, 601 insertions(+), 69 deletions(-)
diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua
index e92b8092a..4709362e5 100644
--- a/apisix/cli/ngx_tpl.lua
+++ b/apisix/cli/ngx_tpl.lua
@@ -57,6 +57,44 @@ env {*name*};
{% end %}
{% end %}
+{% if use_apisix_openresty then %}
+lua {
+ {% if enabled_stream_plugins["prometheus"] then %}
+ lua_shared_dict prometheus-metrics {* meta.lua_shared_dict["prometheus-metrics"] *};
+ {% end %}
+}
+
+{% if enabled_stream_plugins["prometheus"] and not enable_http then %}
+http {
+ init_worker_by_lua_block {
+ require("apisix.plugins.prometheus.exporter").http_init(true)
+ }
+
+ server {
+ listen {* prometheus_server_addr *};
+
+ access_log off;
+
+ location / {
+ content_by_lua_block {
+ local prometheus = require("apisix.plugins.prometheus.exporter")
+ prometheus.export_metrics(true)
+ }
+ }
+
+ {% if with_module_status then %}
+ location = /apisix/nginx_status {
+ allow 127.0.0.0/24;
+ deny all;
+ stub_status;
+ }
+ {% end %}
+ }
+}
+{% end %}
+
+{% end %}
+
{% if stream_proxy then %}
stream {
lua_package_path "{*extra_lua_path*}$prefix/deps/share/lua/5.1/?.lua;$prefix/deps/share/lua/5.1/?/init.lua;]=]
@@ -164,7 +202,7 @@ stream {
}
{% end %}
-{% if enable_admin or not (stream_proxy and stream_proxy.only ~= false) then %}
+{% if enable_http then %}
http {
# put extra_lua_path in front of the builtin path
# so user can override the source code
@@ -211,7 +249,7 @@ http {
lua_shared_dict plugin-limit-count-redis-cluster-slot-lock {* http.lua_shared_dict["plugin-limit-count-redis-cluster-slot-lock"] *};
{% end %}
- {% if enabled_plugins["prometheus"] then %}
+ {% if enabled_plugins["prometheus"] and not enabled_stream_plugins["prometheus"] then %}
lua_shared_dict prometheus-metrics {* http.lua_shared_dict["prometheus-metrics"] *};
{% end %}
@@ -460,7 +498,7 @@ http {
location / {
content_by_lua_block {
- local prometheus = require("apisix.plugins.prometheus")
+ local prometheus = require("apisix.plugins.prometheus.exporter")
prometheus.export_metrics()
}
}
diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua
index 384f2aa1c..937d74106 100644
--- a/apisix/cli/ops.lua
+++ b/apisix/cli/ops.lua
@@ -257,6 +257,13 @@ Please modify "admin_key" in conf/config.yaml .
use_apisix_openresty = false
end
+ local enable_http = true
+ if not yaml_conf.apisix.enable_admin and yaml_conf.apisix.stream_proxy and
+ yaml_conf.apisix.stream_proxy.only ~= false
+ then
+ enable_http = false
+ end
+
local enabled_discoveries = {}
for name in pairs(yaml_conf.discovery or {}) do
enabled_discoveries[name] = true
@@ -346,6 +353,10 @@ Please modify "admin_key" in conf/config.yaml .
end
end
+ if enabled_stream_plugins["prometheus"] and not prometheus_server_addr then
+ util.die("L4 prometheus metric should be exposed via export server\n")
+ end
+
local ip_port_to_check = {}
local function listen_table_insert(listen_table, scheme, ip, port, enable_http2, enable_ipv6)
@@ -540,6 +551,7 @@ Please modify "admin_key" in conf/config.yaml .
with_module_status = with_module_status,
use_apisix_openresty = use_apisix_openresty,
error_log = {level = "warn"},
+ enable_http = enable_http,
enabled_discoveries = enabled_discoveries,
enabled_plugins = enabled_plugins,
enabled_stream_plugins = enabled_stream_plugins,
diff --git a/apisix/plugin.lua b/apisix/plugin.lua
index fc76b2546..5aad12e89 100644
--- a/apisix/plugin.lua
+++ b/apisix/plugin.lua
@@ -567,8 +567,11 @@ function _M.init_worker()
_M.load()
-- some plugins need to be initialized in init* phases
- if ngx.config.subsystem == "http" and local_plugins_hash["prometheus"] then
- require("apisix.plugins.prometheus.exporter").init()
+ if is_http and local_plugins_hash["prometheus"] then
+ local prometheus_enabled_in_stream = stream_local_plugins_hash["prometheus"]
+ require("apisix.plugins.prometheus.exporter").http_init(prometheus_enabled_in_stream)
+ elseif not is_http and stream_local_plugins_hash["prometheus"] then
+ require("apisix.plugins.prometheus.exporter").stream_init()
end
if local_conf and not local_conf.apisix.enable_admin then
diff --git a/apisix/plugins/prometheus.lua b/apisix/plugins/prometheus.lua
index bce99625f..b15469754 100644
--- a/apisix/plugins/prometheus.lua
+++ b/apisix/plugins/prometheus.lua
@@ -14,14 +14,11 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-local ngx = ngx
local core = require("apisix.core")
-local plugin = require("apisix.plugin")
local exporter = require("apisix.plugins.prometheus.exporter")
local plugin_name = "prometheus"
-local default_export_uri = "/apisix/prometheus/metrics"
local schema = {
type = "object",
properties = {
@@ -37,7 +34,7 @@ local _M = {
version = 0.2,
priority = 500,
name = plugin_name,
- log = exporter.log,
+ log = exporter.http_log,
schema = schema,
run_policy = "prefer_route",
}
@@ -53,56 +50,9 @@ function _M.check_schema(conf)
end
-local function get_api(called_by_api_router)
- local export_uri = default_export_uri
- local attr = plugin.plugin_attr(plugin_name)
- if attr and attr.export_uri then
- export_uri = attr.export_uri
- end
-
- local api = {
- methods = {"GET"},
- uri = export_uri,
- handler = exporter.collect
- }
-
- if not called_by_api_router then
- return api
- end
-
- if attr.enable_export_server then
- return {}
- end
-
- return {api}
-end
-
-
function _M.api()
- return get_api(true)
+ return exporter.get_api(true)
end
-function _M.export_metrics()
- local api = get_api(false)
- local uri = ngx.var.uri
- local method = ngx.req.get_method()
-
- if uri == api.uri and method == api.methods[1] then
- local code, body = api.handler()
- if code or body then
- core.response.exit(code, body)
- end
- end
-
- return core.response.exit(404)
-end
-
-
--- only for test
--- function _M.access()
--- ngx.say(exporter.metric_data())
--- end
-
-
return _M
diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index 30534d567..e9295e7d8 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -19,8 +19,10 @@ local core = require("apisix.core")
local plugin = require("apisix.plugin")
local ipairs = ipairs
local ngx = ngx
-local ngx_capture = ngx.location.capture
local re_gmatch = ngx.re.gmatch
+local ffi = require("ffi")
+local C = ffi.C
+local pcall = pcall
local select = select
local type = type
local prometheus
@@ -36,8 +38,14 @@ local get_protos = require("apisix.plugins.grpc-transcode.proto").protos
local service_fetch = require("apisix.http.service").get
local latency_details = require("apisix.utils.log-util").latency_details_in_ms
+local ngx_capture
+if ngx.config.subsystem == "http" then
+ ngx_capture = ngx.location.capture
+end
+local plugin_name = "prometheus"
+local default_export_uri = "/apisix/prometheus/metrics"
-- Default set of latency buckets, 1ms to 60s:
local DEFAULT_BUCKETS = {1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 30000, 60000}
@@ -58,7 +66,14 @@ end
local _M = {}
-function _M.init()
+local function init_stream_metrics()
+ metrics.stream_connection_total = prometheus:counter("stream_connection_total",
+ "Total number of connections handled per stream route in APISIX",
+ {"route"})
+end
+
+
+function _M.http_init(prometheus_enabled_in_stream)
-- todo: support hot reload, we may need to update the lua-prometheus
-- library
if ngx.get_phase() ~= "init" and ngx.get_phase() ~= "init_worker" then
@@ -83,6 +98,7 @@ function _M.init()
end
prometheus = base_prometheus.init("prometheus-metrics", metric_prefix)
+
metrics.connections = prometheus:gauge("nginx_http_current_connections",
"Number of HTTP connections",
{"state"})
@@ -119,10 +135,37 @@ function _M.init()
"Total bandwidth in bytes consumed per service in APISIX",
{"type", "route", "service", "consumer", "node"})
+ if prometheus_enabled_in_stream then
+ init_stream_metrics()
+ end
+end
+
+
+function _M.stream_init()
+ if ngx.get_phase() ~= "init" and ngx.get_phase() ~= "init_worker" then
+ return
+ end
+
+ if not pcall(function() return C.ngx_meta_lua_ffi_shdict_udata_to_zone end) then
+ core.log.error("need to build APISIX-Base to support L4 metrics")
+ return
+ end
+
+ clear_tab(metrics)
+
+ local metric_prefix = "apisix_"
+ local attr = plugin.plugin_attr("prometheus")
+ if attr and attr.metric_prefix then
+ metric_prefix = attr.metric_prefix
+ end
+
+ prometheus = base_prometheus.init("prometheus-metrics", metric_prefix)
+
+ init_stream_metrics()
end
-function _M.log(conf, ctx)
+function _M.http_log(conf, ctx)
local vars = ctx.var
local route_id = ""
@@ -174,6 +217,20 @@ function _M.log(conf, ctx)
end
+function _M.stream_log(conf, ctx)
+ local route_id = ""
+ local matched_route = ctx.matched_route and ctx.matched_route.value
+ if matched_route then
+ route_id = matched_route.id
+ if conf.prefer_name == true then
+ route_id = matched_route.name or route_id
+ end
+ end
+
+ metrics.stream_connection_total:inc(1, gen_arr(route_id))
+end
+
+
local ngx_status_items = {"active", "accepted", "handled", "total",
"reading", "writing", "waiting"}
local label_values = {}
@@ -291,7 +348,7 @@ local function etcd_modify_index()
end
-function _M.collect()
+local function collect(ctx, stream_only)
if not prometheus or not metrics then
core.log.error("prometheus: plugin is not initialized, please make sure ",
" 'prometheus_metrics' shared dict is present in nginx template")
@@ -307,7 +364,8 @@ function _M.collect()
local vars = ngx.var or {}
local hostname = vars.hostname or ""
- if config.type == "etcd" then
+ -- we can't get etcd index in metric server if only stream subsystem is enabled
+ if config.type == "etcd" and not stream_only then
-- etcd modify index
etcd_modify_index()
@@ -338,6 +396,49 @@ function _M.collect()
core.response.set_header("content_type", "text/plain")
return 200, core.table.concat(prometheus:metric_data())
end
+_M.collect = collect
+
+
+local function get_api(called_by_api_router)
+ local export_uri = default_export_uri
+ local attr = plugin.plugin_attr(plugin_name)
+ if attr and attr.export_uri then
+ export_uri = attr.export_uri
+ end
+
+ local api = {
+ methods = {"GET"},
+ uri = export_uri,
+ handler = collect
+ }
+
+ if not called_by_api_router then
+ return api
+ end
+
+ if attr.enable_export_server then
+ return {}
+ end
+
+ return {api}
+end
+_M.get_api = get_api
+
+
+function _M.export_metrics(stream_only)
+ local api = get_api(false)
+ local uri = ngx.var.uri
+ local method = ngx.req.get_method()
+
+ if uri == api.uri and method == api.methods[1] then
+ local code, body = api.handler(nil, stream_only)
+ if code or body then
+ core.response.exit(code, body)
+ end
+ end
+
+ return core.response.exit(404)
+end
function _M.metric_data()
diff --git a/apisix/stream/plugins/prometheus.lua b/apisix/stream/plugins/prometheus.lua
new file mode 100644
index 000000000..46222eca2
--- /dev/null
+++ b/apisix/stream/plugins/prometheus.lua
@@ -0,0 +1,48 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local exporter = require("apisix.plugins.prometheus.exporter")
+
+
+local plugin_name = "prometheus"
+local schema = {
+ type = "object",
+ properties = {
+ prefer_name = {
+ type = "boolean",
+ default = false -- stream route doesn't have name yet
+ }
+ },
+}
+
+
+local _M = {
+ version = 0.1,
+ priority = 500,
+ name = plugin_name,
+ log = exporter.stream_log,
+ schema = schema,
+ run_policy = "prefer_route",
+}
+
+
+function _M.check_schema(conf)
+ return core.schema.check(schema, conf)
+end
+
+
+return _M
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index f60c6577f..8f9e58e5d 100644
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -177,6 +177,10 @@ nginx_config: # config for render the template to generate n
#envs: # allow to get a list of environment variables
# - TEST_ENV
+ meta:
+ lua_shared_dict:
+ prometheus-metrics: 15m
+
stream:
enable_access_log: false # enable access log or not, default false
access_log: logs/access_stream.log
@@ -409,6 +413,7 @@ stream_plugins: # sorted by priority
- ip-restriction # priority: 3000
- limit-conn # priority: 1003
- mqtt-proxy # priority: 1000
+ #- prometheus # priority: 500
- syslog # priority: 401
# <- recommend to use priority (0, 100) for your custom plugins
diff --git a/docs/en/latest/plugins/prometheus.md b/docs/en/latest/plugins/prometheus.md
index f083fecd5..5722733ae 100644
--- a/docs/en/latest/plugins/prometheus.md
+++ b/docs/en/latest/plugins/prometheus.md
@@ -135,7 +135,7 @@ plugin_attr:
export_uri: /apisix/metrics
```
-### Grafana dashboard
+## Grafana dashboard
Metrics exported by the plugin can be graphed in Grafana using a drop in dashboard.
@@ -151,7 +151,7 @@ Or you can goto [Grafana official](https://grafana.com/grafana/dashboards/11719)
-### Available metrics
+## Available metrics
* `Status codes`: HTTP status code returned from upstream services. These status code available per service and across all services.
@@ -282,3 +282,58 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f1
}
}'
```
+
+## Gather L4 metrics
+
+:::info IMPORTANT
+
+This feature requires APISIX to run on [APISIX-Base](../FAQ.md#how-do-i-build-the-apisix-base-environment?).
+
+:::
+
+We can also enable `prometheus` on the stream route:
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+ "plugins": {
+ "prometheus":{}
+ },
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": {
+ "127.0.0.1:80": 1
+ }
+ }
+}'
+```
+
+## L4 available metrics
+
+The following metrics are available when using APISIX as an L4 proxy.
+
+* `Stream Connections`: The number of processed connections at the route level.
+
+ Attributes:
+
+ | Name | Description |
+ | ------------- | -------------------- |
+ | route | matched stream route ID |
+* `Connections`: Various Nginx connection metrics like active, reading, writing, and number of accepted connections.
+* `Info`: Information about the current APISIX node.
+
+Here are examples of APISIX metrics:
+
+```shell
+$ curl http://127.0.0.1:9091/apisix/prometheus/metrics
+```
+
+```
+...
+# HELP apisix_node_info Info of APISIX node
+# TYPE apisix_node_info gauge
+apisix_node_info{hostname="desktop-2022q8f-wsl"} 1
+# HELP apisix_stream_connection_total Total number of connections handled per stream route in APISIX
+# TYPE apisix_stream_connection_total counter
+apisix_stream_connection_total{route="1"} 1
+```
diff --git a/docs/zh/latest/plugins/prometheus.md b/docs/zh/latest/plugins/prometheus.md
index 7755514bc..2d126727a 100644
--- a/docs/zh/latest/plugins/prometheus.md
+++ b/docs/zh/latest/plugins/prometheus.md
@@ -134,7 +134,7 @@ plugin_attr:
export_uri: /apisix/metrics
```
-### Grafana 面板
+## Grafana 面板
插件导出的指标可以在 Grafana 进行图形化绘制显示。
@@ -150,7 +150,7 @@ plugin_attr:
-### 可有的指标
+## 可用的指标
* `Status codes`: upstream 服务返回的 HTTP 状态码，可以统计到每个服务或所有服务的响应状态码的次数总和。具有的维度：
@@ -274,3 +274,56 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f1
}
}'
```
+
+## 采集 L4 指标
+
+:::info IMPORTANT
+
+该功能要求 Apache APISIX 运行在 [APISIX-Base](../FAQ.md#如何构建-APISIX-Base-环境？) 上。
+
+:::
+
+我们也可以在 stream route 上开启 `prometheus`：
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+ "plugins": {
+ "prometheus":{}
+ },
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": {
+ "127.0.0.1:80": 1
+ }
+ }
+}'
+```
+
+## L4 可用的指标
+
+以下是把 APISIX 作为 L4 代理时可用的指标：
+
+* `Stream Connections`: 路由级别的已处理连接数。具有的维度：
+
+ | 名称 | 描述 |
+ | -------------| --------------------|
+ | route | 匹配的 stream route ID|
+* `Connections`: 各种的 Nginx 连接指标，如 active，reading，writing，已建立的连接数。
+* `Info`: 当前 APISIX 节点信息。
+
+这里是 APISIX 指标的范例：
+
+```shell
+$ curl http://127.0.0.1:9091/apisix/prometheus/metrics
+```
+
+```
+...
+# HELP apisix_node_info Info of APISIX node
+# TYPE apisix_node_info gauge
+apisix_node_info{hostname="desktop-2022q8f-wsl"} 1
+# HELP apisix_stream_connection_total Total number of connections handled per stream route in APISIX
+# TYPE apisix_stream_connection_total counter
+apisix_stream_connection_total{route="1"} 1
+```
diff --git a/t/APISIX.pm b/t/APISIX.pm
index b8e87a859..0143aa9d8 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -238,6 +238,15 @@ env PATH; # for searching external plugin runner's binary
env TEST_NGINX_HTML_DIR;
_EOC_
+
+ if ($version =~ m/\/apisix-nginx-module/) {
+ $main_config .= <<_EOC_;
+lua {
+ lua_shared_dict prometheus-metrics 15m;
+}
+_EOC_
+ }
+
# set default `timeout` to 5sec
my $timeout = $block->timeout // 5;
$block->set_value("timeout", $timeout);
@@ -480,7 +489,6 @@ _EOC_
lua_shared_dict plugin-limit-req 10m;
lua_shared_dict plugin-limit-count 10m;
lua_shared_dict plugin-limit-conn 10m;
- lua_shared_dict prometheus-metrics 10m;
lua_shared_dict internal-status 10m;
lua_shared_dict upstream-healthcheck 32m;
lua_shared_dict worker-events 10m;
@@ -526,6 +534,7 @@ _EOC_
balancer_by_lua_block {
apisix.http_balancer_phase()
}
+ }
_EOC_
} else {
$http_config .= <<_EOC_;
@@ -534,11 +543,13 @@ _EOC_
}
keepalive 32;
+ }
+
+ lua_shared_dict prometheus-metrics 10m;
_EOC_
}
$http_config .= <<_EOC_;
- }
$dubbo_upstream
diff --git a/t/cli/common.sh b/t/cli/common.sh
index 9da89a022..a4e2dea54 100644
--- a/t/cli/common.sh
+++ b/t/cli/common.sh
@@ -39,4 +39,5 @@ exit_if_not_customed_nginx() {
openresty -V 2>&1 | grep apisix-nginx-module || exit 0
}
+rm logs/error.log || true # clear previous error log
unset APISIX_PROFILE
diff --git a/t/cli/test_etcd_mtls.sh b/t/cli/test_etcd_mtls.sh
index 6741ea178..371330e93 100755
--- a/t/cli/test_etcd_mtls.sh
+++ b/t/cli/test_etcd_mtls.sh
@@ -115,7 +115,7 @@ if echo "$out" | grep "ouch"; then
exit 1
fi
-rm logs/error.log
+rm logs/error.log || true
make run
sleep 1
make stop
diff --git a/t/cli/test_prometheus_stream.sh b/t/cli/test_prometheus_stream.sh
new file mode 100755
index 000000000..347774b27
--- /dev/null
+++ b/t/cli/test_prometheus_stream.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+. ./t/cli/common.sh
+
+exit_if_not_customed_nginx
+
+echo "
+apisix:
+ enable_admin: true
+ stream_proxy:
+ tcp:
+ - addr: 9100
+stream_plugins:
+ - prometheus
+" > conf/config.yaml
+
+make run
+sleep 0.5
+
+curl -v -k -i -m 20 -o /dev/null -s -X PUT http://127.0.0.1:9080/apisix/admin/stream_routes/1 \
+ -H "X-API-KEY: edd1c9f034335f136f87ad84b625c8f1" \
+ -d '{
+ "plugins": {
+ "prometheus": {}
+ },
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": [{
+ "host": "127.0.0.1",
+ "port": 1995,
+ "weight": 1
+ }]
+ }
+ }'
+
+curl http://127.0.0.1:9100 || true
+sleep 1 # wait for sync
+
+out="$(curl http://127.0.0.1:9091/apisix/prometheus/metrics)"
+if ! echo "$out" | grep "apisix_stream_connection_total{route=\"1\"} 1" > /dev/null; then
+ echo "failed: prometheus can't work in stream subsystem"
+ exit 1
+fi
+
+make stop
+
+echo "passed: prometheus works when both http & stream are enabled"
+
+echo "
+apisix:
+ enable_admin: false
+ stream_proxy:
+ tcp:
+ - addr: 9100
+stream_plugins:
+ - prometheus
+" > conf/config.yaml
+
+make run
+sleep 0.5
+
+curl http://127.0.0.1:9100 || true
+sleep 1 # wait for sync
+
+out="$(curl http://127.0.0.1:9091/apisix/prometheus/metrics)"
+if ! echo "$out" | grep "apisix_stream_connection_total{route=\"1\"} 1" > /dev/null; then
+ echo "failed: prometheus can't work in stream subsystem"
+ exit 1
+fi
+
+if ! echo "$out" | grep "apisix_node_info{hostname=" > /dev/null; then
+ echo "failed: prometheus can't work in stream subsystem"
+ exit 1
+fi
+
+echo "passed: prometheus works when only stream is enabled"
diff --git a/t/stream-plugin/prometheus.t b/t/stream-plugin/prometheus.t
new file mode 100644
index 000000000..796b22028
--- /dev/null
+++ b/t/stream-plugin/prometheus.t
@@ -0,0 +1,162 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+BEGIN {
+ if ($ENV{TEST_NGINX_CHECK_LEAK}) {
+ $SkipReason = "unavailable for the hup tests";
+
+ } else {
+ $ENV{TEST_NGINX_USE_HUP} = 1;
+ undef $ENV{TEST_NGINX_USE_STAP};
+ }
+}
+
+use t::APISIX;
+
+my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx';
+my $version = eval { `$nginx_binary -V 2>&1` };
+
+if ($version !~ m/\/apisix-nginx-module/) {
+ plan(skip_all => "apisix-nginx-module not installed");
+} else {
+ plan('no_plan');
+}
+
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ my $extra_yaml_config = <<_EOC_;
+stream_plugins:
+ - mqtt-proxy
+ - prometheus
+_EOC_
+
+ $block->set_value("extra_yaml_config", $extra_yaml_config);
+
+ if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
+ $block->set_value("no_error_log", "[error]");
+ }
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: pre-create public API route
+--- config
+ location /t {
+ content_by_lua_block {
+ local data = {
+ {
+ url = "/apisix/admin/routes/metrics",
+ data = [[{
+ "plugins": {
+ "public-api": {}
+ },
+ "uri": "/apisix/prometheus/metrics"
+ }]]
+ },
+ {
+ url = "/apisix/admin/stream_routes/mqtt",
+ data = [[{
+ "plugins": {
+ "mqtt-proxy": {
+ "protocol_name": "MQTT",
+ "protocol_level": 4
+ },
+ "prometheus": {}
+ },
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": [{
+ "host": "127.0.0.1",
+ "port": 1995,
+ "weight": 1
+ }]
+ }
+ }]]
+ }
+ }
+
+ local t = require("lib.test_admin").test
+
+ for _, data in ipairs(data) do
+ local code, body = t(data.url, ngx.HTTP_PUT, data.data)
+ if code > 300 then
+ ngx.say(body)
+ return
+ end
+ end
+ }
+ }
+--- response_body
+
+
+
+=== TEST 2: hit
+--- stream_request eval
+"\x10\x0f\x00\x04\x4d\x51\x54\x54\x04\x02\x00\x3c\x00\x03\x66\x6f\x6f"
+--- stream_response
+hello world
+
+
+
+=== TEST 3: fetch the prometheus metric data
+--- request
+GET /apisix/prometheus/metrics
+--- response_body eval
+qr/apisix_stream_connection_total\{route="mqtt"\} 1/
+
+
+
+=== TEST 4: hit, error
+--- stream_request eval
+mmm
+--- error_log
+Received unexpected MQTT packet type+flags
+
+
+
+=== TEST 5: fetch the prometheus metric data
+--- request
+GET /apisix/prometheus/metrics
+--- response_body eval
+qr/apisix_stream_connection_total\{route="mqtt"\} 2/
+
+
+
+=== TEST 6: contains metrics from stub_status
+--- request
+GET /apisix/prometheus/metrics
+--- response_body eval
+qr/apisix_nginx_http_current_connections\{state="active"\} 1/
+
+
+
+=== TEST 7: contains basic metrics
+--- request
+GET /apisix/prometheus/metrics
+--- response_body eval
+qr/apisix_node_info\{hostname="[^"]+"\}/