You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2022/06/02 06:30:45 UTC
[apisix] branch master updated: feat(redis): add metrics (#7183)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 456287534 feat(redis): add metrics (#7183)
456287534 is described below
commit 4562875348fdaab3cb05e6df8aecf0604fa6e157
Author: 罗泽轩 <sp...@gmail.com>
AuthorDate: Thu Jun 2 14:30:38 2022 +0800
feat(redis): add metrics (#7183)
Signed-off-by: spacewander <sp...@gmail.com>
---
apisix/plugins/prometheus/exporter.lua | 4 +
apisix/stream/xrpc.lua | 24 ++-
apisix/stream/xrpc/metrics.lua | 50 +++++
apisix/stream/xrpc/protocols/redis/init.lua | 9 +
apisix/stream/xrpc/protocols/redis/metrics.lua | 33 +++
apisix/stream/xrpc/sdk.lua | 17 ++
docs/en/latest/xrpc.md | 25 +++
docs/en/latest/xrpc/redis.md | 16 ++
t/xrpc/prometheus.t | 273 +++++++++++++++++++++++++
9 files changed, 448 insertions(+), 3 deletions(-)
diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index e9295e7d8..c65a39c48 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -37,6 +37,8 @@ local get_stream_routes = router.stream_routes
local get_protos = require("apisix.plugins.grpc-transcode.proto").protos
local service_fetch = require("apisix.http.service").get
local latency_details = require("apisix.utils.log-util").latency_details_in_ms
+local xrpc = require("apisix.stream.xrpc")
+
local ngx_capture
if ngx.config.subsystem == "http" then
@@ -70,6 +72,8 @@ local function init_stream_metrics()
metrics.stream_connection_total = prometheus:counter("stream_connection_total",
"Total number of connections handled per stream route in APISIX",
{"route"})
+
+ xrpc.init_metrics(prometheus)
end
diff --git a/apisix/stream/xrpc.lua b/apisix/stream/xrpc.lua
index 418ec4cf4..f9cfa8c75 100644
--- a/apisix/stream/xrpc.lua
+++ b/apisix/stream/xrpc.lua
@@ -16,6 +16,7 @@
--
local require = require
local core = require("apisix.core")
+local metrics = require("apisix.stream.xrpc.metrics")
local ipairs = ipairs
local pairs = pairs
local ngx_exit = ngx.exit
@@ -45,7 +46,7 @@ end
function _M.init()
- local local_conf = core.config.local_conf(true)
+ local local_conf = core.config.local_conf()
if not local_conf.xrpc then
return
end
@@ -67,9 +68,26 @@ function _M.init()
end
+function _M.init_metrics(collector)
+ local local_conf = core.config.local_conf()
+ if not local_conf.xrpc then
+ return
+ end
+
+ local prot_conf = local_conf.xrpc.protocols
+ if not prot_conf then
+ return
+ end
+
+ for _, prot in ipairs(prot_conf) do
+ metrics.store(collector, prot.name)
+ end
+end
+
+
function _M.init_worker()
- for _, prot in pairs(registered_protocols) do
- if prot.init_worker then
+ for name, prot in pairs(registered_protocols) do
+ if not is_http and prot.init_worker then
prot.init_worker()
end
end
diff --git a/apisix/stream/xrpc/metrics.lua b/apisix/stream/xrpc/metrics.lua
new file mode 100644
index 000000000..41b77d4c4
--- /dev/null
+++ b/apisix/stream/xrpc/metrics.lua
@@ -0,0 +1,50 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local require = require
+local core = require("apisix.core")
+local pairs = pairs
+local pcall = pcall
+
+
+local _M = {}
+local hubs = {}
+
+
+function _M.store(prometheus, name)
+ local ok, m = pcall(require, "apisix.stream.xrpc.protocols." .. name .. ".metrics")
+ if not ok then
+ core.log.notice("no metric for protocol ", name)
+ return
+ end
+
+ local hub = {}
+ for metric, conf in pairs(m) do
+ core.log.notice("register metric ", metric, " for protocol ", name)
+ hub[metric] = prometheus[conf.type](prometheus, name .. '_' .. metric,
+ conf.help, conf.labels, conf.buckets)
+ end
+
+ hubs[name] = hub
+end
+
+
+function _M.load(name)
+ return hubs[name]
+end
+
+
+return _M
diff --git a/apisix/stream/xrpc/protocols/redis/init.lua b/apisix/stream/xrpc/protocols/redis/init.lua
index 986c561f9..9aff6d0d1 100644
--- a/apisix/stream/xrpc/protocols/redis/init.lua
+++ b/apisix/stream/xrpc/protocols/redis/init.lua
@@ -39,6 +39,7 @@ end)
-- redis protocol spec: https://redis.io/docs/reference/protocol-spec/
-- There is no plan to support inline command format
+local protocol_name = "redis"
local _M = {}
local MAX_LINE_LEN = 128
local MAX_VALUE_LEN = 128
@@ -107,6 +108,7 @@ function _M.init_downstream(session)
session.req_id_seq = 0
session.resp_id_seq = 0
+ session.cmd_labels = {session.route.id, ""}
return xrpc_socket.downstream.socket()
end
@@ -482,6 +484,13 @@ end
function _M.log(session, ctx)
+ local metrics = sdk.get_metrics(session, protocol_name)
+ if metrics then
+ session.cmd_labels[2] = ctx.cmd
+ metrics.commands_total:inc(1, session.cmd_labels)
+ metrics.commands_latency_seconds:observe(ctx.var.rpc_time, session.cmd_labels)
+ end
+
core.tablepool.release("xrpc_redis_cmd_line", ctx.cmd_line)
ctx.cmd_line = nil
end
diff --git a/apisix/stream/xrpc/protocols/redis/metrics.lua b/apisix/stream/xrpc/protocols/redis/metrics.lua
new file mode 100644
index 000000000..6009a5047
--- /dev/null
+++ b/apisix/stream/xrpc/protocols/redis/metrics.lua
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local _M = {
+ commands_total = {
+ type = "counter",
+ help = "Total number of requests for a specific Redis command",
+ labels = {"route", "command"},
+ },
+ commands_latency_seconds = {
+ type = "histogram",
+ help = "Latency of requests for a specific Redis command",
+ labels = {"route", "command"},
+ -- latency buckets, 1ms to 1s:
+ buckets = {0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1}
+ },
+}
+
+
+return _M
diff --git a/apisix/stream/xrpc/sdk.lua b/apisix/stream/xrpc/sdk.lua
index 65bbe4c40..60f100cbf 100644
--- a/apisix/stream/xrpc/sdk.lua
+++ b/apisix/stream/xrpc/sdk.lua
@@ -21,6 +21,7 @@
local core = require("apisix.core")
local config_util = require("apisix.core.config_util")
local router = require("apisix.stream.router.ip_port")
+local metrics = require("apisix.stream.xrpc.metrics")
local apisix_upstream = require("apisix.upstream")
local xrpc_socket = require("resty.apisix.stream.xrpc.socket")
local ngx_now = ngx.now
@@ -182,4 +183,20 @@ function _M.set_upstream(session, conf)
end
+---
+-- Returns the protocol specific metrics object
+--
+-- @function xrpc.sdk.get_metrics
+-- @tparam table session xrpc session
+-- @tparam string protocol_name protocol name
+-- @treturn nil|table the metrics under the specific protocol if available
+function _M.get_metrics(session, protocol_name)
+ local metric_conf = session.route.protocol.metric
+ if not (metric_conf and metric_conf.enable) then
+ return nil
+ end
+ return metrics.load(protocol_name)
+end
+
+
return _M
diff --git a/docs/en/latest/xrpc.md b/docs/en/latest/xrpc.md
index 9ad486e83..88cb44ea1 100644
--- a/docs/en/latest/xrpc.md
+++ b/docs/en/latest/xrpc.md
@@ -172,6 +172,31 @@ The protocol itself defines the granularity of the specific request, and the xRP
For example, in the Redis protocol, the execution of a command is considered a request.
+### Dynamic metrics
+
+xRPC also supports gathering metrics on the fly and exposing them via Prometheus.
+
+To know how to enable Prometheus metrics for TCP and collect them, please refer to [prometheus](./plugins/prometheus.md).
+
+To get the protocol-specific metrics, you need to:
+
+1. Make sure the Prometheus is enabled for TCP
+2. Add the metric field to the specific route and ensure the `enable` is true:
+
+```json
+{
+ ...
+ "protocol": {
+ "name": "redis",
+ "metric": {
+ "enable": true
+ }
+ }
+}
+```
+
+Different protocols will have different metrics. Please refer to the `Metrics` section of their own documentation.
+
## How to write your own protocol
Assuming that your protocol is named `my_proto`, you need to create a directory that can be introduced by `require "apisix.stream.xrpc.protocols.my_proto"`.
diff --git a/docs/en/latest/xrpc/redis.md b/docs/en/latest/xrpc/redis.md
index cc727b66c..63f791724 100644
--- a/docs/en/latest/xrpc/redis.md
+++ b/docs/en/latest/xrpc/redis.md
@@ -63,6 +63,22 @@ Fields under an entry of `faults`:
| key | string | False | | "blahblah" | Key fault is restricted to |
| delay | number | True | | 0.1 | Duration of the delay in seconds |
+## Metrics
+
+* `apisix_redis_commands_total`: Total number of requests for a specific Redis command.
+
+ | Labels | Description |
+ | ------------- | -------------------- |
+ | route | matched stream route ID |
+ | command | the Redis command |
+
+* `apisix_redis_commands_latency_seconds`: Latency of requests for a specific Redis command.
+
+ | Labels | Description |
+ | ------------- | -------------------- |
+ | route | matched stream route ID |
+ | command | the Redis command |
+
## Example usage
Assumed the APISIX is proxying TCP on port `9101`, and the Redis is listening on port `6379`.
diff --git a/t/xrpc/prometheus.t b/t/xrpc/prometheus.t
new file mode 100644
index 000000000..cc267ac6c
--- /dev/null
+++ b/t/xrpc/prometheus.t
@@ -0,0 +1,273 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+BEGIN {
+ if ($ENV{TEST_NGINX_CHECK_LEAK}) {
+ $SkipReason = "unavailable for the hup tests";
+
+ } else {
+ $ENV{TEST_NGINX_USE_HUP} = 1;
+ undef $ENV{TEST_NGINX_USE_STAP};
+ }
+}
+
+use t::APISIX;
+
+my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx';
+my $version = eval { `$nginx_binary -V 2>&1` };
+
+if ($version !~ m/\/apisix-nginx-module/) {
+ plan(skip_all => "apisix-nginx-module not installed");
+} else {
+ plan('no_plan');
+}
+
+$ENV{TEST_NGINX_REDIS_PORT} ||= 1985;
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!$block->extra_yaml_config) {
+ my $extra_yaml_config = <<_EOC_;
+stream_plugins:
+ - prometheus
+xrpc:
+ protocols:
+ - name: redis
+_EOC_
+ $block->set_value("extra_yaml_config", $extra_yaml_config);
+ }
+
+ if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
+ $block->set_value("no_error_log", "[error]\nRPC is not finished");
+ }
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ $block;
+});
+
+worker_connections(1024);
+run_tests;
+
+__DATA__
+
+=== TEST 1: route with metrics
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ {
+ uri = "/apisix/prometheus/metrics",
+ plugins = {
+ ["public-api"] = {}
+ }
+ }
+ )
+ if code >= 300 then
+ ngx.status = code
+ ngx.say(body)
+ return
+ end
+
+ local code, body = t('/apisix/admin/stream_routes/1',
+ ngx.HTTP_PUT,
+ {
+ protocol = {
+ name = "redis",
+ conf = {
+ faults = {
+ {delay = 0.08, commands = {"hmset"}},
+ {delay = 0.3, commands = {"hmget"}},
+ }
+ },
+ metric = {
+ enable = true,
+ }
+ },
+ upstream = {
+ nodes = {
+ ["127.0.0.1:6379"] = 1
+ },
+ type = "roundrobin"
+ }
+ }
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 2: hit
+--- config
+ location /t {
+ content_by_lua_block {
+ local redis = require "resty.redis"
+ local red = redis:new()
+
+ local ok, err = red:connect("127.0.0.1", $TEST_NGINX_REDIS_PORT)
+ if not ok then
+ ngx.say("failed to connect: ", err)
+ return
+ end
+
+ local res, err = red:hmset("animals", "dog", "bark", "cat", "meow")
+ if not res then
+ ngx.say("failed to set animals: ", err)
+ return
+ end
+
+ local res, err = red:hmget("animals", "dog", "cat")
+ if not res then
+ ngx.say("failed to get animals: ", err)
+ return
+ end
+ }
+ }
+--- response_body
+--- stream_conf_enable
+
+
+
+=== TEST 3: check metrics
+--- request
+GET /apisix/prometheus/metrics
+--- response_body eval
+qr/apisix_redis_commands_latency_seconds_bucket\{route="1",command="hmget",le="0.5"\} 1/ and
+qr/apisix_redis_commands_latency_seconds_bucket\{route="1",command="hmset",le="0.1"\} 1/ and
+qr/apisix_redis_commands_total\{route="1",command="hmget"\} 1
+apisix_redis_commands_total\{route="1",command="hmset"\} 1/
+
+
+
+=== TEST 4: ignore metric if prometheus is disabled
+--- config
+ location /t {
+ content_by_lua_block {
+ local redis = require "resty.redis"
+ local red = redis:new()
+
+ local ok, err = red:connect("127.0.0.1", $TEST_NGINX_REDIS_PORT)
+ if not ok then
+ ngx.say("failed to connect: ", err)
+ return
+ end
+
+ local res, err = red:hmset("animals", "dog", "bark", "cat", "meow")
+ if not res then
+ ngx.say("failed to set animals: ", err)
+ return
+ end
+ }
+ }
+--- response_body
+--- extra_yaml_config
+stream_plugins:
+ - ip-restriction
+xrpc:
+ protocols:
+ - name: redis
+--- stream_conf_enable
+
+
+
+=== TEST 5: check metrics
+--- request
+GET /apisix/prometheus/metrics
+--- response_body eval
+qr/apisix_redis_commands_total\{route="1",command="hmset"\} 1/
+
+
+
+=== TEST 6: ignore metric if metric is disabled
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/stream_routes/1',
+ ngx.HTTP_PUT,
+ {
+ protocol = {
+ name = "redis",
+ conf = {
+ faults = {
+ {delay = 0.08, commands = {"hmset"}},
+ {delay = 0.3, commands = {"hmget"}},
+ }
+ },
+ metric = {
+ enable = false
+ }
+ },
+ upstream = {
+ nodes = {
+ ["127.0.0.1:6379"] = 1
+ },
+ type = "roundrobin"
+ }
+ }
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 7: hit
+--- config
+ location /t {
+ content_by_lua_block {
+ local redis = require "resty.redis"
+ local red = redis:new()
+
+ local ok, err = red:connect("127.0.0.1", $TEST_NGINX_REDIS_PORT)
+ if not ok then
+ ngx.say("failed to connect: ", err)
+ return
+ end
+
+ local res, err = red:hmset("animals", "dog", "bark", "cat", "meow")
+ if not res then
+ ngx.say("failed to set animals: ", err)
+ return
+ end
+ }
+ }
+--- response_body
+--- stream_conf_enable
+
+
+
+=== TEST 8: check metrics
+--- request
+GET /apisix/prometheus/metrics
+--- response_body eval
+qr/apisix_redis_commands_total\{route="1",command="hmset"\} 1/