You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2023/03/01 01:46:57 UTC
[apisix] branch master updated: feat: Support clickhouse-logger plugin log option of request/response body (#8722)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 376ea7f63 feat: Support clickhouse-logger plugin log option of request/response body (#8722)
376ea7f63 is described below
commit 376ea7f635d6cdea73d2e540620845178f9a4f4c
Author: pixelpig <62...@qq.com>
AuthorDate: Wed Mar 1 09:46:46 2023 +0800
feat: Support clickhouse-logger plugin log option of request/response body (#8722)
---
apisix/plugins/clickhouse-logger.lua | 21 +++
docs/en/latest/plugins/clickhouse-logger.md | 4 +
docs/zh/latest/plugins/clickhouse-logger.md | 4 +
t/plugin/clickhouse-logger2.t | 244 ++++++++++++++++++++++++++++
4 files changed, 273 insertions(+)
diff --git a/apisix/plugins/clickhouse-logger.lua b/apisix/plugins/clickhouse-logger.lua
index 2c1208f92..4e6a4c849 100644
--- a/apisix/plugins/clickhouse-logger.lua
+++ b/apisix/plugins/clickhouse-logger.lua
@@ -41,6 +41,22 @@ local schema = {
name = {type = "string", default = "clickhouse logger"},
ssl_verify = {type = "boolean", default = true},
log_format = {type = "object"},
+ include_req_body = {type = "boolean", default = false},
+ include_req_body_expr = {
+ type = "array",
+ minItems = 1,
+ items = {
+ type = "array"
+ }
+ },
+ include_resp_body = {type = "boolean", default = false},
+ include_resp_body_expr = {
+ type = "array",
+ minItems = 1,
+ items = {
+ type = "array"
+ }
+ }
},
oneOf = {
{required = {"endpoint_addr", "user", "password", "database", "logtable"}},
@@ -146,6 +162,11 @@ local function send_http_data(conf, log_message)
end
+function _M.body_filter(conf, ctx)
+ log_util.collect_body(conf, ctx)
+end
+
+
function _M.log(conf, ctx)
local entry = log_util.get_log_entry(plugin_name, conf, ctx)
diff --git a/docs/en/latest/plugins/clickhouse-logger.md b/docs/en/latest/plugins/clickhouse-logger.md
index 96cb50a7e..b7dccd0f3 100644
--- a/docs/en/latest/plugins/clickhouse-logger.md
+++ b/docs/en/latest/plugins/clickhouse-logger.md
@@ -45,6 +45,10 @@ The `clickhouse-logger` Plugin is used to push logs to [ClickHouse](https://clic
| name | string | False | "clickhouse logger" | | Unique identifier for the logger. |
| ssl_verify | boolean | False | true | [true,false] | When set to `true`, verifies SSL. |
| log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
+| include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. |
+| include_req_body_expr | array | False | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. |
+| include_resp_body | boolean | False | false | [false, true] | When set to `true` includes the response body in the log. |
+| include_resp_body_expr | array | False | | | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. |
NOTE: `encrypt_fields = {"password"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields).
diff --git a/docs/zh/latest/plugins/clickhouse-logger.md b/docs/zh/latest/plugins/clickhouse-logger.md
index 8957661f6..f78cf936c 100644
--- a/docs/zh/latest/plugins/clickhouse-logger.md
+++ b/docs/zh/latest/plugins/clickhouse-logger.md
@@ -45,6 +45,10 @@ description: 本文介绍了 API 网关 Apache APISIX 如何使用 clickhouse-lo
| name | string | 否 | "clickhouse logger" | | 标识 logger 的唯一标识符。 |
| ssl_verify | boolean | 否 | true | [true,false] | 当设置为 `true` 时，验证证书。 |
| log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分，仅支持字符串。如果是以 `$` 开头，则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 |
+| include_req_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时，包含请求体。**注意**：如果请求体无法完全存放在内存中，由于 NGINX 的限制，APISIX 无法将它记录下来。|
+| include_req_body_expr | array | 否 | | | 当 `include_req_body` 属性设置为 `true` 时进行过滤。只有当此处设置的表达式计算结果为 `true` 时，才会记录请求体。更多信息，请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 |
+| include_resp_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时，包含响应体。 |
+| include_resp_body_expr | array | 否 | | | 当 `include_resp_body` 属性设置为 `true` 时进行过滤。只有当此处设置的表达式计算结果为 `true` 时才会记录响应体。更多信息，请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。|
注意：schema 中还定义了 `encrypt_fields = {"password"}`，这意味着该字段将会被加密存储在 etcd 中。具体参考 [加密存储字段](../plugin-develop.md#加密存储字段)。
diff --git a/t/plugin/clickhouse-logger2.t b/t/plugin/clickhouse-logger2.t
new file mode 100644
index 000000000..d36ada30f
--- /dev/null
+++ b/t/plugin/clickhouse-logger2.t
@@ -0,0 +1,244 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+no_long_string();
+no_root_location();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: collect response body option
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "clickhouse-logger": {
+ "user": "default",
+ "password": "a",
+ "database": "default",
+ "logtable": "t",
+ "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server",
+ "inactive_timeout":1,
+ "include_resp_body": true
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1982": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/hello"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 2: collect response log
+--- request
+GET /hello
+--- error_log eval
+qr/clickhouse body: .*\{.*"body":"hello world\\n"/
+--- wait: 3
+
+
+
+=== TEST 3: collect response body with eval option
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "clickhouse-logger": {
+ "user": "default",
+ "password": "a",
+ "database": "default",
+ "logtable": "t",
+ "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server",
+ "inactive_timeout":1,
+ "include_resp_body": true,
+ "include_resp_body_expr": [
+ [
+ "arg_foo",
+ "==",
+ "bar"
+ ]
+ ]
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1982": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/hello"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 4: skip collect response log for condition
+--- request
+GET /hello?foo=unknown
+--- no_error_log eval
+qr/clickhouse body: .*\{.*response":\{.*"body":"hello world\\n"/
+--- wait: 3
+
+
+
+=== TEST 5: collect request body log
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "clickhouse-logger": {
+ "user": "default",
+ "password": "a",
+ "database": "default",
+ "logtable": "t",
+ "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server",
+ "inactive_timeout":1,
+ "include_req_body": true
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1982": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/hello"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 6: collect request body with eval option
+--- request
+POST /hello?foo=bar
+{"sample":"hello"}
+--- error_log eval
+qr/clickhouse body: .*\{.*request":\{.*"body":"\{\\"sample\\":\\"hello\\"/
+--- wait: 3
+
+
+
+=== TEST 7: collect request body
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "clickhouse-logger": {
+ "user": "default",
+ "password": "a",
+ "database": "default",
+ "logtable": "t",
+ "endpoint_addr": "http://127.0.0.1:1980/clickhouse_logger_server",
+ "inactive_timeout":1,
+ "include_req_body": true,
+ "include_req_body_expr": [
+ [
+ "arg_foo",
+ "==",
+ "bar"
+ ]
+ ]
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1982": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/hello"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 8: skip collect request body log for condition
+--- request
+POST /hello?foo=unknown
+{"sample":"hello"}
+--- no_error_log eval
+qr/clickhouse body: .*\{.*request":\{.*"body":"\{\\"sample\\":\\"hello\\"/
+--- wait: 3