You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by me...@apache.org on 2020/09/24 02:58:53 UTC
[apisix] branch master updated: feat(http-logger): support to
concat multiple log with separator. (#2286)
This is an automated email from the ASF dual-hosted git repository.
membphis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 3f9685f feat(http-logger): support to concat multiple log with separator. (#2286)
3f9685f is described below
commit 3f9685f823050f3c3324d20ffacafec4e3e70150
Author: YuanSheng Wang <me...@gmail.com>
AuthorDate: Thu Sep 24 10:58:44 2020 +0800
feat(http-logger): support to concat multiple log with separator. (#2286)
---
apisix/plugins/http-logger.lua | 37 ++++---
doc/plugins/http-logger.md | 8 +-
doc/zh-cn/plugins/http-logger.md | 3 +-
t/lib/server.lua | 6 ++
t/plugin/http-logger-new-line.t | 223 +++++++++++++++++++++++++++++++++++++++
5 files changed, 260 insertions(+), 17 deletions(-)
diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua
index 44df6ae..4694b60 100644
--- a/apisix/plugins/http-logger.lua
+++ b/apisix/plugins/http-logger.lua
@@ -23,6 +23,7 @@ local tostring = tostring
local http = require "resty.http"
local url = require "net.url"
local buffers = {}
+local ipairs = ipairs
local schema = {
type = "object",
@@ -36,7 +37,9 @@ local schema = {
buffer_duration = {type = "integer", minimum = 1, default = 60},
inactive_timeout = {type = "integer", minimum = 1, default = 5},
batch_max_size = {type = "integer", minimum = 1, default = 1000},
- include_req_body = {type = "boolean", default = false}
+ include_req_body = {type = "boolean", default = false},
+ concat_method = {type = "string", default = "json",
+ enum = {"json", "new_line"}}
},
required = {"uri"}
}
@@ -110,13 +113,6 @@ local function send_http_data(conf, log_message)
.. "body[" .. httpc_res:read_body() .. "]"
end
- -- keep the connection alive
- ok, err = httpc:set_keepalive(conf.keepalive)
-
- if not ok then
- core.log.debug("failed to keep the connection alive", err)
- end
-
return res, err_msg
end
@@ -139,10 +135,26 @@ function _M.log(conf)
-- Generate a function to be executed by the batch processor
local func = function(entries, batch_max_size)
local data, err
- if batch_max_size == 1 then
- data, err = core.json.encode(entries[1]) -- encode as single {}
- else
- data, err = core.json.encode(entries) -- encode as array [{}]
+ if conf.concat_method == "json" then
+ if batch_max_size == 1 then
+ data, err = core.json.encode(entries[1]) -- encode as single {}
+ else
+ data, err = core.json.encode(entries) -- encode as array [{}]
+ end
+
+ elseif conf.concat_method == "new_line" then
+ if batch_max_size == 1 then
+ data, err = core.json.encode(entries[1]) -- encode as single {}
+ else
+ local t = core.table.new(#entries, 0)
+ for i, entrie in ipairs(entries) do
+ t[i], err = core.json.encode(entrie)
+ if err then
+ break
+ end
+ end
+ data = core.table.concat(t, "\n") -- encode as multiple string
+ end
end
if not data then
@@ -173,4 +185,5 @@ function _M.log(conf)
log_buffer:push(entry)
end
+
return _M
diff --git a/doc/plugins/http-logger.md b/doc/plugins/http-logger.md
index f8c0941..3fd5b9e 100644
--- a/doc/plugins/http-logger.md
+++ b/doc/plugins/http-logger.md
@@ -20,6 +20,7 @@
- [中文](../zh-cn/plugins/http-logger.md)
# Summary
+
- [**Name**](#name)
- [**Attributes**](#attributes)
- [**How To Enable**](#how-to-enable)
@@ -35,7 +36,6 @@ This will provide the ability to send Log data requests as JSON objects to Monit
## Attributes
-
| Name | Type | Requirement | Default | Valid | Description |
| ---------------- | ------- | ----------- | ------------- | ------- | ---------------------------------------------------------------------------------------- |
| uri | string | required | | | URI of the server |
@@ -48,6 +48,7 @@ This will provide the ability to send Log data requests as JSON objects to Monit
| max_retry_count | integer | optional | 0 | [0,...] | Maximum number of retries before removing from the processing pipe line |
| retry_delay | integer | optional | 1 | [0,...] | Number of seconds the process execution should be delayed if the execution fails |
| include_req_body | boolean | optional | false | | Whether to include the request body |
+| concat_method | string | optional | "json" | | Enum type, `json` and `new_line`. **json**: use `json.encode` for all pending logs. **new_line**: use `json.encode` for each pending log and concat them with "\n" line. |
## How To Enable
@@ -58,7 +59,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
{
"plugins": {
"http-logger": {
- "uri": "127.0.0.1:80/postendpoint?param=1",
+ "uri": "127.0.0.1:80/postendpoint?param=1",
}
},
"upstream": {
@@ -73,7 +74,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
## Test Plugin
-* success:
+> success:
```shell
$ curl -i http://127.0.0.1:9080/hello
@@ -90,7 +91,6 @@ APISIX plugins are hot-reloaded, therefore no need to restart APISIX.
```shell
$ curl http://127.0.0.1:2379/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d value='
{
- "methods": ["GET"],
"uri": "/hello",
"plugins": {},
"upstream": {
diff --git a/doc/zh-cn/plugins/http-logger.md b/doc/zh-cn/plugins/http-logger.md
index 7ea7dc1..7791007 100644
--- a/doc/zh-cn/plugins/http-logger.md
+++ b/doc/zh-cn/plugins/http-logger.md
@@ -47,6 +47,7 @@
| max_retry_count | integer | 可选 | 0 | [0,...] | 从处理管道中移除之前的最大重试次数 |
| retry_delay | integer | 可选 | 1 | [0,...] | 如果执行失败，则应延迟执行流程的秒数 |
| include_req_body | boolean | 可选 | | | 是否包括请求 body |
+| concat_method | string | 可选 | "json" | | 枚举类型，`json`、`new_line`。**json**: 对所有待发日志使用 `json.encode` 编码。**new_line**: 对每一条待发日志单独使用 `json.encode` 编码并使用 "\n" 连接起来。 |
## 如何开启
@@ -57,7 +58,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
{
"plugins": {
"http-logger": {
- "uri": "127.0.0.1:80/postendpoint?param=1"
+ "uri": "127.0.0.1:80/postendpoint?param=1"
}
},
"upstream": {
diff --git a/t/lib/server.lua b/t/lib/server.lua
index c9e89e9..3a0edae 100644
--- a/t/lib/server.lua
+++ b/t/lib/server.lua
@@ -282,4 +282,10 @@ function _M.headers()
ngx.say("/headers")
end
+function _M.log()
+ ngx.req.read_body()
+ local body = ngx.req.get_body_data()
+ ngx.log(ngx.WARN, "request log: ", body or "nil")
+end
+
return _M
diff --git a/t/plugin/http-logger-new-line.t b/t/plugin/http-logger-new-line.t
new file mode 100644
index 0000000..14c2daa
--- /dev/null
+++ b/t/plugin/http-logger-new-line.t
@@ -0,0 +1,223 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+log_level('info');
+repeat_each(1);
+no_long_string();
+no_root_location();
+run_tests;
+
+__DATA__
+
+=== TEST 1: sanity, batch_max_size=1
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "http-logger": {
+ "uri": "http://127.0.0.1:1980/log",
+ "batch_max_size": 1,
+ "max_retry_count": 1,
+ "retry_delay": 2,
+ "buffer_duration": 2,
+ "inactive_timeout": 2,
+ "concat_method": "new_line"
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1982": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/hello"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: hit route and report http logger
+--- request
+GET /hello
+--- response_body
+hello world
+--- wait: 0.5
+--- no_error_log
+[error]
+--- error_log
+request log: {"upstream":"127.0.0.1:1982"
+
+
+
+=== TEST 3: sanity, batch_max_size=1
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "http-logger": {
+ "uri": "http://127.0.0.1:1980/log",
+ "batch_max_size": 3,
+ "max_retry_count": 3,
+ "retry_delay": 2,
+ "buffer_duration": 2,
+ "inactive_timeout": 1,
+ "concat_method": "new_line"
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1982": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/hello"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 4: hit route, and no report log
+--- request
+GET /hello
+--- response_body
+hello world
+--- no_error_log
+[error]
+request log:
+
+
+
+=== TEST 5: hit route, and report log
+--- config
+location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+
+ for i = 1, 6 do
+ t('/hello', ngx.HTTP_GET)
+ end
+
+ ngx.sleep(3)
+ ngx.say("done")
+ }
+}
+--- request
+GET /t
+--- timeout: 10
+--- no_error_log
+[error]
+--- grep_error_log eval
+qr/request log:/
+--- grep_error_log_out
+request log:
+request log:
+
+
+
+=== TEST 6: hit route, and report log
+--- config
+location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+
+ for i = 1, 6 do
+ t('/hello', ngx.HTTP_GET)
+ end
+
+ ngx.sleep(3)
+ ngx.say("done")
+ }
+}
+--- request
+GET /t
+--- timeout: 10
+--- no_error_log
+[error]
+--- grep_error_log eval
+qr/"upstream":"127.0.0.1:1982"/
+--- grep_error_log_out
+"upstream":"127.0.0.1:1982"
+"upstream":"127.0.0.1:1982"
+"upstream":"127.0.0.1:1982"
+"upstream":"127.0.0.1:1982"
+"upstream":"127.0.0.1:1982"
+"upstream":"127.0.0.1:1982"
+
+
+
+=== TEST 7: hit route, and report log
+--- config
+location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+
+ for i = 1, 5 do
+ t('/hello', ngx.HTTP_GET)
+ end
+
+ ngx.sleep(3)
+ ngx.say("done")
+ }
+}
+--- request
+GET /t
+--- timeout: 10
+--- no_error_log
+[error]
+--- grep_error_log eval
+qr/"upstream":"127.0.0.1:1982"/
+--- grep_error_log_out
+"upstream":"127.0.0.1:1982"
+"upstream":"127.0.0.1:1982"
+"upstream":"127.0.0.1:1982"
+"upstream":"127.0.0.1:1982"
+"upstream":"127.0.0.1:1982"