You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2020/10/13 06:23:06 UTC
[apisix] branch master updated: feat(http-logger): support for
specified the log formats via admin API (#2309)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 5614a95 feat(http-logger): support for specified the log formats via admin API (#2309)
5614a95 is described below
commit 5614a95758c1569f62eed6eb65cd51a39096cfbb
Author: YuanSheng Wang <me...@gmail.com>
AuthorDate: Tue Oct 13 14:21:54 2020 +0800
feat(http-logger): support for specified the log formats via admin API (#2309)
curl http://****/apisix/admin/plugin_metadata/http-logger -d '
{
"log_format": {
"host": "$host",
"@timestamp": "$time_iso8601",
"client_ip": "$remote_addr"
}
}'
when we enabled plugin http-logger, we will get the message body like:
{"host":"localhost","@timestamp":"2020-09-23T18:29:07-04:00","client_ip":"127.0.0.1","route_id":"1"}
{"host":"localhost","@timestamp":"2020-09-23T18:29:07-04:00","client_ip":"127.0.0.1","route_id":"1"}
---
apisix/plugin.lua | 16 ++
apisix/plugins/http-logger.lua | 89 +++++++++--
bin/apisix | 2 +-
doc/zh-cn/plugins/http-logger.md | 28 +++-
...-logger-new-line.t => http-logger-log-format.t} | 164 ++++++---------------
t/plugin/http-logger-new-line.t | 83 +++++++++++
6 files changed, 249 insertions(+), 133 deletions(-)
diff --git a/apisix/plugin.lua b/apisix/plugin.lua
index f992dc3..3077b94 100644
--- a/apisix/plugin.lua
+++ b/apisix/plugin.lua
@@ -25,6 +25,7 @@ local type = type
local local_plugins = core.table.new(32, 0)
local ngx = ngx
local tostring = tostring
+local error = error
local local_plugins_hash = core.table.new(0, 32)
local stream_local_plugins = core.table.new(32, 0)
local stream_local_plugins_hash = core.table.new(0, 32)
@@ -352,6 +353,21 @@ end
function _M.init_worker()
_M.load()
+
+ local plugin_metadatas, err = core.config.new("/plugin_metadata",
+ {automatic = true}
+ )
+ if not plugin_metadatas then
+ error("failed to create etcd instance for fetching /plugin_metadatas : "
+ .. err)
+ end
+
+ _M.plugin_metadatas = plugin_metadatas
+end
+
+
+function _M.plugin_metadata(name)
+ return _M.plugin_metadatas:get(name)
end
diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua
index 4694b60..b18efba 100644
--- a/apisix/plugins/http-logger.lua
+++ b/apisix/plugins/http-logger.lua
@@ -14,16 +14,26 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-local core = require("apisix.core")
-local log_util = require("apisix.utils.log-util")
+
local batch_processor = require("apisix.utils.batch-processor")
-local plugin_name = "http-logger"
-local ngx = ngx
+local log_util = require("apisix.utils.log-util")
+local core = require("apisix.core")
+local http = require("resty.http")
+local url = require("net.url")
+local plugin = require("apisix.plugin")
+local ngx = ngx
local tostring = tostring
-local http = require "resty.http"
-local url = require "net.url"
-local buffers = {}
+local pairs = pairs
local ipairs = ipairs
+local str_byte = string.byte
+
+
+local plugin_name = "http-logger"
+local buffers = {}
+local lru_log_format = core.lrucache.new({
+ ttl = 300, count = 512
+})
+
local schema = {
type = "object",
@@ -45,11 +55,28 @@ local schema = {
}
+local metadata_schema = {
+ type = "object",
+ properties = {
+ log_format = {
+ type = "object",
+ default = {
+ ["host"] = "$host",
+ ["@timestamp"] = "$time_iso8601",
+ ["client_ip"] = "$remote_addr",
+ },
+ },
+ },
+ additionalProperties = false,
+}
+
+
local _M = {
version = 0.1,
priority = 410,
name = plugin_name,
schema = schema,
+ metadata_schema = metadata_schema,
}
@@ -117,12 +144,52 @@ local function send_http_data(conf, log_message)
end
-function _M.log(conf)
- local entry = log_util.get_full_log(ngx, conf)
+local function gen_log_format(metadata)
+ local log_format = {}
+ if metadata == nil then
+ return log_format
+ end
+
+ for k, var_name in pairs(metadata.value.log_format) do
+ if var_name:byte(1, 1) == str_byte("/") then
+ log_format[k] = {true, var_name:sub(2)}
+ else
+ log_format[k] = {false, var_name}
+ end
+ end
+ core.log.info("log_format: ", core.json.delay_encode(log_format))
+ return log_format
+end
+
+
+function _M.log(conf, ctx)
+ local metadata = plugin.plugin_metadata(plugin_name)
+ core.log.info("metadata: ", core.json.delay_encode(metadata))
+
+ local entry
+ local log_format = lru_log_format(metadata or "", nil, gen_log_format,
+ metadata)
+ if core.table.nkeys(log_format) > 0 then
+ entry = core.table.new(0, core.table.nkeys(log_format))
+ for k, var_attr in pairs(log_format) do
+ if var_attr[1] then
+ entry[k] = ctx.var[var_attr[2]]
+ else
+ entry[k] = var_attr[2]
+ end
+ end
+
+ local matched_route = ctx.matched_route and ctx.matched_route.value
+ if matched_route then
+ entry.service_id = matched_route.service_id
+ entry.route_id = matched_route.id
+ end
+ else
+ entry = log_util.get_full_log(ngx, conf)
+ end
if not entry.route_id then
- core.log.error("failed to obtain the route id for http logger")
- return
+ entry.route_id = "no-matched"
end
local log_buffer = buffers[entry.route_id]
diff --git a/bin/apisix b/bin/apisix
index dddeaa7..6a44095 100755
--- a/bin/apisix
+++ b/bin/apisix
@@ -1021,7 +1021,7 @@ local function init_etcd(show_output)
for _, dir_name in ipairs({"/routes", "/upstreams", "/services",
"/plugins", "/consumers", "/node_status",
"/ssl", "/global_rules", "/stream_routes",
- "/proto"}) do
+ "/proto", "/plugin_metadata"}) do
local key = (etcd_conf.prefix or "") .. dir_name .. "/"
local base64_encode = require("base64").encode
diff --git a/doc/zh-cn/plugins/http-logger.md b/doc/zh-cn/plugins/http-logger.md
index 7791007..5569432 100644
--- a/doc/zh-cn/plugins/http-logger.md
+++ b/doc/zh-cn/plugins/http-logger.md
@@ -51,7 +51,7 @@
## 如何开启
-1. 这是有关如何为特定路由启用 http-logger 插件的示例。
+这是有关如何为特定路由启用 http-logger 插件的示例。
```shell
curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
@@ -82,6 +82,32 @@ HTTP/1.1 200 OK
hello, world
```
+## 插件元数据设置
+
+| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
+| ---------------- | ------- | ------ | ------------- | ------- | ------------------------------------------------ |
+| log_format | object | 可选 | | | 以 Hash 对象方式声明日志格式。对 value 部分，仅支持字符串。如果是以`$`开头，则表明是要获取 [Nginx 内置变量](http://nginx.org/en/docs/varindex.html)。特别的，该设置是全局生效的，意味着指定 log_format 后，将对所有绑定 http-logger 的 Route 或 Service 生效。 |
+
+### 设置日志格式示例
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/plugin_metadata/http-logger -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+ "log_format": {
+ "host": "$host",
+ "@timestamp": "$time_iso8601",
+ "client_ip": "$remote_addr"
+ }
+}'
+```
+
+在日志收集处，将得到类似下面的日志：
+
+```shell
+{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"}
+{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"}
+```
+
## 禁用插件
在插件配置中删除相应的 json 配置以禁用 http-logger。APISIX 插件是热重载的，因此无需重新启动 APISIX：
diff --git a/t/plugin/http-logger-new-line.t b/t/plugin/http-logger-log-format.t
similarity index 50%
copy from t/plugin/http-logger-new-line.t
copy to t/plugin/http-logger-log-format.t
index 14c2daa..d092eeb 100644
--- a/t/plugin/http-logger-new-line.t
+++ b/t/plugin/http-logger-log-format.t
@@ -20,42 +20,40 @@ log_level('info');
repeat_each(1);
no_long_string();
no_root_location();
+
run_tests;
__DATA__
-=== TEST 1: sanity, batch_max_size=1
+=== TEST 1: add plugin metadata
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
- local code, body = t('/apisix/admin/routes/1',
- ngx.HTTP_PUT,
- [[{
- "plugins": {
- "http-logger": {
- "uri": "http://127.0.0.1:1980/log",
- "batch_max_size": 1,
- "max_retry_count": 1,
- "retry_delay": 2,
- "buffer_duration": 2,
- "inactive_timeout": 2,
- "concat_method": "new_line"
+ local code, body = t('/apisix/admin/plugin_metadata/http-logger',
+ ngx.HTTP_PUT,
+ [[{
+ "log_format": {
+ "host": "$host",
+ "@timestamp": "$time_iso8601",
+ "client_ip": "$remote_addr"
+ }
+ }]],
+ [[{
+ "node": {
+ "value": {
+ "log_format": {
+ "host": "$host",
+ "@timestamp": "$time_iso8601",
+ "client_ip": "$remote_addr"
}
- },
- "upstream": {
- "nodes": {
- "127.0.0.1:1982": 1
- },
- "type": "roundrobin"
- },
- "uri": "/hello"
+ }
+ },
+ "action": "set"
}]]
)
- if code >= 300 then
- ngx.status = code
- end
+ ngx.status = code
ngx.say(body)
}
}
@@ -68,20 +66,7 @@ passed
-=== TEST 2: hit route and report http logger
---- request
-GET /hello
---- response_body
-hello world
---- wait: 0.5
---- no_error_log
-[error]
---- error_log
-request log: {"upstream":"127.0.0.1:1982"
-
-
-
-=== TEST 3: sanity, batch_max_size=1
+=== TEST 2: sanity, batch_max_size=1
--- config
location /t {
content_by_lua_block {
@@ -92,11 +77,11 @@ request log: {"upstream":"127.0.0.1:1982"
"plugins": {
"http-logger": {
"uri": "http://127.0.0.1:1980/log",
- "batch_max_size": 3,
- "max_retry_count": 3,
+ "batch_max_size": 1,
+ "max_retry_count": 1,
"retry_delay": 2,
"buffer_duration": 2,
- "inactive_timeout": 1,
+ "inactive_timeout": 2,
"concat_method": "new_line"
}
},
@@ -125,99 +110,38 @@ passed
-=== TEST 4: hit route, and no report log
+=== TEST 3: hit route and report http logger
--- request
GET /hello
--- response_body
hello world
+--- wait: 0.5
--- no_error_log
[error]
-request log:
-
-
-
-=== TEST 5: hit route, and report log
---- config
-location /t {
- content_by_lua_block {
- local t = require("lib.test_admin").test
-
- for i = 1, 6 do
- t('/hello', ngx.HTTP_GET)
- end
-
- ngx.sleep(3)
- ngx.say("done")
- }
-}
---- request
-GET /t
---- timeout: 10
---- no_error_log
-[error]
---- grep_error_log eval
-qr/request log:/
---- grep_error_log_out
-request log:
-request log:
-
-
-
-=== TEST 6: hit route, and report log
---- config
-location /t {
- content_by_lua_block {
- local t = require("lib.test_admin").test
-
- for i = 1, 6 do
- t('/hello', ngx.HTTP_GET)
- end
-
- ngx.sleep(3)
- ngx.say("done")
- }
-}
---- request
-GET /t
---- timeout: 10
---- no_error_log
-[error]
---- grep_error_log eval
-qr/"upstream":"127.0.0.1:1982"/
---- grep_error_log_out
-"upstream":"127.0.0.1:1982"
-"upstream":"127.0.0.1:1982"
-"upstream":"127.0.0.1:1982"
-"upstream":"127.0.0.1:1982"
-"upstream":"127.0.0.1:1982"
-"upstream":"127.0.0.1:1982"
+--- error_log
+request log: {
-=== TEST 7: hit route, and report log
+=== TEST 4: remove plugin metadata
--- config
-location /t {
- content_by_lua_block {
- local t = require("lib.test_admin").test
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/plugin_metadata/http-logger',
+ ngx.HTTP_DELETE
+ )
- for i = 1, 5 do
- t('/hello', ngx.HTTP_GET)
- end
+ if code >= 300 then
+ ngx.status = code
+ end
- ngx.sleep(3)
- ngx.say("done")
+ ngx.say(body)
+ }
}
-}
--- request
GET /t
---- timeout: 10
+--- response_body
+passed
--- no_error_log
[error]
---- grep_error_log eval
-qr/"upstream":"127.0.0.1:1982"/
---- grep_error_log_out
-"upstream":"127.0.0.1:1982"
-"upstream":"127.0.0.1:1982"
-"upstream":"127.0.0.1:1982"
-"upstream":"127.0.0.1:1982"
-"upstream":"127.0.0.1:1982"
diff --git a/t/plugin/http-logger-new-line.t b/t/plugin/http-logger-new-line.t
index 14c2daa..ce5517d 100644
--- a/t/plugin/http-logger-new-line.t
+++ b/t/plugin/http-logger-new-line.t
@@ -221,3 +221,86 @@ qr/"upstream":"127.0.0.1:1982"/
"upstream":"127.0.0.1:1982"
"upstream":"127.0.0.1:1982"
"upstream":"127.0.0.1:1982"
+
+
+
+=== TEST 8: set in global rule
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/global_rules/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "http-logger": {
+ "uri": "http://127.0.0.1:1980/log",
+ "batch_max_size": 3,
+ "max_retry_count": 3,
+ "retry_delay": 2,
+ "buffer_duration": 2,
+ "inactive_timeout": 1,
+ "concat_method": "new_line"
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 9: not hit route, and report log
+--- config
+location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+
+ for i = 1, 5 do
+ t('/not_hit_route', ngx.HTTP_GET)
+ end
+
+ ngx.sleep(3)
+ ngx.say("done")
+ }
+}
+--- request
+GET /t
+--- timeout: 10
+--- no_error_log
+[error]
+
+
+
+=== TEST 10: delete the global rule
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/global_rules/1',
+ ngx.HTTP_DELETE
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]