You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2022/06/20 03:09:10 UTC
[apisix] branch master updated: feat: export some importent params for kafka-client (#7266)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 067b2eb38 feat: export some importent params for kafka-client (#7266)
067b2eb38 is described below
commit 067b2eb389ca0255727ff42ce1a0bc1ab70d5498
Author: mika <mi...@qq.com>
AuthorDate: Mon Jun 20 11:09:04 2022 +0800
feat: export some importent params for kafka-client (#7266)
---
apisix/plugins/kafka-logger.lua | 9 +++++++++
docs/en/latest/plugins/kafka-logger.md | 4 ++++
docs/zh/latest/plugins/kafka-logger.md | 4 ++++
3 files changed, 17 insertions(+)
diff --git a/apisix/plugins/kafka-logger.lua b/apisix/plugins/kafka-logger.lua
index 2947d145e..cb43ae3db 100644
--- a/apisix/plugins/kafka-logger.lua
+++ b/apisix/plugins/kafka-logger.lua
@@ -83,6 +83,11 @@ local schema = {
-- in lua-resty-kafka, cluster_name is defined as number
-- see https://github.com/doujiang24/lua-resty-kafka#new-1
cluster_name = {type = "integer", minimum = 1, default = 1},
+ -- config for lua-resty-kafka, default value is same as lua-resty-kafka
+ producer_batch_num = {type = "integer", minimum = 1, default = 200},
+ producer_batch_size = {type = "integer", minimum = 0, default = 1048576},
+ producer_max_buffering = {type = "integer", minimum = 1, default = 50000},
+ producer_time_linger = {type = "integer", minimum = 1, default = 1}
},
required = {"broker_list", "kafka_topic"}
}
@@ -208,6 +213,10 @@ function _M.log(conf, ctx)
broker_config["request_timeout"] = conf.timeout * 1000
broker_config["producer_type"] = conf.producer_type
broker_config["required_acks"] = conf.required_acks
+ broker_config["batch_num"] = conf.producer_batch_num
+ broker_config["batch_size"] = conf.producer_batch_size
+ broker_config["max_buffering"] = conf.producer_max_buffering
+ broker_config["flush_time"] = conf.producer_time_linger * 1000
local prod, err = core.lrucache.plugin_ctx(lrucache, ctx, nil, create_producer,
broker_list, broker_config, conf.cluster_name)
diff --git a/docs/en/latest/plugins/kafka-logger.md b/docs/en/latest/plugins/kafka-logger.md
index e32f64d12..c2f50b9bb 100644
--- a/docs/en/latest/plugins/kafka-logger.md
+++ b/docs/en/latest/plugins/kafka-logger.md
@@ -47,6 +47,10 @@ For more info on Batch-Processor in Apache APISIX please refer.
| include_resp_body| boolean | optional | false | [false, true] | Whether to include the response body. The response body is included if and only if it is `true`. |
| include_resp_body_expr | array | optional | | | When `include_resp_body` is true, control the behavior based on the result of the [lua-resty-expr](https://github.com/api7/lua-resty-expr) expression. If present, only log the response body when the result is true. |
| cluster_name | integer | optional | 1 | [0,...] | the name of the cluster. When there are two or more kafka clusters, you can specify different names. And this only works with async producer_type.|
+| producer_batch_num | integer | optional | 200 | [1,...] | `batch_num` param in [lua-resty-kafka](https://github.com/doujiang24/lua-resty-kafka), merge message and batch send to server, unit is message count |
+| producer_batch_size | integer | optional | 1048576 | [0,...] | `batch_size` param in [lua-resty-kafka](https://github.com/doujiang24/lua-resty-kafka), unit is byte |
+| producer_max_buffering | integer | optional | 50000 | [1,...] | `max_buffering` param in [lua-resty-kafka](https://github.com/doujiang24/lua-resty-kafka), max buffer size, unit is message count |
+| producer_time_linger | integer | optional | 1 | [1,...] | `flush_time` param in [lua-resty-kafka](https://github.com/doujiang24/lua-resty-kafka), unit is second |
The plugin supports the use of batch processors to aggregate and process entries(logs/data) in a batch. This avoids frequent data submissions by the plugin, which by default the batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. For information or custom batch processor parameter settings, see [Batch-Processor](../batch-processor.md#configuration) configuration section.
diff --git a/docs/zh/latest/plugins/kafka-logger.md b/docs/zh/latest/plugins/kafka-logger.md
index 9257be4f0..302b273f3 100644
--- a/docs/zh/latest/plugins/kafka-logger.md
+++ b/docs/zh/latest/plugins/kafka-logger.md
@@ -47,6 +47,10 @@ title: kafka-logger
| include_resp_body| boolean | 可选 | false | [false, true] | 是否包括响应体。包含响应体，当为`true`。 |
| include_resp_body_expr | array | 可选 | | | 是否采集响体，基于 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 该选项需要开启 `include_resp_body`|
| cluster_name | integer | 可选 | 1 | [0,...] | kafka 集群的名称。当有两个或多个 kafka 集群时，可以指定不同的名称。只适用于 producer_type 是 async 模式。|
+| producer_batch_num | integer | 可选 | 200 | [1,...] | 对应 [lua-resty-kafka](https://github.com/doujiang24/lua-resty-kafka) 中的`batch_num`参数，聚合消息批量提交，单位为消息条数 |
+| producer_batch_size | integer | 可选 | 1048576 | [0,...] | 对应 [lua-resty-kafka](https://github.com/doujiang24/lua-resty-kafka) 中的`batch_size`参数，单位为字节 |
+| producer_max_buffering | integer | 可选 | 50000 | [1,...] | 对应 [lua-resty-kafka](https://github.com/doujiang24/lua-resty-kafka) 中的`max_buffering`参数，最大缓冲区，单位为条 |
+| producer_time_linger | integer | 可选 | 1 | [1,...] | 对应 [lua-resty-kafka](https://github.com/doujiang24/lua-resty-kafka) 中的`flush_time`参数，单位为秒 |
本插件支持使用批处理器来聚合并批量处理条目（日志/数据）。这样可以避免插件频繁地提交数据，默认设置情况下批处理器会每 `5` 秒钟或队列中的数据达到 `1000` 条时提交数据，如需了解或自定义批处理器相关参数设置，请参考 [Batch-Processor](../batch-processor.md#配置) 配置部分。