You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2020/10/26 13:51:02 UTC
[apisix] branch master updated: change(batch-requests): limit the
body size to 1M (#2507)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 2a66762 change(batch-requests): limit the body size to 1M (#2507)
2a66762 is described below
commit 2a66762ca6719ca7d8334f2d1428b42218263845
Author: 罗泽轩 <sp...@gmail.com>
AuthorDate: Mon Oct 26 21:50:51 2020 +0800
change(batch-requests): limit the body size to 1M (#2507)
To avoid blocking on reading large body file and
spending too much memory in encode JSON body.
---
apisix/core/request.lua | 54 ++++++++-
apisix/plugins/batch-requests.lua | 72 ++++++------
doc/plugins/batch-requests.md | 21 ++++
doc/zh-cn/plugins/batch-requests.md | 19 ++++
t/lib/test_admin.lua | 11 +-
t/plugin/batch-requests.t | 215 +++++++++++++++++++++++++++++++++++-
6 files changed, 354 insertions(+), 38 deletions(-)
diff --git a/apisix/core/request.lua b/apisix/core/request.lua
index fcf6e07..2e5af94 100644
--- a/apisix/core/request.lua
+++ b/apisix/core/request.lua
@@ -16,12 +16,15 @@
--
local lfs = require("lfs")
+local log = require("apisix.core.log")
local ngx = ngx
local get_headers = ngx.req.get_headers
+local clear_header = ngx.req.clear_header
local tonumber = tonumber
local error = error
local type = type
local str_fmt = string.format
+local str_lower = string.lower
local io_open = io.open
local req_read_body = ngx.req.read_body
local req_get_body_data = ngx.req.get_body_data
@@ -116,11 +119,52 @@ local function get_file(file_name)
end
-function _M.get_body(max_size)
+local function check_size(size, max_size)
+ if max_size and size > max_size then
+ return nil, "request size " .. size .. " is greater than the "
+ .. "maximum size " .. max_size .. " allowed"
+ end
+
+ return true
+end
+
+
+local function test_expect(var)
+ local expect = var.http_expect
+ return expect and str_lower(expect) == "100-continue"
+end
+
+
+function _M.get_body(max_size, ctx)
+ -- TODO: improve the check with set client_max_body dynamically
+ -- which requires to change Nginx source code
+ if max_size then
+ local var = ctx and ctx.var or ngx.var
+ local content_length = tonumber(var.http_content_length)
+ if content_length then
+ local ok, err = check_size(content_length, max_size)
+ if not ok then
+ -- When client_max_body_size is exceeded, Nginx will set r->expect_tested = 1 to
+ -- avoid sending the 100 CONTINUE.
+ -- We use trick below to imitate this behavior.
+ if test_expect(var) then
+ clear_header("expect")
+ end
+
+ return nil, err
+ end
+ end
+ end
+
req_read_body()
local req_body = req_get_body_data()
if req_body then
+ local ok, err = check_size(#req_body, max_size)
+ if not ok then
+ return nil, err
+ end
+
return req_body
end
@@ -129,15 +173,17 @@ function _M.get_body(max_size)
return nil
end
+ log.info("attempt to read body from file: ", file_name)
+
if max_size then
local size, err = lfs.attributes (file_name, "size")
if not size then
return nil, err
end
- if size > max_size then
- return nil, "request size " .. size .. " is greater than the "
- .. "maximum size " .. max_size .. " allowed"
+ local ok, err = check_size(size, max_size)
+ if not ok then
+ return nil, err
end
end
diff --git a/apisix/plugins/batch-requests.lua b/apisix/plugins/batch-requests.lua
index 7187821..dfe0dfd 100644
--- a/apisix/plugins/batch-requests.lua
+++ b/apisix/plugins/batch-requests.lua
@@ -16,8 +16,8 @@
--
local core = require("apisix.core")
local http = require("resty.http")
+local plugin = require("apisix.plugin")
local ngx = ngx
-local io_open = io.open
local ipairs = ipairs
local pairs = pairs
local str_find = string.find
@@ -31,6 +31,20 @@ local schema = {
additionalProperties = false,
}
+local default_max_body_size = 1024 * 1024 -- 1MiB
+local metadata_schema = {
+ type = "object",
+ properties = {
+ max_body_size = {
+ description = "max pipeline body size in bytes",
+ type = "integer",
+ exclusiveMinimum = 0,
+ default = default_max_body_size,
+ },
+ },
+ additionalProperties = false,
+}
+
local req_schema = {
type = "object",
properties = {
@@ -95,7 +109,8 @@ local _M = {
version = 0.1,
priority = 4010,
name = plugin_name,
- schema = schema
+ schema = schema,
+ metadata_schema = metadata_schema,
}
@@ -178,51 +193,46 @@ local function set_common_query(data)
end
-local function get_file(file_name)
- local f = io_open(file_name, 'r')
- if f then
- local req_body = f:read("*all")
- f:close()
- return req_body
- end
-
- return
-end
+local function batch_requests(ctx)
+ local metadata = plugin.plugin_metadata(plugin_name)
+ core.log.info("metadata: ", core.json.delay_encode(metadata))
+ local max_body_size
+ if metadata then
+ max_body_size = metadata.value.max_body_size
+ else
+ max_body_size = default_max_body_size
+ end
-local function batch_requests()
- ngx.req.read_body()
- local req_body = ngx.req.get_body_data()
+ local req_body, err = core.request.get_body(max_body_size, ctx)
+ if err then
+ -- Nginx doesn't support 417: https://trac.nginx.org/nginx/ticket/2062
+ -- So always return 413 instead
+ return 413, { error_msg = err }
+ end
if not req_body then
- local file_name = ngx.req.get_body_file()
- if file_name then
- req_body = get_file(file_name)
- end
-
- if not req_body then
- core.response.exit(400, {
- error_msg = "no request body, you should give at least one pipeline setting"
- })
- end
+ return 400, {
+ error_msg = "no request body, you should give at least one pipeline setting"
+ }
end
local data, err = core.json.decode(req_body)
if not data then
- core.response.exit(400, {
+ return 400, {
error_msg = "invalid request body: " .. req_body .. ", err: " .. err
- })
+ }
end
local code, body = check_input(data)
if code then
- core.response.exit(code, body)
+ return code, body
end
local httpc = http.new()
httpc:set_timeout(data.timeout)
local ok, err = httpc:connect("127.0.0.1", ngx.var.server_port)
if not ok then
- core.response.exit(500, {error_msg = "connect to apisix failed: " .. err})
+ return 500, {error_msg = "connect to apisix failed: " .. err}
end
ensure_header_lowercase(data)
@@ -231,7 +241,7 @@ local function batch_requests()
local responses, err = httpc:request_pipeline(data.pipeline)
if not responses then
- core.response.exit(400, {error_msg = "request failed: " .. err})
+ return 400, {error_msg = "request failed: " .. err}
end
local aggregated_resp = {}
@@ -252,7 +262,7 @@ local function batch_requests()
end
core.table.insert(aggregated_resp, sub_resp)
end
- core.response.exit(200, aggregated_resp)
+ return 200, aggregated_resp
end
diff --git a/doc/plugins/batch-requests.md b/doc/plugins/batch-requests.md
index 480d60a..7df2102 100644
--- a/doc/plugins/batch-requests.md
+++ b/doc/plugins/batch-requests.md
@@ -24,6 +24,8 @@
- [**Description**](#Description)
- [**Attributes**](#Attributes)
- [**How To Enable**](#how-to-Enable)
+- [**How To Configure**](#how-to-configure)
+- [**Metadata**](#metadata)
- [**Batch Api Request/Response**](#batch-api-request/response)
- [**Test Plugin**](#test-plugin)
- [**Disable Plugin**](#disable-plugin)
@@ -49,6 +51,25 @@ You may need to use [interceptors](plugin-interceptors.md) to protect it.
Default enabled
+## How To Configure
+
+By default, the maximun body size sent to the `/apisix/batch-requests` can't be larger than 1 MiB.
+You can configure it via `apisix/admin/plugin_metadata/batch-requests`:
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/plugin_metadata/batch-requests -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+ "max_body_size": 4194304
+}'
+```
+
+## Metadata
+
+| Name | Type | Requirement | Default | Valid | Description |
+| ---------------- | ------- | ------ | ------------- | ------- | ------------------------------------------------ |
+| max_body_size | integer | required | 1048576 | > 0 | the maximun of request body size in bytes |
+
+
## Batch API Request/Response
The plugin will create a API in `apisix` to handle your batch request.
diff --git a/doc/zh-cn/plugins/batch-requests.md b/doc/zh-cn/plugins/batch-requests.md
index cdf3cef..e73c43f 100644
--- a/doc/zh-cn/plugins/batch-requests.md
+++ b/doc/zh-cn/plugins/batch-requests.md
@@ -24,6 +24,8 @@
- [**简介**](#简介)
- [**属性**](#属性)
- [**如何启用**](#如何启用)
+- [**如何配置**](#如何配置)
+- [**元数据**](#元数据)
- [**批量接口请求/响应**](#批量接口请求/响应)
- [**测试插件**](#测试插件)
- [**禁用插件**](#禁用插件)
@@ -49,6 +51,23 @@
本插件默认启用。
+## 如何配置
+
+默认本插件限制请求体的大小不能大于 1 MiB。这个限制可以通过 `apisix/admin/plugin_metadata/batch-requests` 来修改。
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/plugin_metadata/batch-requests -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+ "max_body_size": 4194304
+}'
+```
+
+## 元数据
+
+| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
+| ---------------- | ------- | ------ | ------------- | ------- | ------------------------------------------------ |
+| max_body_size | integer | 必选 | 1048576 | > 0 | 请求体的最大大小，单位为字节。|
+
## 批量接口请求/响应
插件会为 `apisix` 创建一个 `/apisix/batch-requests` 的接口来处理你的批量请求。
diff --git a/t/lib/test_admin.lua b/t/lib/test_admin.lua
index ac6d22e..bba93cd 100644
--- a/t/lib/test_admin.lua
+++ b/t/lib/test_admin.lua
@@ -105,13 +105,20 @@ end
function _M.comp_tab(left_tab, right_tab)
+ local err
dir_names = {}
if type(left_tab) == "string" then
- left_tab = json.decode(left_tab)
+ left_tab, err = json.decode(left_tab)
+ if not left_tab then
+ return false, "failed to decode expected data: " .. err
+ end
end
if type(right_tab) == "string" then
- right_tab = json.decode(right_tab)
+ right_tab, err = json.decode(right_tab)
+ if not right_tab then
+ return false, "failed to decode expected data: " .. err
+ end
end
local ok, err = com_tab(left_tab, right_tab)
diff --git a/t/plugin/batch-requests.t b/t/plugin/batch-requests.t
index c409cee..e438a1c 100644
--- a/t/plugin/batch-requests.t
+++ b/t/plugin/batch-requests.t
@@ -19,7 +19,7 @@ use t::APISIX 'no_plan';
repeat_each(1);
no_long_string();
no_root_location();
-log_level("info");
+log_level("debug");
run_tests;
@@ -778,3 +778,216 @@ GET /aggregate
passed
--- no_error_log
[error]
+
+
+
+=== TEST 16: exceed default body limit size (check header)
+--- config
+ location = /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/batch-requests',
+ ngx.HTTP_POST,
+ ("1234"):rep(1024 * 1024)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 413
+--- response_body eval
+qr/\{"error_msg":"request size 4194304 is greater than the maximum size 1048576 allowed"\}/
+--- no_error_log
+[error]
+
+
+
+=== TEST 17: exceed default body limit size (check file size)
+--- request eval
+"POST /apisix/batch-requests
+" . ("1000\r
+" . ("11111111" x 512) . "\r
+") x 257 . "0\r
+\r
+"
+--- more_headers
+Transfer-Encoding: chunked
+--- error_code: 413
+--- response_body eval
+qr/\{"error_msg":"request size 1052672 is greater than the maximum size 1048576 allowed"\}/
+--- no_error_log
+[error]
+--- error_log
+attempt to read body from file
+
+
+
+=== TEST 18: add plugin metadata
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/plugin_metadata/batch-requests',
+ ngx.HTTP_PUT,
+ [[{
+ "max_body_size": 2048
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 19: exceed body limit size
+--- config
+ location = /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/batch-requests',
+ ngx.HTTP_POST,
+ ("1234"):rep(1024)
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 413
+--- response_body eval
+qr/\{"error_msg":"request size 4096 is greater than the maximum size 2048 allowed"\}/
+--- no_error_log
+[error]
+
+
+
+=== TEST 20: exceed body limit size (expected)
+--- config
+ location = /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body, res_data = t('/apisix/batch-requests',
+ ngx.HTTP_POST,
+ ("1234"):rep(1024),
+ nil,
+ {EXPECT = "100-CONTINUE", ["content-length"] = 4096}
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 413
+--- response_body eval
+qr/\{"error_msg":"request size 4096 is greater than the maximum size 2048 allowed"\}/
+--- no_error_log
+[error]
+
+
+
+=== TEST 21: don't exceed body limit size
+--- config
+ location = /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/batch-requests',
+ ngx.HTTP_POST,
+ [=[{
+ "headers": {
+ "Base-Header": "base"
+ },
+ "pipeline":[
+ {
+ "path": "/a",
+ "headers": {
+ "Header1": "hello",
+ "Header2": "world"
+ }
+ }
+ ]
+ }]=],
+ [=[[
+ {
+ "status": 200,
+ "body":"A",
+ "headers": {
+ "Base-Header": "base",
+ "X-Res": "a",
+ "X-Header1": "hello",
+ "X-Header2": "world"
+ }
+ }
+ ]]=])
+
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+ location = /a {
+ content_by_lua_block {
+ ngx.status = 200
+ ngx.header["Base-Header"] = ngx.req.get_headers()["Base-Header"]
+ ngx.header["X-Header1"] = ngx.req.get_headers()["Header1"]
+ ngx.header["X-Header2"] = ngx.req.get_headers()["Header2"]
+ ngx.header["X-Res"] = "a"
+ ngx.print("A")
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 22: invalid body size
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/plugin_metadata/batch-requests',
+ ngx.HTTP_PUT,
+ [[{
+ "max_body_size": 0
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body eval
+qr/\{"error_msg":"invalid configuration: property \\"max_body_size\\" validation failed: expected 0 to be sctrictly greater than 0"\}/
+--- no_error_log
+[error]