You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2022/04/11 01:46:24 UTC
[apisix] branch master updated: feat(response-rewrite): support filters (#6750)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new e89ab8d65 feat(response-rewrite): support filters (#6750)
e89ab8d65 is described below
commit e89ab8d654fca554175a86d1af566d6547c5c393
Author: kwanhur <hu...@163.com>
AuthorDate: Mon Apr 11 09:46:18 2022 +0800
feat(response-rewrite): support filters (#6750)
Co-authored-by: 罗泽轩 <sp...@gmail.com>
---
apisix/plugins/response-rewrite.lua | 75 +++-
docs/en/latest/plugins/response-rewrite.md | 21 +-
docs/zh/latest/plugins/response-rewrite.md | 21 +-
t/plugin/response-rewrite2.t | 547 +++++++++++++++++++++++++++++
4 files changed, 649 insertions(+), 15 deletions(-)
diff --git a/apisix/plugins/response-rewrite.lua b/apisix/plugins/response-rewrite.lua
index 7706bc96e..b2c94f2af 100644
--- a/apisix/plugins/response-rewrite.lua
+++ b/apisix/plugins/response-rewrite.lua
@@ -16,10 +16,15 @@
--
local core = require("apisix.core")
local expr = require("resty.expr.v1")
+local re_compile = require("resty.core.regex").re_match_compile
local plugin_name = "response-rewrite"
local ngx = ngx
+local re_sub = ngx.re.sub
+local re_gsub = ngx.re.gsub
local pairs = pairs
+local ipairs = ipairs
local type = type
+local pcall = pcall
local schema = {
@@ -48,6 +53,40 @@ local schema = {
vars = {
type = "array",
},
+ filters = {
+ description = "a group of filters that modify response body" ..
+ "by replacing one specified string by another",
+ type = "array",
+ minItems = 1,
+ items = {
+ description = "filter that modifies response body",
+ type = "object",
+ required = {"regex", "replace"},
+ properties = {
+ regex = {
+ description = "match pattern on response body",
+ type = "string",
+ minLength = 1,
+ },
+ scope = {
+ description = "regex substitution range",
+ type = "string",
+ enum = {"once", "global"},
+ default = "once",
+ },
+ replace = {
+ description = "regex substitution content",
+ type = "string",
+ },
+ options = {
+ description = "regex options",
+ type = "string",
+ default = "jo",
+ }
+ },
+ },
+ },
+ oneOf = {"body", "filters"},
},
minProperties = 1,
}
@@ -115,6 +154,16 @@ function _M.check_schema(conf)
end
end
+ if conf.filters then
+ for _, filter in ipairs(conf.filters) do
+ local ok, err = pcall(re_compile, filter.regex, filter.options)
+ if not ok then
+ return false, "regex \"" .. filter.regex ..
+ "\" validation failed: " .. err
+ end
+ end
+ end
+
return true
end
@@ -126,6 +175,29 @@ function _M.body_filter(conf, ctx)
return
end
+ if conf.filters then
+
+ local body = core.response.hold_body_chunk(ctx)
+ if not body then
+ return
+ end
+
+ local err
+ for _, filter in ipairs(conf.filters) do
+ if filter.scope == "once" then
+ body, _, err = re_sub(body, filter.regex, filter.replace, filter.options)
+ else
+ body, _, err = re_gsub(body, filter.regex, filter.replace, filter.options)
+ end
+ if err ~= nil then
+ core.log.error("regex \"" .. filter.regex .. "\" substitutes failed:" .. err)
+ end
+ end
+
+ ngx.arg[1] = body
+ return
+ end
+
if conf.body then
if conf.body_base64 then
@@ -148,7 +220,8 @@ function _M.header_filter(conf, ctx)
ngx.status = conf.status_code
end
- if conf.body then
+ -- if filters have no any match, response body won't be modified.
+ if conf.filters or conf.body then
core.response.clear_header_as_body_modified()
end
diff --git a/docs/en/latest/plugins/response-rewrite.md b/docs/en/latest/plugins/response-rewrite.md
index 6920e2643..a32ccfed3 100644
--- a/docs/en/latest/plugins/response-rewrite.md
+++ b/docs/en/latest/plugins/response-rewrite.md
@@ -32,13 +32,20 @@ response rewrite plugin, rewrite the content returned by the upstream as well as
## Attributes
-| Name | Type | Requirement | Default | Valid | Description |
-| ----------- | ------- | ----------- | ------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| status_code | integer | optional | | [200, 598] | New `status code` to client, keep the original response code by default. |
-| body | string | optional | | | New `body` to client, and the content-length will be reset too. |
-| body_base64 | boolean | optional | false | | Identify if `body` in configuration need base64 decoded before rewrite to client. |
-| headers | object | optional | | | Set the new `headers` for client, can set up multiple. If it exists already from upstream, will rewrite the header, otherwise will add the header. You can set the corresponding value to an empty string to remove a header. The value can contain Nginx variables in `$var` format, like `$remote_addr $balancer_ip` |
-| vars | array[] | optional | | | A DSL to evaluate with the given ngx.var. See `vars` [lua-resty-expr](https://github.com/api7/lua-resty-expr#operator-list). if the `vars` is empty, then all rewrite operations will be executed unconditionally |
+| Name | Type | Requirement | Default | Valid | Description |
+|-----------------|---------|-------------|---------|-----------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| status_code | integer | optional | | [200, 598] | New `status code` to client, keep the original response code by default. |
+| body | string | optional | | | New `body` to client, and the content-length will be reset too. |
+| body_base64 | boolean | optional | false | | Identify if `body` in configuration need base64 decoded before rewrite to client. |
+| headers | object | optional | | | Set the new `headers` for client, can set up multiple. If it exists already from upstream, will rewrite the header, otherwise will add the header. You can set the corresponding value to an empty string to remove a header. The value can contain Nginx variables in `$var` format, like `$remote_addr $balancer_ip`. |
+| vars | array[] | optional | | | A DSL to evaluate with the given ngx.var. See `vars` [lua-resty-expr](https://github.com/api7/lua-resty-expr#operator-list). if the `vars` is empty, then all rewrite operations will be executed unconditionally. |
+| filters | array[] | optional | | | A group of filters that modify response body by replacing one specified string by another. |
+| filters.regex | string | required | | | match pattern on response body. |
+| filters.scope | string | optional | "once" | "once","global" | substitution range, "once" substitutes the first match of `filters.regex` on response body, "global" does global substitution. |
+| filters.replace | string | required | | | substitution content. |
+| filters.options | string | optional | "jo" | | regex options, See [ngx.re.match](https://github.com/openresty/lua-nginx-module#ngxrematch). |
+
+Only one of `body`, `filters` can be specified.
## How To Enable
diff --git a/docs/zh/latest/plugins/response-rewrite.md b/docs/zh/latest/plugins/response-rewrite.md
index 7cee59665..15e77c9fa 100644
--- a/docs/zh/latest/plugins/response-rewrite.md
+++ b/docs/zh/latest/plugins/response-rewrite.md
@@ -33,13 +33,20 @@ title: response-rewrite
## 属性
-| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
-| ----------- | ------- | ------ | ------ | ---------- | -------------------------------------------------------------------------------------------------------------------------------------- |
-| status_code | integer | 可选 | | [200, 598] | 修改上游返回状态码，默认保留原始响应代码。 |
-| body | string | 可选 | | | 修改上游返回的 `body` 内容，如果设置了新内容，header 里面的 content-length 字段也会被去掉 |
-| body_base64 | boolean | 可选 | false | | 描述 `body` 字段是否需要 base64 解码之后再返回给客户端，用在某些图片和 Protobuffer 场景 |
-| headers | object | 可选 | | | 返回给客户端的 `headers`，这里可以设置多个。头信息如果存在将重写，不存在则添加。想要删除某个 header 的话，把对应的值设置为空字符串即可。这个值能够以 `$var` 的格式包含 Nginx 变量，比如 `$remote_addr $balancer_ip` |
-| vars | array[] | 可选 | | | `vars` 是一个表达式列表，只有满足条件的请求和响应才会修改 body 和 header 信息，来自 [lua-resty-expr](https://github.com/api7/lua-resty-expr#operator-list)。如果 `vars` 字段为空，那么所有的重写动作都会被无条件的执行。 |
+| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
+|-----------------|---------|-----|--------|-----------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| status_code | integer | 可选 | | [200, 598] | 修改上游返回状态码，默认保留原始响应代码。 |
+| body | string | 可选 | | | 修改上游返回的 `body` 内容，如果设置了新内容，header 里面的 content-length 字段也会被去掉。 |
+| body_base64 | boolean | 可选 | false | | 描述 `body` 字段是否需要 base64 解码之后再返回给客户端，用在某些图片和 Protobuffer 场景。 |
+| headers | object | 可选 | | | 返回给客户端的 `headers`，这里可以设置多个。头信息如果存在将重写，不存在则添加。想要删除某个 header 的话，把对应的值设置为空字符串即可。这个值能够以 `$var` 的格式包含 Nginx 变量，比如 `$remote_addr $balancer_ip`。 |
+| vars | array[] | 可选 | | | `vars` 是一个表达式列表，只有满足条件的请求和响应才会修改 body 和 header 信息，来自 [lua-resty-expr](https://github.com/api7/lua-resty-expr#operator-list)。如果 `vars` 字段为空，那么所有的重写动作都会被无条件的执行。 |
+| filters | array[] | 可选 | | | 一组过滤器，采用指定字符串表达式修改响应体。 |
+| filters.regex | string | 必选 | | | 用于匹配响应体正则表达式。 |
+| filters.scope | string | 可选 | "once" | "once","global" | 替换范围，"once" 表达式 `filters.regex` 仅替换首次匹配上响应体的内容，"global" 则进行全局替换。 |
+| filters.replace | string | 必选 | | | 替换后的内容。 |
+| filters.options | string | 可选 | "jo" | | 正则匹配有效参数，可选项见 [ngx.re.match](https://github.com/openresty/lua-nginx-module#ngxrematch)。 |
+
+`body` 和 `filters`，两个只能配置其中一个。
## 示例
diff --git a/t/plugin/response-rewrite2.t b/t/plugin/response-rewrite2.t
new file mode 100644
index 000000000..88712888a
--- /dev/null
+++ b/t/plugin/response-rewrite2.t
@@ -0,0 +1,547 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+BEGIN {
+ if ($ENV{TEST_NGINX_CHECK_LEAK}) {
+ $SkipReason = "unavailable for the hup tests";
+
+ } else {
+ $ENV{TEST_NGINX_USE_HUP} = 1;
+ undef $ENV{TEST_NGINX_USE_STAP};
+ }
+}
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+run_tests;
+
+__DATA__
+
+=== TEST 1: add plugin with valid filters
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.response-rewrite")
+ local ok, err = plugin.check_schema({
+ filters = {
+ {
+ regex = "Hello",
+ scope = "global",
+ replace = "World",
+ options = "jo"
+ }
+ }
+ })
+ if not ok then
+ ngx.say(err)
+ end
+
+ ngx.say("done")
+ }
+ }
+--- request
+GET /t
+--- response_body
+done
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: add plugin with invalid filter required filed
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.response-rewrite")
+ local ok, err = plugin.check_schema({
+ filters = {
+ {
+ regex = "Hello",
+ }
+ }
+ })
+ if not ok then
+ ngx.say(err)
+ else
+ ngx.say("done")
+ end
+ }
+ }
+--- request
+GET /t
+--- response_body
+property "filters" validation failed: failed to validate item 1: property "replace" is required
+--- no_error_log
+[error]
+
+
+
+=== TEST 3: add plugin with invalid filter scope
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.response-rewrite")
+ local ok, err = plugin.check_schema({
+ filters = {
+ {
+ regex = "Hello",
+ scope = "two",
+ replace = "World",
+ options = "jo"
+ }
+ }
+ })
+ if not ok then
+ ngx.say(err)
+ else
+ ngx.say("done")
+ end
+ }
+ }
+--- request
+GET /t
+--- response_body
+property "filters" validation failed: failed to validate item 1: property "scope" validation failed: matches none of the enum values
+--- no_error_log
+[error]
+
+
+
+=== TEST 4: add plugin with invalid filter empty value
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.response-rewrite")
+ local ok, err = plugin.check_schema({
+ filters = {
+ {
+ regex = "",
+ replace = "world"
+ }
+ }
+ })
+ if not ok then
+ ngx.say(err)
+ else
+ ngx.say("done")
+ end
+ }
+ }
+--- request
+GET /t
+--- response_body
+property "filters" validation failed: failed to validate item 1: property "regex" validation failed: string too short, expected at least 1, got 0
+--- no_error_log
+[error]
+
+
+
+=== TEST 5: add plugin with invalid filter regex options
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.response-rewrite")
+ local ok, err = plugin.check_schema({
+ filters = {
+ {
+ regex = "hello",
+ replace = "HELLO",
+ options = "h"
+ }
+ }
+ })
+ if not ok then
+ ngx.say(err)
+ else
+ ngx.say("done")
+ end
+ }
+ }
+--- request
+GET /t
+--- error_code eval
+200
+--- response_body
+regex "hello" validation failed: unknown flag "h" (flags "h")
+--- no_error_log
+[error]
+
+
+
+=== TEST 6: set route with filters and vars expr
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "response-rewrite": {
+ "vars": [
+ ["status","==",200]
+ ],
+ "filters": [
+ {
+ "regex": "hello",
+ "replace": "test"
+ }
+ ]
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uris": ["/hello"]
+ }]]
+ )
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 7: check http body that matches filters
+--- request
+GET /hello
+--- response_body
+test world
+
+
+
+=== TEST 8: filter substitute global
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "response-rewrite": {
+ "vars": [
+ ["status","==",200]
+ ],
+ "filters": [
+ {
+ "regex": "l",
+ "replace": "t",
+ "scope": "global"
+ }
+ ]
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uris": ["/hello"]
+ }]]
+ )
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 9: check http body that substitute global
+--- request
+GET /hello
+--- response_body
+hetto wortd
+
+
+
+=== TEST 10: filter replace with empty
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "response-rewrite": {
+ "vars": [
+ ["status","==",200]
+ ],
+ "filters": [
+ {
+ "regex": "hello",
+ "replace": ""
+ }
+ ]
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uris": ["/hello"]
+ }]]
+ )
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 11: check http body that replace with empty
+--- request
+GET /hello
+--- response_body
+ world
+
+
+
+=== TEST 12: filter replace with words
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "response-rewrite": {
+ "vars": [
+ ["status","==",200]
+ ],
+ "filters": [
+ {
+ "regex": "\\w\\S+$",
+ "replace": "*"
+ }
+ ]
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uris": ["/hello"]
+ }]]
+ )
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 13: check http body that replace with words
+--- request
+GET /hello
+--- response_body
+hello *
+
+
+
+=== TEST 14: set body and filters(body no effect)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "response-rewrite": {
+ "vars": [
+ ["status","==",200]
+ ],
+ "body": "new body",
+ "filters": [
+ {
+ "regex": "hello",
+ "replace": "HELLO"
+ }
+ ]
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uris": ["/hello"]
+ }]]
+ )
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 15: check http body that set body and filters
+--- request
+GET /hello
+--- response_body
+HELLO world
+
+
+
+=== TEST 16: set multiple filters
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "response-rewrite": {
+ "vars": [
+ ["status","==",200]
+ ],
+ "filters": [
+ {
+ "regex": "hello",
+ "replace": "HELLO"
+ },
+ {
+ "regex": "L",
+ "replace": "T"
+ }
+ ]
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uris": ["/hello"]
+ }]]
+ )
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 17: check http body that set multiple filters
+--- request
+GET /hello
+--- response_body
+HETLO world
+
+
+
+=== TEST 18: filters no any match
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "response-rewrite": {
+ "vars": [
+ ["status","==",200]
+ ],
+ "filters": [
+ {
+ "regex": "test",
+ "replace": "TEST"
+ }
+ ]
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uris": ["/hello"]
+ }]]
+ )
+
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 19: check http body that filters no any match
+--- request
+GET /hello
+--- response_body
+hello world