You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by me...@apache.org on 2020/08/05 06:11:31 UTC
[apisix] branch master updated: feat: script distribute and run
(#1982)
This is an automated email from the ASF dual-hosted git repository.
membphis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 65df727 feat: script distribute and run (#1982)
65df727 is described below
commit 65df727af4edf91339c79f00d7559ebb2f22158c
Author: nic-chen <33...@users.noreply.github.com>
AuthorDate: Wed Aug 5 14:11:22 2020 +0800
feat: script distribute and run (#1982)
Supporting distribution and execution scripts, we can implement many new features,
such as plug-in orchestration.
---
apisix/admin/routes.lua | 12 +++
apisix/admin/services.lua | 13 ++++
apisix/init.lua | 35 ++++++---
apisix/schema_def.lua | 11 +++
apisix/script.lua | 59 +++++++++++++++
doc/admin-api.md | 1 +
doc/architecture-design.md | 20 +++++
doc/zh-cn/admin-api.md | 11 +--
doc/zh-cn/architecture-design.md | 22 ++++++
t/admin/routes.t | 39 ++++++++++
t/script/script_distribute.t | 158 +++++++++++++++++++++++++++++++++++++++
t/script/script_test.lua | 43 +++++++++++
12 files changed, 407 insertions(+), 17 deletions(-)
diff --git a/apisix/admin/routes.lua b/apisix/admin/routes.lua
index d917435..eac54c4 100644
--- a/apisix/admin/routes.lua
+++ b/apisix/admin/routes.lua
@@ -124,6 +124,18 @@ local function check_conf(id, conf, need_id)
end
end
+ if conf.script then
+ local obj, err = loadstring(conf.script)
+ if not obj then
+ return nil, {error_msg = "failed to load 'script' string: "
+ .. err}
+ end
+
+ if type(obj()) ~= "table" then
+ return nil, {error_msg = "'script' should be a Lua object"}
+ end
+ end
+
return need_id and id or true
end
diff --git a/apisix/admin/services.lua b/apisix/admin/services.lua
index cbce7d1..2200333 100644
--- a/apisix/admin/services.lua
+++ b/apisix/admin/services.lua
@@ -21,6 +21,7 @@ local upstreams = require("apisix.admin.upstreams")
local tostring = tostring
local ipairs = ipairs
local type = type
+local loadstring = loadstring
local _M = {
@@ -91,6 +92,18 @@ local function check_conf(id, conf, need_id)
end
end
+ if conf.script then
+ local obj, err = loadstring(conf.script)
+ if not obj then
+ return nil, {error_msg = "failed to load 'script' string: "
+ .. err}
+ end
+
+ if type(obj()) ~= "table" then
+ return nil, {error_msg = "'script' should be a Lua object"}
+ end
+ end
+
return need_id and id or true
end
diff --git a/apisix/init.lua b/apisix/init.lua
index 29536de..bbb22a7 100644
--- a/apisix/init.lua
+++ b/apisix/init.lua
@@ -18,6 +18,7 @@ local require = require
local core = require("apisix.core")
local config_util = require("apisix.core.config_util")
local plugin = require("apisix.plugin")
+local script = require("apisix.script")
local service_fetch = require("apisix.http.service").get
local admin_init = require("apisix.admin.init")
local get_var = require("resty.ngxvar").fetch
@@ -452,19 +453,24 @@ function _M.http_access_phase()
api_ctx.var.upstream_connection = api_ctx.var.http_connection
end
- local plugins = plugin.filter(route)
- api_ctx.plugins = plugins
-
- run_plugin("rewrite", plugins, api_ctx)
- if api_ctx.consumer then
- local changed
- route, changed = plugin.merge_consumer_route(route, api_ctx.consumer)
- if changed then
- core.table.clear(api_ctx.plugins)
- api_ctx.plugins = plugin.filter(route, api_ctx.plugins)
+ if route.value.script then
+ script.load(route, api_ctx)
+ script.run("access", api_ctx)
+ else
+ local plugins = plugin.filter(route)
+ api_ctx.plugins = plugins
+
+ run_plugin("rewrite", plugins, api_ctx)
+ if api_ctx.consumer then
+ local changed
+ route, changed = plugin.merge_consumer_route(route, api_ctx.consumer)
+ if changed then
+ core.table.clear(api_ctx.plugins)
+ api_ctx.plugins = plugin.filter(route, api_ctx.plugins)
+ end
end
+ run_plugin("access", plugins, api_ctx)
end
- run_plugin("access", plugins, api_ctx)
local ok, err = set_upstream(route, api_ctx)
if not ok then
@@ -553,7 +559,12 @@ local function common_phase(phase_name)
core.tablepool.release("plugins", plugins)
end
- run_plugin(phase_name, nil, api_ctx)
+ if api_ctx.script_obj then
+ script.run(phase_name, api_ctx)
+ else
+ run_plugin(phase_name, nil, api_ctx)
+ end
+
return api_ctx
end
diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua
index 1b537d3..9e4604f 100644
--- a/apisix/schema_def.lua
+++ b/apisix/schema_def.lua
@@ -422,6 +422,8 @@ _M.route = {
pattern = [[^function]],
},
+ script = {type = "string", minLength = 10, maxLength = 102400},
+
plugins = plugins_schema,
upstream = upstream_schema,
@@ -441,6 +443,13 @@ _M.route = {
{required = {"upstream", "uris"}},
{required = {"upstream_id", "uris"}},
{required = {"service_id", "uris"}},
+ {required = {"script", "uri"}},
+ {required = {"script", "uris"}},
+ },
+ ["not"] = {
+ anyOf = {
+ {required = {"script", "plugins"}}
+ }
},
additionalProperties = false,
}
@@ -455,11 +464,13 @@ _M.service = {
upstream_id = id_schema,
name = {type = "string", maxLength = 50},
desc = {type = "string", maxLength = 256},
+ script = {type = "string", minLength = 10, maxLength = 102400},
},
anyOf = {
{required = {"upstream"}},
{required = {"upstream_id"}},
{required = {"plugins"}},
+ {required = {"script"}},
},
additionalProperties = false,
}
diff --git a/apisix/script.lua b/apisix/script.lua
new file mode 100644
index 0000000..a1e0c02
--- /dev/null
+++ b/apisix/script.lua
@@ -0,0 +1,59 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local require = require
+local core = require("apisix.core")
+local loadstring = loadstring
+local error = error
+
+
+local _M = {}
+
+
+function _M.load(route, api_ctx)
+ local script = route.value.script
+ if script == nil or script == "" then
+ error("missing valid script")
+ end
+
+ local loadfun, err = loadstring(script, "route#" .. route.value.id)
+ if not loadfun then
+ error("failed to load script: " .. err .. " script: " .. script)
+ return nil
+ end
+ api_ctx.script_obj = loadfun()
+end
+
+
+function _M.run(phase, api_ctx)
+ local obj = api_ctx and api_ctx.script_obj
+ if not obj then
+ core.log.error("missing loaded script object")
+ return api_ctx
+ end
+
+ core.log.info("loaded script_obj: ", core.json.delay_encode(obj, true))
+
+ local phase_fun = obj[phase]
+ if phase_fun then
+ phase_fun(api_ctx)
+ end
+
+ return api_ctx
+end
+
+
+return _M
diff --git a/doc/admin-api.md b/doc/admin-api.md
index 665f585..9a3def9 100644
--- a/doc/admin-api.md
+++ b/doc/admin-api.md
@@ -64,6 +64,7 @@
|vars |False |Match Rules |A list of one or more `{var, operator, val}` elements, like this: `{{var, operator, val}, {var, operator, val}, ...}}`. For example: `{"arg_name", "==", "json"}` means that the current request parameter `name` is `json`. The `var` here is consistent with the internal variable name of Nginx, so you can also use `request_uri`, `host`, etc. For the operator part, the currently supported operators are `==`, `~=`,`>`, `<`, and `~~`. For the `>` and `<` operat [...]
|filter_func|False|Match Rules|User-defined filtering function. You can use it to achieve matching requirements for special scenarios. This function accepts an input parameter named `vars` by default, which you can use to get Nginx variables.|function(vars) return vars["arg_name"] == "json" end|
|plugins |False |Plugin|See [Plugin](architecture-design.md#plugin) for more ||
+|script |False |Script|See [Script](architecture-design.md#script) for more ||
|upstream |False |Upstream|Enabled Upstream configuration, see [Upstream](architecture-design.md#upstream) for more||
|upstream_id|False |Upstream|Enabled upstream id, see [Upstream](architecture-design.md#upstream) for more ||
|service_id|False |Service|Binded Service configuration, see [Service](architecture-design.md#service) for more ||
diff --git a/doc/architecture-design.md b/doc/architecture-design.md
index ece3ff5..d52ba30 100644
--- a/doc/architecture-design.md
+++ b/doc/architecture-design.md
@@ -26,6 +26,7 @@
- [**Route**](#route)
- [**Service**](#service)
- [**Plugin**](#plugin)
+- [**Script**](#script)
- [**Upstream**](#upstream)
- [**Router**](#router)
- [**Consumer**](#consumer)
@@ -216,6 +217,25 @@ Not all plugins have specific configuration items. For example, there is no spec
[Back to top](#Table-of-contents)
+## Script
+
+`Script` represents a script that will be executed during the `HTTP` request/response life cycle.
+
+The `Script` configuration can be directly bound to the `Route`.
+
+`Script` and `Plugin` are mutually exclusive, and `Script` is executed first. This means that after configuring `Script`, the `Plugin` configured on `Route` will not be executed.
+
+In theory, you can write arbitrary Lua code in `Script`, or you can directly call existing plugins to reuse existing code.
+
+`Script` also has the concept of execution phase, supporting `access`, `header_filer`, `body_filter` and `log` phase. The system will automatically execute the code of the corresponding phase in the `Script` script in the corresponding phase.
+
+```json
+{
+ ...
+ "script": "local _M = {} \n function _M.access(api_ctx) \n ngx.log(ngx.INFO,\"hit access phase\") \n end \nreturn _M"
+}
+```
+
## Upstream
Upstream is a virtual host abstraction that performs load balancing on a given set of service nodes according to configuration rules. Upstream address information can be directly configured to `Route` (or `Service`). When Upstream has duplicates, you need to use "reference" to avoid duplication.
diff --git a/doc/zh-cn/admin-api.md b/doc/zh-cn/admin-api.md
index 1638420..529d102 100644
--- a/doc/zh-cn/admin-api.md
+++ b/doc/zh-cn/admin-api.md
@@ -57,10 +57,11 @@
|---------|---------|----|-----------|----|
|uri |与 `uris` 二选一 |匹配规则|除了如 `/foo/bar`、`/foo/gloo` 这种全量匹配外，使用不同 [Router](architecture-design.md#router) 还允许更高级匹配，更多见 [Router](architecture-design.md#router)。|"/hello"|
|uris |与 `uri` 二选一 |匹配规则|数组形式，可以匹配多个 `uri`|["/hello", "/world"]|
-|plugins |`plugins`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Plugin|详见 [Plugin](architecture-design.md#plugin) ||
-|upstream |`plugins`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Upstream|启用的 Upstream 配置，详见 [Upstream](architecture-design.md#upstream)||
-|upstream_id|`plugins`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Upstream|启用的 upstream id，详见 [Upstream](architecture-design.md#upstream)||
-|service_id|`plugins`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Service|绑定的 Service 配置，详见 [Service](architecture-design.md#service)||
+|plugins |`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Plugin|详见 [Plugin](architecture-design.md#plugin) ||
+|script |`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Script|详见 [Script](architecture-design.md#script) ||
+|upstream |`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Upstream|启用的 Upstream 配置，详见 [Upstream](architecture-design.md#upstream)||
+|upstream_id|`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Upstream|启用的 upstream id，详见 [Upstream](architecture-design.md#upstream)||
+|service_id|`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Service|绑定的 Service 配置，详见 [Service](architecture-design.md#service)||
|service_protocol|可选|上游协议类型|只可以是 "grpc", "http" 二选一。|默认 "http"，使用gRPC proxy 或gRPC transcode 时，必须用"grpc"|
|name |可选 |辅助 |标识路由名称|route-xxxx|
|desc |可选 |辅助 |标识描述、使用场景等。|客户 xxxx|
@@ -75,7 +76,7 @@
有两点需要特别注意：
-* 除了 `uri`/`uris` 是必选的之外，`plugins`、`upstream`/`upstream_id`、`service_id` 这三类必须选择其中至少一个。
+* 除了 `uri`/`uris` 是必选的之外，`plugins`、`script`、`upstream`/`upstream_id`、`service_id` 这三类必须选择其中至少一个。
* 对于同一类参数比如 `uri`与 `uris`，`upstream` 与 `upstream_id`，`host` 与 `hosts`，`remote_addr` 与 `remote_addrs` 等，是不能同时存在，二者只能选择其一。如果同时启用，接口会报错。
route 对象 json 配置内容：
diff --git a/doc/zh-cn/architecture-design.md b/doc/zh-cn/architecture-design.md
index ecacc58..2d696cc 100644
--- a/doc/zh-cn/architecture-design.md
+++ b/doc/zh-cn/architecture-design.md
@@ -24,6 +24,7 @@
- [**Route**](#route)
- [**Service**](#service)
- [**Plugin**](#plugin)
+- [**Script**](#script)
- [**Upstream**](#upstream)
- [**Router**](#router)
- [**Consumer**](#consumer)
@@ -220,6 +221,27 @@ curl http://127.0.0.1:9080/apisix/admin/routes/102 -H 'X-API-KEY: edd1c9f034335f
[返回目录](#目录)
+## Script
+
+`Script` 表示将在 `HTTP` 请求/响应生命周期期间执行的脚本。
+
+`Script` 配置可直接绑定在 `Route` 上。
+
+`Script` 与 `Plugin` 互斥，且优先执行 `Script` ，这意味着配置 `Script` 后，`Route` 上配置的 `Plugin` 将不被执行。
+
+理论上，在 `Script` 中可以写任意 lua 代码，也可以直接调用已有插件以重用已有的代码。
+
+`Script` 也有执行阶段概念，支持 `access`、`header_filer`、`body_filter` 和 `log` 阶段。系统会在相应阶段中自动执行 `Script` 脚本中对应阶段的代码。
+
+```json
+{
+ ...
+ "script": "local _M = {} \n function _M.access(api_ctx) \n ngx.log(ngx.INFO,\"hit access phase\") \n end \nreturn _M"
+}
+```
+
+[返回目录](#目录)
+
## Upstream
Upstream 是虚拟主机抽象，对给定的多个服务节点按照配置规则进行负载均衡。Upstream 的地址信息可以直接配置到 `Route`（或 `Service`) 上，当 Upstream 有重复时，就需要用“引用”方式避免重复了。
diff --git a/t/admin/routes.t b/t/admin/routes.t
index a5e4b6e..afa846a 100644
--- a/t/admin/routes.t
+++ b/t/admin/routes.t
@@ -2108,3 +2108,42 @@ GET /t
{"error_msg":"invalid request body: request size 1678025 is greater than the maximum size 1572864 allowed"}
--- error_log
failed to read request body: request size 1678025 is greater than the maximum size 1572864 allowed
+
+
+
+=== TEST 58: uri + plugins + script failed
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin").test
+ local code, message, res = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "limit-count": {
+ "count": 2,
+ "time_window": 60,
+ "rejected_code": 503,
+ "key": "remote_addr"
+ }
+ },
+ "script": "local _M = {} \n function _M.access(api_ctx) \n ngx.log(ngx.INFO,\"hit access phase\") \n end \nreturn _M",
+ "uri": "/index.html"
+ }]]
+ )
+
+ if code ~= 200 then
+ ngx.status = code
+ ngx.say(message)
+ return
+ end
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body_like
+{"error_msg":"invalid configuration: value wasn't supposed to match schema"}
+--- no_error_log
+[error]
diff --git a/t/script/script_distribute.t b/t/script/script_distribute.t
new file mode 100644
index 0000000..75e7b13
--- /dev/null
+++ b/t/script/script_distribute.t
@@ -0,0 +1,158 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_root_location();
+no_shuffle();
+
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: set route(host + uri)
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin")
+
+ local script = t.read_file("t/script/script_test.lua")
+ local data = {
+ script = script,
+ uri = "/hello",
+ upstream = {
+ nodes = {
+ ["127.0.0.1:1980"] = 1
+ },
+ type = "roundrobin"
+ }
+ }
+
+ local code, body = t.test('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ core.json.encode(data))
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- yaml_config eval: $::yaml_config
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: hit routes
+--- request
+GET /hello
+--- yaml_config eval: $::yaml_config
+--- response_body
+hello world
+--- no_error_log
+[error]
+--- error_log
+string "route#1"
+phase_fun(): hit access phase
+phase_fun(): hit header_filter phase
+phase_fun(): hit body_filter phase
+phase_fun(): hit body_filter phase
+phase_fun(): hit log phase while
+
+
+
+=== TEST 3: invalid script in route
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin")
+
+ local data = {
+ script = "invalid script",
+ uri = "/hello",
+ upstream = {
+ nodes = {
+ ["127.0.0.1:1980"] = 1
+ },
+ type = "roundrobin"
+ }
+ }
+
+ local code, body = t.test('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ core.json.encode(data))
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.print(body)
+ }
+ }
+--- yaml_config eval: $::yaml_config
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"failed to load 'script' string: [string \"invalid script\"]:1: '=' expected near 'script'"}
+--- no_error_log
+[error]
+
+
+
+=== TEST 4: invalid script in service
+--- config
+ location /t {
+ content_by_lua_block {
+ local core = require("apisix.core")
+ local t = require("lib.test_admin")
+
+ local data = {
+ script = "invalid script",
+ upstream = {
+ nodes = {
+ ["127.0.0.1:1980"] = 1
+ },
+ type = "roundrobin"
+ }
+ }
+
+ local code, body = t.test('/apisix/admin/services/1',
+ ngx.HTTP_PUT,
+ core.json.encode(data))
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.print(body)
+ }
+ }
+--- yaml_config eval: $::yaml_config
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"failed to load 'script' string: [string \"invalid script\"]:1: '=' expected near 'script'"}
+--- no_error_log
+[error]
diff --git a/t/script/script_test.lua b/t/script/script_test.lua
new file mode 100644
index 0000000..e2590ab
--- /dev/null
+++ b/t/script/script_test.lua
@@ -0,0 +1,43 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+
+
+local _M = {}
+
+
+function _M.access(api_ctx)
+ core.log.warn("hit access phase")
+end
+
+
+function _M.header_filter(ctx)
+ core.log.warn("hit header_filter phase")
+end
+
+
+function _M.body_filter(ctx)
+ core.log.warn("hit body_filter phase")
+end
+
+
+function _M.log(ctx)
+ core.log.warn("hit log phase")
+end
+
+
+return _M