You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by me...@apache.org on 2020/10/27 05:42:15 UTC
[apisix] branch master updated: feat: implement api breaker plugin.
(#2455)
This is an automated email from the ASF dual-hosted git repository.
membphis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 6a7dfa6 feat: implement api breaker plugin. (#2455)
6a7dfa6 is described below
commit 6a7dfa677562876538f3afe8b61f25ad2af39e20
Author: YuanSheng Wang <me...@gmail.com>
AuthorDate: Tue Oct 27 13:40:23 2020 +0800
feat: implement api breaker plugin. (#2455)
Co-authored-by: liuheng <li...@gmail.com>
---
apisix/plugins/api-breaker.lua | 248 +++++++++++++++
bin/apisix | 1 +
conf/config-default.yaml | 1 +
doc/plugins/api-breaker.md | 117 +++++++
doc/zh-cn/plugins/api-breaker.md | 116 +++++++
t/APISIX.pm | 1 +
t/admin/plugins.t | 2 +-
t/debug/debug-mode.t | 1 +
t/lib/server.lua | 9 +
t/plugin/api-breaker.t | 651 +++++++++++++++++++++++++++++++++++++++
10 files changed, 1146 insertions(+), 1 deletion(-)
diff --git a/apisix/plugins/api-breaker.lua b/apisix/plugins/api-breaker.lua
new file mode 100644
index 0000000..d97577d
--- /dev/null
+++ b/apisix/plugins/api-breaker.lua
@@ -0,0 +1,248 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local core = require("apisix.core")
+local plugin_name = "api-breaker"
+local ngx = ngx
+local math = math
+local error = error
+local ipairs = ipairs
+
+local shared_buffer = ngx.shared['plugin-'.. plugin_name]
+if not shared_buffer then
+ error("failed to get ngx.shared dict when load plugin " .. plugin_name)
+end
+
+
+local schema = {
+ type = "object",
+ properties = {
+ break_response_code = {
+ type = "integer",
+ minimum = 200,
+ maximum = 599,
+ },
+ max_breaker_sec = {
+ type = "integer",
+ minimum = 3,
+ default = 300,
+ },
+ unhealthy = {
+ type = "object",
+ properties = {
+ http_statuses = {
+ type = "array",
+ minItems = 1,
+ items = {
+ type = "integer",
+ minimum = 500,
+ maximum = 599,
+ },
+ uniqueItems = true,
+ default = {500}
+ },
+ failures = {
+ type = "integer",
+ minimum = 1,
+ default = 3,
+ }
+ },
+ default = {http_statuses = {500}, failures = 3}
+ },
+ healthy = {
+ type = "object",
+ properties = {
+ http_statuses = {
+ type = "array",
+ minItems = 1,
+ items = {
+ type = "integer",
+ minimum = 200,
+ maximum = 499,
+ },
+ uniqueItems = true,
+ default = {200}
+ },
+ successes = {
+ type = "integer",
+ minimum = 1,
+ default = 3,
+ }
+ },
+ default = {http_statuses = {200}, successes = 3}
+ }
+ },
+ required = {"break_response_code"},
+}
+
+
+-- todo: we can move this into `core.talbe`
+local function array_find(array, val)
+ for i, v in ipairs(array) do
+ if v == val then
+ return i
+ end
+ end
+
+ return nil
+end
+
+
+local function gen_healthy_key(ctx)
+ return "healthy-" .. core.request.get_host(ctx) .. ctx.var.uri
+end
+
+
+local function gen_unhealthy_key(ctx)
+ return "unhealthy-" .. core.request.get_host(ctx) .. ctx.var.uri
+end
+
+
+local function gen_lasttime_key(ctx)
+ return "unhealthy-lastime" .. core.request.get_host(ctx) .. ctx.var.uri
+end
+
+
+local _M = {
+ version = 0.1,
+ name = plugin_name,
+ priority = 1005,
+ schema = schema,
+}
+
+
+function _M.check_schema(conf)
+ return core.schema.check(schema, conf)
+end
+
+
+function _M.access(conf, ctx)
+ local unhealthy_key = gen_unhealthy_key(ctx)
+ -- unhealthy counts
+ local unhealthy_count, err = shared_buffer:get(unhealthy_key)
+ if err then
+ core.log.warn("failed to get unhealthy_key: ",
+ unhealthy_key, " err: ", err)
+ return
+ end
+
+ if not unhealthy_count then
+ return
+ end
+
+ -- timestamp of the last time a unhealthy state was triggered
+ local lasttime_key = gen_lasttime_key(ctx)
+ local lasttime, err = shared_buffer:get(lasttime_key)
+ if err then
+ core.log.warn("failed to get lasttime_key: ",
+ lasttime_key, " err: ", err)
+ return
+ end
+
+ if not lasttime then
+ return
+ end
+
+ local failure_times = math.ceil(unhealthy_count / conf.unhealthy.failures)
+ if failure_times < 1 then
+ failure_times = 1
+ end
+
+ -- cannot exceed the maximum value of the user configuration
+ local breaker_time = 2 ^ failure_times
+ if breaker_time > conf.max_breaker_sec then
+ breaker_time = conf.max_breaker_sec
+ end
+ core.log.info("breaker_time: ", breaker_time)
+
+ -- breaker
+ if lasttime + breaker_time >= ngx.time() then
+ return conf.break_response_code
+ end
+
+ return
+end
+
+
+function _M.log(conf, ctx)
+ local unhealthy_key = gen_unhealthy_key(ctx)
+ local healthy_key = gen_healthy_key(ctx)
+ local upstream_status = core.response.get_upstream_status(ctx)
+
+ if not upstream_status then
+ return
+ end
+
+ -- unhealth process
+ if array_find(conf.unhealthy.http_statuses, upstream_status) then
+ local unhealthy_count, err = shared_buffer:incr(unhealthy_key, 1, 0)
+ if err then
+ core.log.warn("failed to incr unhealthy_key: ", unhealthy_key,
+ " err: ", err)
+ end
+ core.log.info("unhealthy_key: ", unhealthy_key, " count: ",
+ unhealthy_count)
+
+ shared_buffer:delete(healthy_key)
+
+ -- whether the user-configured number of failures has been reached,
+ -- and if so, the timestamp for entering the unhealthy state.
+ if unhealthy_count % conf.unhealthy.failures == 0 then
+ shared_buffer:set(gen_lasttime_key(ctx), ngx.time(),
+ conf.max_breaker_sec)
+ core.log.info("update unhealthy_key: ", unhealthy_key, " to ",
+ unhealthy_count)
+ end
+
+ return
+ end
+
+ -- health process
+ if not array_find(conf.healthy.http_statuses, upstream_status) then
+ return
+ end
+
+ local unhealthy_count, err = shared_buffer:get(unhealthy_key)
+ if err then
+ core.log.warn("failed to `get` unhealthy_key: ", unhealthy_key,
+ " err: ", err)
+ end
+
+ if not unhealthy_count then
+ return
+ end
+
+ local healthy_count, err = shared_buffer:incr(healthy_key, 1, 0)
+ if err then
+ core.log.warn("failed to `incr` healthy_key: ", healthy_key,
+ " err: ", err)
+ end
+
+ -- clear related status
+ if healthy_count >= conf.healthy.successes then
+ -- stat change to normal
+ core.log.info("chagne to normal, ", healthy_key, " ", healthy_count)
+ shared_buffer:delete(gen_lasttime_key(ctx))
+ shared_buffer:delete(unhealthy_key)
+ shared_buffer:delete(healthy_key)
+ end
+
+ return
+end
+
+
+return _M
diff --git a/bin/apisix b/bin/apisix
index 5e24fac..efefcea 100755
--- a/bin/apisix
+++ b/bin/apisix
@@ -191,6 +191,7 @@ http {
lua_shared_dict balancer_ewma_locks 10m;
lua_shared_dict balancer_ewma_last_touched_at 10m;
lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m;
+ lua_shared_dict plugin-api-breaker 10m;
# for openid-connect plugin
lua_shared_dict discovery 1m; # cache for discovery metadata documents
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index 00e0cbc..688fa54 100644
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -196,6 +196,7 @@ plugins: # plugin list
- proxy-mirror
- request-id
- hmac-auth
+ - api-breaker
stream_plugins:
- mqtt-proxy
diff --git a/doc/plugins/api-breaker.md b/doc/plugins/api-breaker.md
new file mode 100644
index 0000000..5d94236
--- /dev/null
+++ b/doc/plugins/api-breaker.md
@@ -0,0 +1,117 @@
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+- [中文](../zh-cn/plugins/api-blocker.md)
+
+# Summary
+
+- [**Name**](#name)
+- [**Attributes**](#attributes)
+- [**How To Enable**](#how-to-enable)
+- [**Test Plugin**](#test-plugin)
+- [**Disable Plugin**](#disable-plugin)
+
+## Name
+
+The plugin implements API fuse functionality to help us protect our upstream business services.
+
+> About the breaker timeout logic
+
+the code logic automatically **triggers the unhealthy state** incrementation of the number of operations.
+
+Whenever the upstream service returns a status code from the `unhealthy.http_statuses` configuration (e.g., 500), up to `unhealthy.failures` (e.g., three times) and considers the upstream service to be in an unhealthy state.
+
+The first time unhealthy status is triggered, **breaken for 2 seconds**.
+
+Then, the request is forwarded to the upstream service again after 2 seconds, and if the `unhealthy.http_statuses` status code is returned, and the count reaches `unhealthy.failures` again, **broken for 4 seconds**.
+
+and so on, 2, 4, 8, 16, 32, 64, ..., 256, 300. `300` is the maximum value of `max_breaker_sec`, allow users to specify.
+
+In an unhealthy state, when a request is forwarded to an upstream service and the status code in the `healthy.http_statuses` configuration is returned (e.g., 200) that `healthy.successes` is reached (e.g., three times), and the upstream service is considered healthy again.
+
+## Attributes
+
+| Name | Type | Requirement | Default | Valid | Description |
+| ------------- | ------------- | ----------- | ------- | ---------- | --------------------------------------------------------------------------- |
+| break_response_code | integer | required | | [200, ..., 600] | return error code when unhealthy |
+| max_breaker_sec | integer | optional | 300 | >=60 | Maximum breaker time(seconds) |
+| unhealthy.http_statuses | array[integer] | optional | {500} | [500, ..., 599] | Status codes when unhealthy |
+| unhealthy.failures | integer | optional | 1 | >=1 | Number of consecutive error requests that triggered an unhealthy state |
+| healthy.http_statuses | array[integer] | optional | {200, 206} | [200, ..., 499] | Status codes when healthy |
+| healthy.successes | integer | optional | 1 | >=1 | Number of consecutive normal requests that trigger health status |
+
+## How To Enable
+
+Here's an example, enable the `api-breaker` plugin on the specified route.
+
+Response 500 or 503 three times in a row to trigger a unhealthy. Response 200 once in a row to restore healthy.
+
+```shell
+curl "http://127.0.0.1:9080/apisix/admin/routes/1" -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+ "plugins": {
+ "api-breaker": {
+ "break_response_code": 502,
+ "unhealthy": {
+ "http_statuses": [500, 503],
+ "failures": 3
+ },
+ "healthy": {
+ "http_statuses": [200],
+ "successes": 1
+ }
+ }
+ },
+ "uri": "/hello",
+ "host": "127.0.0.1",
+}'
+```
+
+## Test Plugin
+
+Then. Like the configuration above, if your upstream service returns 500. 3 times in a row. The client will receive a 502 (break_response_code) response.
+
+```shell
+$ curl -i -X POST "http://127.0.0.1:9080/get"
+HTTP/1.1 502 Bad Gateway
+Content-Type: application/octet-stream
+Connection: keep-alive
+Server: APISIX/1.5
+
+... ...
+```
+
+## Disable Plugin
+
+When you want to disable the `api-breader` plugin, it is very simple, you can delete the corresponding json configuration in the plugin configuration, no need to restart the service, it will take effect immediately:
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+ "uri": "/hello",
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": {
+ "127.0.0.1:1980": 1
+ }
+ }
+}'
+```
+
+The `api-breaker` plugin has been disabled now. It works for other plugins.
diff --git a/doc/zh-cn/plugins/api-breaker.md b/doc/zh-cn/plugins/api-breaker.md
new file mode 100644
index 0000000..5b90bb6
--- /dev/null
+++ b/doc/zh-cn/plugins/api-breaker.md
@@ -0,0 +1,116 @@
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+- [English](../../plugins/api-blocker.md)
+
+# 目录
+
+- [**定义**](#定义)
+- [**属性列表**](#属性列表)
+- [**启用方式**](#启用方式)
+- [**测试插件**](#测试插件)
+- [**禁用插件**](#禁用插件)
+
+## 定义
+
+该插件实现 API 熔断功能，帮助我们保护上游业务服务。
+
+> 关于熔断超时逻辑
+
+由代码逻辑自动按**触发不健康状态**的次数递增运算：
+
+每当上游服务返回`unhealthy.http_statuses`配置中的状态码(比如：500)，达到`unhealthy.failures`次时(比如：3 次)，认为上游服务处于不健康状态。
+
+第一次触发不健康状态，**熔断 2 秒**。
+
+然后，2 秒过后重新开始转发请求到上游服务，如果继续返回`unhealthy.http_statuses`状态码，记数再次达到`unhealthy.failures`次时，**熔断 4 秒**（倍数方式）。
+
+依次类推，2, 4, 8, 16, 32, 64, ..., 256, 最大到 300。 300 是 `max_breaker_sec` 的最大值，允许自定义修改。
+
+在不健康状态时，当转发请求到上游服务并返回`healthy.http_statuses`配置中的状态码(比如：200)，达到`healthy.successes`次时(比如：3 次)，认为上游服务恢复健康状态。
+
+## 属性列表
+
+| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
+| ----------------------- | -------------- | ------ | ---------- | --------------- | -------------------------------- |
+| break_response_code | integer | 必须 | 无 | [200, ..., 600] | 不健康返回错误码 |
+| max_breaker_sec | integer | 可选 | 300 | >=60 | 最大熔断持续时间 |
+| unhealthy.http_statuses | array[integer] | 可选 | {500} | [500, ..., 599] | 不健康时候的状态码 |
+| unhealthy.failures | integer | 可选 | 1 | >=1 | 触发不健康状态的连续错误请求次数 |
+| healthy.http_statuses | array[integer] | 可选 | {200, 206} | [200, ..., 499] | 健康时候的状态码 |
+| healthy.successes | integer | 可选 | 1 | >=1 | 触发健康状态的连续正常请求次数 |
+
+## 启用方式
+
+这是一个示例，在指定的路由上启用`api-breaker`插件。
+应答 500 或 503 连续 3 次，触发熔断。应答 200 连续 1 次，恢复健康。
+
+```shell
+curl "http://127.0.0.1:9080/apisix/admin/routes/1" -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+ "plugins": {
+ "api-breaker": {
+ "break_response_code": 502,
+ "unhealthy": {
+ "http_statuses": [500, 503],
+ "failures": 3
+ },
+ "healthy": {
+ "http_statuses": [200],
+ "successes": 1
+ }
+ }
+ },
+ "uri": "/hello",
+ "host": "127.0.0.1"
+}'
+```
+
+## 测试插件
+
+使用上游的配置，如果你的上流服务返回 500，连续 3 次。客户端将会收到 502（break_response_code）应答。
+
+```shell
+$ curl -i "http://127.0.0.1:9080/get"
+HTTP/1.1 502 Bad Gateway
+Content-Type: application/octet-stream
+Connection: keep-alive
+Server: APISIX/1.5
+
+... ...
+```
+
+## 禁用插件
+
+当想禁用`api-breaker`插件时，非常简单，只需要在插件配置中删除相应的 json 配置，无需重启服务，即可立即生效：
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+ "uri": "/hello",
+ "upstream": {
+ "type": "roundrobin",
+ "nodes": {
+ "127.0.0.1:1980": 1
+ }
+ }
+}'
+```
+
+`api-breaker` 插件现在已被禁用，它也适用于其他插件。
diff --git a/t/APISIX.pm b/t/APISIX.pm
index 17448e7..66fd0cc 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -241,6 +241,7 @@ _EOC_
lua_shared_dict balancer_ewma_locks 1m;
lua_shared_dict balancer_ewma_last_touched_at 1m;
lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m;
+ lua_shared_dict plugin-api-breaker 10m;
resolver $dns_addrs_str;
resolver_timeout 5;
diff --git a/t/admin/plugins.t b/t/admin/plugins.t
index 4caca83..5f3b3c6 100644
--- a/t/admin/plugins.t
+++ b/t/admin/plugins.t
@@ -30,7 +30,7 @@ __DATA__
--- request
GET /apisix/admin/plugins/list
--- response_body_like eval
-qr/\["request-id","fault-injection","serverless-pre-function","batch-requests","cors","ip-restriction","referer-restriction","uri-blocker","request-validation","openid-connect","wolf-rbac","hmac-auth","basic-auth","jwt-auth","key-auth","consumer-restriction","authz-keycloak","proxy-mirror","proxy-cache","proxy-rewrite","limit-conn","limit-count","limit-req","node-status","redirect","response-rewrite","grpc-transcode","prometheus","echo","http-logger","tcp-logger","kafka-logger","syslog", [...]
+qr/\["request-id","fault-injection","serverless-pre-function","batch-requests","cors","ip-restriction","referer-restriction","uri-blocker","request-validation","openid-connect","wolf-rbac","hmac-auth","basic-auth","jwt-auth","key-auth","consumer-restriction","authz-keycloak","proxy-mirror","proxy-cache","proxy-rewrite","api-breaker","limit-conn","limit-count","limit-req","node-status","redirect","response-rewrite","grpc-transcode","prometheus","echo","http-logger","tcp-logger","kafka-log [...]
--- no_error_log
[error]
diff --git a/t/debug/debug-mode.t b/t/debug/debug-mode.t
index 9ad669f..0ad80ce 100644
--- a/t/debug/debug-mode.t
+++ b/t/debug/debug-mode.t
@@ -66,6 +66,7 @@ loaded plugin and sort by priority: 2000 name: authz-keycloak
loaded plugin and sort by priority: 1010 name: proxy-mirror
loaded plugin and sort by priority: 1009 name: proxy-cache
loaded plugin and sort by priority: 1008 name: proxy-rewrite
+loaded plugin and sort by priority: 1005 name: api-breaker
loaded plugin and sort by priority: 1003 name: limit-conn
loaded plugin and sort by priority: 1002 name: limit-count
loaded plugin and sort by priority: 1001 name: limit-req
diff --git a/t/lib/server.lua b/t/lib/server.lua
index afed16e..0919d2a 100644
--- a/t/lib/server.lua
+++ b/t/lib/server.lua
@@ -262,6 +262,15 @@ function _M.websocket_handshake()
end
_M.websocket_handshake_route = _M.websocket_handshake
+function _M.api_breaker()
+ ngx.exit(tonumber(ngx.var.arg_code))
+end
+
+function _M.mysleep()
+ ngx.sleep(tonumber(ngx.var.arg_seconds))
+ ngx.say(ngx.var.arg_seconds)
+end
+
local function print_uri()
ngx.say(ngx.var.uri)
end
diff --git a/t/plugin/api-breaker.t b/t/plugin/api-breaker.t
new file mode 100644
index 0000000..9b5ff4e
--- /dev/null
+++ b/t/plugin/api-breaker.t
@@ -0,0 +1,651 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+log_level('info');
+run_tests;
+
+__DATA__
+
+=== TEST 1: sanity
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.api-breaker")
+ local ok, err = plugin.check_schema({
+ break_response_code = 502,
+ unhealthy = {
+ http_statuses = {500},
+ failures = 1,
+ },
+ healthy = {
+ http_statuses = {200},
+ successes = 1,
+ },
+ })
+ if not ok then
+ ngx.say(err)
+ end
+
+ ngx.say("done")
+ }
+ }
+--- request
+GET /t
+--- response_body
+done
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: default configuration
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.api-breaker")
+ local conf = {
+ break_response_code = 502
+ }
+
+ local ok, err = plugin.check_schema(conf)
+ if not ok then
+ ngx.say(err)
+ end
+
+ ngx.say(require("lib.json_sort").encode(conf))
+ }
+ }
+--- request
+GET /t
+--- response_body
+{"break_response_code":502,"healthy":{"http_statuses":[200],"successes":3},"max_breaker_sec":300,"unhealthy":{"failures":3,"http_statuses":[500]}}
+--- no_error_log
+[error]
+
+
+
+=== TEST 3: default `healthy`
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.api-breaker")
+ local conf = {
+ break_response_code = 502,
+ healthy = {}
+ }
+
+ local ok, err = plugin.check_schema(conf)
+ if not ok then
+ ngx.say(err)
+ end
+
+ ngx.say(require("lib.json_sort").encode(conf))
+ }
+ }
+--- request
+GET /t
+--- response_body
+{"break_response_code":502,"healthy":{"http_statuses":[200],"successes":3},"max_breaker_sec":300,"unhealthy":{"failures":3,"http_statuses":[500]}}
+--- no_error_log
+[error]
+
+
+
+=== TEST 4: default `unhealthy`
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.api-breaker")
+ local conf = {
+ break_response_code = 502,
+ unhealthy = {}
+ }
+
+ local ok, err = plugin.check_schema(conf)
+ if not ok then
+ ngx.say(err)
+ end
+
+ ngx.say(require("lib.json_sort").encode(conf))
+ }
+ }
+--- request
+GET /t
+--- response_body
+{"break_response_code":502,"healthy":{"http_statuses":[200],"successes":3},"max_breaker_sec":300,"unhealthy":{"failures":3,"http_statuses":[500]}}
+--- no_error_log
+[error]
+
+
+
+=== TEST 5: bad break_response_code
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "api-breaker": {
+ "break_response_code": 199,
+ "unhealthy": {
+ "http_statuses": [500, 503],
+ "failures": 3
+ },
+ "healthy": {
+ "http_statuses": [200, 206],
+ "successes": 3
+ }
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/api_breaker"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"failed to check the configuration of plugin api-breaker err: property \"break_response_code\" validation failed: expected 199 to be greater than 200"}
+--- no_error_log
+[error]
+
+
+
+=== TEST 6: bad max_breaker_sec
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "api-breaker": {
+ "break_response_code": 200,
+ "max_breaker_sec": -1
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/api_breaker"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- no_error_log
+[error]
+
+
+
+=== TEST 7: bad unhealthy.http_statuses
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "api-breaker": {
+ "break_response_code": 200,
+ "max_breaker_sec": 40,
+ "unhealthy": {
+ "http_statuses": [500, 603],
+ "failures": 3
+ },
+ "healthy": {
+ "http_statuses": [200, 206],
+ "successes": 3
+ }
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/api_breaker"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- no_error_log
+[error]
+
+
+
+=== TEST 8: same http_statuses in healthy
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "api-breaker": {
+ "break_response_code": 500,
+ "unhealthy": {
+ "http_statuses": [500, 503],
+ "failures": 3
+ },
+ "healthy": {
+ "http_statuses": [206, 206],
+ "successes": 3
+ }
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/api_breaker"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.print(body)
+ }
+ }
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"failed to check the configuration of plugin api-breaker err: property \"healthy\" validation failed: property \"http_statuses\" validation failed: expected unique items but items 2 and 1 are equal"}
+--- no_error_log
+[error]
+
+
+
+=== TEST 9: set route, http_statuses: [500, 503]
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "api-breaker": {
+ "break_response_code": 599,
+ "unhealthy": {
+ "http_statuses": [500, 503],
+ "failures": 3
+ },
+ "healthy": {
+ "http_statuses": [200, 206],
+ "successes": 3
+ }
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/api_breaker"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 10: trigger breaker
+--- request eval
+[
+ "GET /api_breaker?code=200", "GET /api_breaker?code=500",
+ "GET /api_breaker?code=503", "GET /api_breaker?code=500",
+ "GET /api_breaker?code=500", "GET /api_breaker?code=500"
+]
+--- error_code eval
+[200, 500, 503, 500, 599, 599]
+--- no_error_log
+[error]
+
+
+
+=== TEST 11: trigger reset status
+--- request eval
+[
+ "GET /api_breaker?code=500", "GET /api_breaker?code=500",
+
+ "GET /api_breaker?code=200", "GET /api_breaker?code=200",
+ "GET /api_breaker?code=200",
+
+ "GET /api_breaker?code=500", "GET /api_breaker?code=500"
+]
+--- error_code eval
+[
+ 500, 500,
+ 200, 200, 200,
+ 500, 500
+]
+--- no_error_log
+[error]
+
+
+
+=== TEST 12: trigger del healthy numeration
+--- config
+location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local json = require("lib.json_sort")
+
+ -- trigger to unhealth
+ for i = 1, 4 do
+ local code = t('/api_breaker?code=500', ngx.HTTP_GET)
+ ngx.say("code: ", code)
+ end
+
+ -- break for 3 seconds
+ ngx.sleep(3)
+
+ -- make a try
+ for i = 1, 4 do
+ local code = t('/api_breaker?code=200', ngx.HTTP_GET)
+ ngx.say("code: ", code)
+ end
+
+ for i = 1, 4 do
+ local code = t('/api_breaker?code=500', ngx.HTTP_GET)
+ ngx.say("code: ", code)
+ end
+ }
+}
+--- request
+GET /t
+--- response_body
+code: 500
+code: 500
+code: 500
+code: 599
+code: 200
+code: 200
+code: 200
+code: 200
+code: 500
+code: 500
+code: 500
+code: 599
+--- no_error_log
+[error]
+breaker_time: 4
+--- error_log
+breaker_time: 2
+
+
+
+=== TEST 13: add plugin with default config value
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "api-breaker": {
+ "break_response_code": 502,
+ "unhealthy": {
+ "failures": 3
+ },
+ "healthy": {
+ "successes": 3
+ }
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/api_breaker"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 14: default value
+--- request
+GET /api_breaker?code=500
+--- error_code: 500
+--- no_error_log
+[error]
+
+
+
+=== TEST 15: trigger default value of unhealthy.http_statuses breaker
+--- request eval
+[
+ "GET /api_breaker?code=200", "GET /api_breaker?code=500",
+ "GET /api_breaker?code=503", "GET /api_breaker?code=500",
+ "GET /api_breaker?code=500", "GET /api_breaker?code=500"
+]
+--- error_code eval
+[200, 500, 503, 500, 500, 502]
+--- no_error_log
+[error]
+
+
+
+=== TEST 16: unhealthy -> timeout -> normal
+--- config
+ location /sleep1 {
+ proxy_pass "http://127.0.0.1:1980/sleep1";
+ }
+--- request eval
+[
+ "GET /api_breaker?code=500",
+ "GET /api_breaker?code=500",
+ "GET /api_breaker?code=500",
+ "GET /api_breaker?code=200",
+
+ "GET /sleep1",
+ "GET /sleep1",
+ "GET /sleep1",
+
+ "GET /api_breaker?code=200",
+ "GET /api_breaker?code=200",
+ "GET /api_breaker?code=200",
+ "GET /api_breaker?code=200",
+ "GET /api_breaker?code=200"]
+--- error_code eval
+[
+ 500, 500, 500, 502,
+ 200, 200, 200,
+ 200, 200, 200, 200,200
+]
+--- no_error_log
+[error]
+
+
+
+=== TEST 17: unhealthy -> timeout -> unhealthy
+--- config
+location /sleep1 {
+ proxy_pass "http://127.0.0.1:1980/sleep1";
+}
+--- request eval
+[
+ "GET /api_breaker?code=500", "GET /api_breaker?code=500",
+ "GET /api_breaker?code=500", "GET /api_breaker?code=200",
+
+ "GET /sleep1", "GET /sleep1", "GET /sleep1",
+
+ "GET /api_breaker?code=500","GET /api_breaker?code=500",
+ "GET /api_breaker?code=500","GET /api_breaker?code=500"
+ ]
+--- error_code eval
+[
+ 500, 500, 500, 502,
+ 200, 200, 200,
+ 500,502,502,502
+]
+--- no_error_log
+[error]
+
+
+
+=== TEST 18: enable plugin, unhealthy.failures=1
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "api-breaker": {
+ "break_response_code": 502,
+ "max_breaker_sec": 10,
+ "unhealthy": {
+ "http_statuses": [500, 503],
+ "failures": 1
+ },
+ "healthy": {
+ "successes": 3
+ }
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/api_breaker"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 19: hit route 20 times, confirm the breaker time
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local json = require("lib.json_sort")
+
+ local status_count = {}
+ for i = 1, 20 do
+ local code = t('/api_breaker?code=500', ngx.HTTP_GET)
+ code = tostring(code)
+ status_count[code] = (status_count[code] or 0) + 1
+ ngx.sleep(1)
+ end
+
+ ngx.say(json.encode(status_count))
+ }
+ }
+--- request
+GET /t
+--- no_error_log
+[error]
+phase_func(): breaker_time: 16
+--- error_log
+phase_func(): breaker_time: 2
+phase_func(): breaker_time: 4
+phase_func(): breaker_time: 8
+phase_func(): breaker_time: 10
+--- response_body
+{"500":4,"502":16}
+--- timeout: 25