You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by as...@apache.org on 2025/09/03 06:21:00 UTC
(apisix) branch master updated: feat: add fallback mechanism for specific error codes in ai-proxy-multi (#12571)
This is an automated email from the ASF dual-hosted git repository.
ashishtiwari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 1934d5602 feat: add fallback mechanism for specific error codes in ai-proxy-multi (#12571)
1934d5602 is described below
commit 1934d5602cb8f9d0fe538d81acd9df7a62041c4a
Author: Ashish Tiwari <as...@gmail.com>
AuthorDate: Wed Sep 3 11:50:55 2025 +0530
feat: add fallback mechanism for specific error codes in ai-proxy-multi (#12571)
---
apisix/plugins/ai-drivers/openai-base.lua | 5 +
apisix/plugins/ai-proxy-multi.lua | 50 ++-
apisix/plugins/ai-proxy/base.lua | 93 +++--
apisix/plugins/ai-proxy/schema.lua | 16 +-
apisix/plugins/ai-request-rewrite.lua | 6 +-
docs/en/latest/plugins/ai-proxy-multi.md | 8 +-
docs/en/latest/plugins/ai-rate-limiting.md | 8 +-
t/plugin/ai-proxy-multi.balancer.t | 641 +++++++++++++++++++++++++++++
8 files changed, 772 insertions(+), 55 deletions(-)
diff --git a/apisix/plugins/ai-drivers/openai-base.lua b/apisix/plugins/ai-drivers/openai-base.lua
index fcf956ba0..cb48c3e07 100644
--- a/apisix/plugins/ai-drivers/openai-base.lua
+++ b/apisix/plugins/ai-drivers/openai-base.lua
@@ -272,6 +272,11 @@ function _M.request(self, ctx, conf, request_table, extra_opts)
return handle_error(err)
end
+ -- handling this error separately is needed for retries
+ if res.status == 429 or (res.status >= 500 and res.status < 600 )then
+ return res.status
+ end
+
local code, body = read_response(ctx, res)
if conf.keepalive then
diff --git a/apisix/plugins/ai-proxy-multi.lua b/apisix/plugins/ai-proxy-multi.lua
index 2a48bd4dc..47e2e00bc 100644
--- a/apisix/plugins/ai-proxy-multi.lua
+++ b/apisix/plugins/ai-proxy-multi.lua
@@ -45,6 +45,26 @@ local _M = {
schema = schema.ai_proxy_multi_schema,
}
+local function fallback_strategy_has(strategy, name)
+ if not strategy then
+ return false
+ end
+
+ if type(strategy) == "string" then
+ return strategy == name
+ end
+
+ if type(strategy) == "table" then
+ for _, v in ipairs(strategy) do
+ if v == name then
+ return true
+ end
+ end
+ end
+
+ return false
+end
+
local function get_chash_key_schema(hash_on)
if hash_on == "vars" then
@@ -306,7 +326,8 @@ local function pick_target(ctx, conf, ups_tab)
return nil, nil, err
end
ctx.balancer_server = instance_name
- if conf.fallback_strategy == "instance_health_and_rate_limiting" then
+ if conf.fallback_strategy == "instance_health_and_rate_limiting" or -- for backwards compatible
+ fallback_strategy_has(conf.fallback_strategy, "rate_limiting") then
local ai_rate_limiting = require("apisix.plugins.ai-rate-limiting")
for _ = 1, #conf.instances do
if ai_rate_limiting.check_instance_status(nil, ctx, instance_name) then
@@ -363,7 +384,32 @@ function _M.access(conf, ctx)
end
-_M.before_proxy = base.before_proxy
+local function retry_on_error(ctx, conf, code)
+ if not ctx.server_picker then
+ return code
+ end
+ ctx.server_picker.after_balance(ctx, true)
+ if (code == 429 and fallback_strategy_has(conf.fallback_strategy, "http_429")) or
+ (code >= 500 and code < 600 and
+ fallback_strategy_has(conf.fallback_strategy, "http_5xx")) then
+ local name, ai_instance, err = pick_ai_instance(ctx, conf)
+ if err then
+ core.log.error("failed to pick new AI instance: ", err)
+ return 502
+ end
+ ctx.balancer_ip = name
+ ctx.picked_ai_instance_name = name
+ ctx.picked_ai_instance = ai_instance
+ return
+ end
+ return code
+end
+
+function _M.before_proxy(conf, ctx)
+ return base.before_proxy(conf, ctx, function (ctx, conf, code)
+ return retry_on_error(ctx, conf, code)
+ end)
+end
function _M.log(conf, ctx)
if conf.logging then
diff --git a/apisix/plugins/ai-proxy/base.lua b/apisix/plugins/ai-proxy/base.lua
index 4fc032e47..e6e3fd9f3 100644
--- a/apisix/plugins/ai-proxy/base.lua
+++ b/apisix/plugins/ai-proxy/base.lua
@@ -20,7 +20,6 @@ local core = require("apisix.core")
local require = require
local pcall = pcall
local exporter = require("apisix.plugins.prometheus.exporter")
-local bad_request = ngx.HTTP_BAD_REQUEST
local _M = {}
@@ -47,51 +46,63 @@ function _M.set_logging(ctx, summaries, payloads)
end
-function _M.before_proxy(conf, ctx)
- local ai_instance = ctx.picked_ai_instance
- local ai_driver = require("apisix.plugins.ai-drivers." .. ai_instance.provider)
+-- when on_error function is passed, before_proxy will keep on retrying until
+-- on_error returns abort code
+function _M.before_proxy(conf, ctx, on_error)
+ while true do
+ local ai_instance = ctx.picked_ai_instance
+ local ai_driver = require("apisix.plugins.ai-drivers." .. ai_instance.provider)
- local request_body, err = ai_driver.validate_request(ctx)
- if not request_body then
- return bad_request, err
- end
-
- local extra_opts = {
- endpoint = core.table.try_read_attr(ai_instance, "override", "endpoint"),
- query_params = ai_instance.auth.query or {},
- headers = (ai_instance.auth.header or {}),
- model_options = ai_instance.options,
- }
+ local request_body, err = ai_driver.validate_request(ctx)
+ if not request_body then
+ return 400, err
+ end
- if request_body.stream then
- request_body.stream_options = {
- include_usage = true
+ local extra_opts = {
+ endpoint = core.table.try_read_attr(ai_instance, "override", "endpoint"),
+ query_params = ai_instance.auth.query or {},
+ headers = (ai_instance.auth.header or {}),
+ model_options = ai_instance.options,
}
- ctx.var.request_type = "ai_stream"
- else
- ctx.var.request_type = "ai_chat"
- end
- if request_body.model then
- ctx.var.request_llm_model = request_body.model
- end
- local model = ai_instance.options and ai_instance.options.model or request_body.model
- if model then
- ctx.var.llm_model = model
- end
- local do_request = function()
- ctx.llm_request_start_time = ngx.now()
- ctx.var.llm_request_body = request_body
- return ai_driver:request(ctx, conf, request_body, extra_opts)
- end
- exporter.inc_llm_active_connections(ctx)
- local ok, code_or_err, body = pcall(do_request)
- exporter.dec_llm_active_connections(ctx)
- if not ok then
- core.log.error("failed to send request to AI service: ", code_or_err)
- return 500
+ if request_body.stream then
+ request_body.stream_options = {
+ include_usage = true
+ }
+ ctx.var.request_type = "ai_stream"
+ else
+ ctx.var.request_type = "ai_chat"
+ end
+ if request_body.model then
+ ctx.var.request_llm_model = request_body.model
+ end
+ local model = ai_instance.options and ai_instance.options.model or request_body.model
+ if model then
+ ctx.var.llm_model = model
+ end
+
+ local do_request = function()
+ ctx.llm_request_start_time = ngx.now()
+ ctx.var.llm_request_body = request_body
+ return ai_driver:request(ctx, conf, request_body, extra_opts)
+ end
+
+ exporter.inc_llm_active_connections(ctx)
+ local ok, code_or_err, body = pcall(do_request)
+ exporter.dec_llm_active_connections(ctx)
+ if not ok then
+ core.log.error("failed to send request to AI service: ", code_or_err)
+ return 500
+ end
+ if code_or_err and on_error then
+ local abort_code = on_error(ctx, conf, code_or_err)
+ if abort_code then
+ return abort_code, body
+ end
+ else
+ return code_or_err, body
+ end
end
- return code_or_err, body
end
diff --git a/apisix/plugins/ai-proxy/schema.lua b/apisix/plugins/ai-proxy/schema.lua
index bdcb17ee6..994cffd32 100644
--- a/apisix/plugins/ai-proxy/schema.lua
+++ b/apisix/plugins/ai-proxy/schema.lua
@@ -196,9 +196,19 @@ _M.ai_proxy_multi_schema = {
instances = ai_instance_schema,
logging_schema = logging_schema,
fallback_strategy = {
- type = "string",
- enum = { "instance_health_and_rate_limiting" },
- default = "instance_health_and_rate_limiting",
+ anyOf = {
+ {
+ type = "string",
+ enum = {"instance_health_and_rate_limiting", "http_429", "http_5xx"}
+ },
+ {
+ type = "array",
+ items = {
+ type = "string",
+ enum = {"rate_limiting", "http_429", "http_5xx"}
+ }
+ }
+ }
},
timeout = {
type = "integer",
diff --git a/apisix/plugins/ai-request-rewrite.lua b/apisix/plugins/ai-request-rewrite.lua
index 31d181983..356254d9d 100644
--- a/apisix/plugins/ai-request-rewrite.lua
+++ b/apisix/plugins/ai-request-rewrite.lua
@@ -213,11 +213,15 @@ function _M.access(conf, ctx)
}
-- Send request to LLM service
- local _, _, err = request_to_llm(conf, ai_request_table, ctx)
+ local code, _, err = request_to_llm(conf, ai_request_table, ctx)
if err then
core.log.error("failed to request LLM: ", err)
return HTTP_INTERNAL_SERVER_ERROR
end
+ if code == 429 or (code >= 500 and code < 600 ) then
+ core.log.error("LLM service returned error status: ", code)
+ return HTTP_INTERNAL_SERVER_ERROR
+ end
end
return _M
diff --git a/docs/en/latest/plugins/ai-proxy-multi.md b/docs/en/latest/plugins/ai-proxy-multi.md
index 94cd5d841..16c96d848 100644
--- a/docs/en/latest/plugins/ai-proxy-multi.md
+++ b/docs/en/latest/plugins/ai-proxy-multi.md
@@ -51,7 +51,7 @@ In addition, the Plugin also supports logging LLM request information in the acc
| Name | Type | Required | Default | Valid Values | Description |
|------------------------------------|----------------|----------|-----------------------------------|--------------|-------------|
-| fallback_strategy | string | False | instance_health_and_rate_limiting | instance_health_and_rate_limiting | Fallback strategy. When set, the Plugin will check whether the specified instance’s token has been exhausted when a request is forwarded. If so, forward the request to the next instance regardless of the instance priority. When not set, the Plugin will not forward the request to low priority instances when token of the high priority instance is exhausted. |
+| fallback_strategy | string or array | False | | string: "instance_health_and_rate_limiting", "http_429", "http_5xx"<br>array: ["rate_limiting", "http_429", "http_5xx"] | Fallback strategy. When set, the Plugin will check whether the specified instance’s token has been exhausted when a request is forwarded. If so, forward the request to the next instance regardless of the instance priority. When not set, the Plugin will not forward the request to low priorit [...]
| balancer | object | False | | | Load balancing configurations. |
| balancer.algorithm | string | False | roundrobin | [roundrobin, chash] | Load balancing algorithm. When set to `roundrobin`, weighted round robin algorithm is used. When set to `chash`, consistent hashing algorithm is used. |
| balancer.hash_on | string | False | | [vars, headers, cookie, consumer, vars_combinations] | Used when `type` is `chash`. Support hashing on [NGINX variables](https://nginx.org/en/docs/varindex.html), headers, cookie, consumer, or a combination of [NGINX variables](https://nginx.org/en/docs/varindex.html). |
@@ -186,7 +186,7 @@ DeepSeek responses: 2
### Configure Instance Priority and Rate Limiting
-The following example demonstrates how you can configure two models with different priorities and apply rate limiting on the instance with a higher priority. In the case where `fallback_strategy` is set to `instance_health_and_rate_limiting`, the Plugin should continue to forward requests to the low priority instance once the high priority instance's rate limiting quota is fully consumed.
+The following example demonstrates how you can configure two models with different priorities and apply rate limiting on the instance with a higher priority. In the case where `fallback_strategy` is set to `["rate_limiting"]`, the Plugin should continue to forward requests to the low priority instance once the high priority instance's rate limiting quota is fully consumed.
Create a Route as such and update with your LLM providers, models, API keys, and endpoints if applicable:
@@ -199,7 +199,7 @@ curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \
"methods": ["POST"],
"plugins": {
"ai-proxy-multi": {
- "fallback_strategy: "instance_health_and_rate_limiting",
+ "fallback_strategy: ["rate_limiting"],
"instances": [
{
"name": "openai-instance",
@@ -423,7 +423,7 @@ curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \
"plugins": {
"key-auth": {},
"ai-proxy-multi": {
- "fallback_strategy: "instance_health_and_rate_limiting",
+ "fallback_strategy: ["rate_limiting"],
"instances": [
{
"name": "openai-instance",
diff --git a/docs/en/latest/plugins/ai-rate-limiting.md b/docs/en/latest/plugins/ai-rate-limiting.md
index 84bad6161..9cc21a914 100644
--- a/docs/en/latest/plugins/ai-rate-limiting.md
+++ b/docs/en/latest/plugins/ai-rate-limiting.md
@@ -413,9 +413,9 @@ X-AI-RateLimit-Reset-deepseek-instance: 0
### Configure Instance Priority and Rate Limiting
-The following example demonstrates how you can configure two models with different priorities and apply rate limiting on the instance with a higher priority. In the case where `fallback_strategy` is set to `instance_health_and_rate_limiting`, the Plugin should continue to forward requests to the low priority instance once the high priority instance's rate limiting quota is fully consumed.
+The following example demonstrates how you can configure two models with different priorities and apply rate limiting on the instance with a higher priority. In the case where `fallback_strategy` is set to `["rate_limiting"]`, the Plugin should continue to forward requests to the low priority instance once the high priority instance's rate limiting quota is fully consumed.
-Create a Route as such to set rate limiting and a higher priority on `openai-instance` instance and set the `fallback_strategy` to `instance_health_and_rate_limiting`. Update with your LLM providers, models, API keys, and endpoints, if applicable:
+Create a Route as such to set rate limiting and a higher priority on `openai-instance` instance and set the `fallback_strategy` to `["rate_limiting"]`. Update with your LLM providers, models, API keys, and endpoints, if applicable:
```shell
curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \
@@ -426,7 +426,7 @@ curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \
"methods": ["POST"],
"plugins": {
"ai-proxy-multi": {
- "fallback_strategy: "instance_health_and_rate_limiting",
+ "fallback_strategy: ["rate_limiting"],
"instances": [
{
"name": "openai-instance",
@@ -650,7 +650,7 @@ curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \
"plugins": {
"key-auth": {},
"ai-proxy-multi": {
- "fallback_strategy: "instance_health_and_rate_limiting",
+ "fallback_strategy: ["rate_limiting"],
"instances": [
{
"name": "openai-instance",
diff --git a/t/plugin/ai-proxy-multi.balancer.t b/t/plugin/ai-proxy-multi.balancer.t
index 2cf676405..ccc84adc7 100644
--- a/t/plugin/ai-proxy-multi.balancer.t
+++ b/t/plugin/ai-proxy-multi.balancer.t
@@ -47,6 +47,42 @@ _EOC_
$block->set_value("extra_yaml_config", $user_yaml_config);
my $http_config = $block->http_config // <<_EOC_;
+ server {
+ server_name openai_rate_limit;
+ default_type 'application/json';
+ listen 6726;
+ location / {
+ content_by_lua_block {
+ ngx.status = 429
+ ngx.say([[{ "error": {"message":"rate limit exceeded"}}]])
+ return
+ }
+ }
+ }
+ server {
+ server_name openai_internal_error;
+ default_type 'application/json';
+ listen 6727;
+ location / {
+ content_by_lua_block {
+ ngx.status = 500
+ ngx.say([[{ "error": {"message":"internal server error"}}]])
+ return
+ }
+ }
+ }
+ server {
+ server_name openai_internal_error;
+ default_type 'application/json';
+ listen 6728;
+ location / {
+ content_by_lua_block {
+ ngx.status = 503
+ ngx.say([[{ "error": {"message":"service unavailable"}}]])
+ return
+ }
+ }
+ }
server {
server_name openai;
listen 6724;
@@ -349,3 +385,608 @@ GET /t
--- error_log
distribution: deepseek: 2
distribution: openai: 8
+
+
+
+=== TEST 5: set route with fallback_strategy with 500 response code openai.
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "fallback_strategy": ["http_5xx"],
+ "balancer": {
+ "algorithm": "chash",
+ "hash_on": "vars",
+ "key": "query_string"
+ },
+ "instances": [
+ {
+ "name": "openai",
+ "provider": "openai",
+ "weight": 4,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "options": {
+ "model": "gpt-4",
+ "max_tokens": 512,
+ "temperature": 1.0
+ },
+ "override": {
+ "endpoint": "http://localhost:6727"
+ }
+ },
+ {
+ "name": "deepseek",
+ "provider": "deepseek",
+ "weight": 1,
+ "auth": {"header": {"Authorization": "Bearer token"}},
+ "options": {"model": "deepseek-chat","max_tokens": 512,"temperature": 1.0},"override": {"endpoint": "http://localhost:6724/chat/completions"}}
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 6: test all requests success with fallback deepseek
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/anything"
+ local restab = {}
+ local body = [[{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }]]
+ for i = 1, 10 do
+ local httpc = http.new()
+ local query = {
+ index = i
+ }
+ local res, err = httpc:request_uri(uri, {method = "POST", body = body, query = query})
+ if not res then
+ ngx.say(err)
+ return
+ end
+ table.insert(restab, res.body)
+ end
+ local count = {}
+ for _, value in ipairs(restab) do
+ count[value] = (count[value] or 0) + 1
+ end
+ for p, num in pairs(count) do
+ ngx.log(ngx.WARN, "distribution: ", p, ": ", num)
+ end
+ }
+ }
+--- request
+GET /t
+--- timeout: 10
+--- error_log
+distribution: deepseek: 10
+
+
+
+=== TEST 7: set route with fallback_strategy with too many requests openai.
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "fallback_strategy": ["http_429"],
+ "balancer": {
+ "algorithm": "chash",
+ "hash_on": "vars",
+ "key": "query_string"
+ },
+ "instances": [
+ {
+ "name": "openai",
+ "provider": "openai",
+ "weight": 4,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "options": {
+ "model": "gpt-4",
+ "max_tokens": 512,
+ "temperature": 1.0
+ },
+ "override": {
+ "endpoint": "http://localhost:6726"
+ }
+ },
+ {"name":"deepseek","provider":"deepseek","weight":1,"auth":{"header":{"Authorization":"Bearer token"}},"options":{"model":"deepseek-chat","max_tokens":512,"temperature":1.0},"override":{"endpoint":"http://localhost:6724/chat/completions"}}
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 8: test all requests success with fallback deepseek
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/anything"
+ local restab = {}
+ local body = [[{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }]]
+ for i = 1, 10 do
+ local httpc = http.new()
+ local query = {
+ index = i
+ }
+ local res, err = httpc:request_uri(uri, {method = "POST", body = body, query = query})
+ if not res then
+ ngx.say(err)
+ return
+ end
+ table.insert(restab, res.body)
+ end
+ local count = {}
+ for _, value in ipairs(restab) do
+ count[value] = (count[value] or 0) + 1
+ end
+ for p, num in pairs(count) do
+ ngx.log(ngx.WARN, "distribution: ", p, ": ", num)
+ end
+ }
+ }
+--- request
+GET /t
+--- timeout: 10
+--- error_log
+distribution: deepseek: 10
+
+
+
+=== TEST 9: set route with fallback_strategy with unreachable openai.
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "fallback_strategy": ["http_5xx"],
+ "balancer": {
+ "algorithm": "chash",
+ "hash_on": "vars",
+ "key": "query_string"
+ },
+ "instances": [
+ {
+ "name": "openai",
+ "provider": "openai",
+ "weight": 4,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "options": {
+ "model": "gpt-4",
+ "max_tokens": 512,
+ "temperature": 1.0
+ },
+ "override": {
+ "endpoint": "http://localhost:6725"
+ }
+ },
+ {"name":"deepseek","provider":"deepseek","weight":1,"auth":{"header":{"Authorization":"Bearer token"}},"options":{"model":"deepseek-chat","max_tokens":512,"temperature":1.0},"override":{"endpoint":"http://localhost:6724/chat/completions"}}
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 10: test all requests success with fallback deepseek
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/anything"
+ local restab = {}
+ local body = [[{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }]]
+ for i = 1, 10 do
+ local httpc = http.new()
+ local query = {
+ index = i
+ }
+ local res, err = httpc:request_uri(uri, {method = "POST", body = body, query = query})
+ if not res then
+ ngx.say(err)
+ return
+ end
+ table.insert(restab, res.body)
+ end
+ local count = {}
+ for _, value in ipairs(restab) do
+ count[value] = (count[value] or 0) + 1
+ end
+ for p, num in pairs(count) do
+ ngx.log(ngx.WARN, "distribution: ", p, ": ", num)
+ end
+ }
+ }
+--- request
+GET /t
+--- timeout: 10
+--- error_log
+distribution: deepseek: 10
+
+
+
+=== TEST 11: set route with fallback_strategy with service unavailable openai.
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "fallback_strategy": ["http_5xx"],
+ "balancer": {
+ "algorithm": "chash",
+ "hash_on": "vars",
+ "key": "query_string"
+ },
+ "instances": [
+ {
+ "name": "openai",
+ "provider": "openai",
+ "weight": 4,
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "options": {
+ "model": "gpt-4",
+ "max_tokens": 512,
+ "temperature": 1.0
+ },
+ "override": {
+ "endpoint": "http://localhost:6728"
+ }
+ },
+ {"name":"deepseek","provider":"deepseek","weight":1,"auth":{"header":{"Authorization":"Bearer token"}},"options":{"model":"deepseek-chat","max_tokens":512,"temperature":1.0},"override":{"endpoint":"http://localhost:6724/chat/completions"}}
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 12: test all requests success with fallback deepseek
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/anything"
+ local restab = {}
+ local body = [[{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }]]
+ for i = 1, 10 do
+ local httpc = http.new()
+ local query = {
+ index = i
+ }
+ local res, err = httpc:request_uri(uri, {method = "POST", body = body, query = query})
+ if not res then
+ ngx.say(err)
+ return
+ end
+ table.insert(restab, res.body)
+ end
+ local count = {}
+ for _, value in ipairs(restab) do
+ count[value] = (count[value] or 0) + 1
+ end
+ for p, num in pairs(count) do
+ ngx.log(ngx.WARN, "distribution: ", p, ": ", num)
+ end
+ }
+ }
+--- request
+GET /t
+--- timeout: 10
+--- error_log
+distribution: deepseek: 10
+
+
+
+=== TEST 13: set route with fallback_strategy with service unavailable openai having high priority.
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": {
+ "algorithm": "roundrobin",
+ "hash_on": "vars"
+ },
+ "fallback_strategy": [
+ "http_429",
+ "http_5xx"
+ ],
+ "instances": [
+ {"auth":{"header":{"Authorization":"Bearer token"}},"name":"mock-429","override":{"endpoint":"http://localhost:6726"},"priority":10,"provider":"openai-compatible","weight":10},{"auth":{"header":{"Authorization":"Bearer token"}},"name":"mock-500","override":{"endpoint":"http://localhost:6727"},"priority":0,"provider":"openai-compatible","weight":10},{"auth":{"header":{"Authorization":"Bearer token"}},"name":"mock-200","override":{"endpoint":"http://localhost [...]
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 14: test all requests success with fallback deepseek
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/anything"
+ local restab = {}
+ local body = [[{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }]]
+ for i = 1, 10 do
+ local httpc = http.new()
+ local query = {
+ index = i
+ }
+ local res, err = httpc:request_uri(uri, {method = "POST", body = body, query = query})
+ if not res then
+ ngx.say(err)
+ return
+ end
+ table.insert(restab, res.body)
+ end
+ local count = {}
+ for _, value in ipairs(restab) do
+ count[value] = (count[value] or 0) + 1
+ end
+ for p, num in pairs(count) do
+ ngx.log(ngx.WARN, "distribution: ", p, ": ", num)
+ end
+ }
+ }
+--- request
+GET /t
+--- timeout: 10
+--- error_log
+distribution: deepseek: 10
+
+
+
+=== TEST 15: set route with fallback_strategy with only service unavailable and 429.
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/anything",
+ "plugins": {
+ "ai-proxy-multi": {
+ "balancer": {
+ "algorithm": "roundrobin",
+ "hash_on": "vars"
+ },
+ "fallback_strategy": [
+ "http_429",
+ "http_5xx"
+ ],
+ "instances": [
+ {
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "name": "mock-429",
+ "override": {
+ "endpoint": "http://localhost:6726"
+ },
+ "priority": 10,
+ "provider": "openai-compatible",
+ "weight": 10
+ },
+ {
+ "auth": {
+ "header": {
+ "Authorization": "Bearer token"
+ }
+ },
+ "name": "mock-500",
+ "override": {
+ "endpoint": "http://localhost:6727"
+ },
+ "priority": 0,
+ "provider": "openai-compatible",
+ "weight": 10
+ }
+ ],
+ "ssl_verify": false
+ }
+ }
+ }]]
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 16: test all requests success with fallback deepseek should return 502
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/anything"
+ local restab = {}
+ local body = [[{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }]]
+ for i = 1, 10 do
+ local httpc = http.new()
+ local query = {
+ index = i
+ }
+ local res, err = httpc:request_uri(uri, {method = "POST", body = body, query = query})
+ if not res then
+ ngx.say(err)
+ return
+ end
+ table.insert(restab, res.status)
+ end
+ local count = {}
+ for _, value in ipairs(restab) do
+ count[value] = (count[value] or 0) + 1
+ end
+ for p, num in pairs(count) do
+ ngx.log(ngx.WARN, "distribution: ", p, ": ", num)
+ end
+ }
+ }
+--- request
+GET /t
+--- timeout: 10
+--- error_log
+distribution: 502: 10
+
+
+
+=== TEST 17: set route with only one instance and configure it with http_5xx fallback_strategy
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local json = require("toolkit.json")
+ local request_body = {
+ uri = "/anything",
+ plugins = {
+ ["ai-proxy-multi"] = {
+ fallback_strategy = {"http_5xx"},
+ instances = {
+ {
+ name = "deepseek",
+ provider = "deepseek",
+ auth = {
+ header = {Authorization = "Bearer token" }
+ },
+ weight = 1,
+ override = { endpoint = "http://localhost:6727" }
+ }
+ },
+ ssl_verify = false
+ }
+ }
+ }
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ json.encode(request_body)
+ )
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 18: send request
+--- request
+POST /anything
+{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }
+--- more_headers
+Host: openai_internal_error
+--- error_code: 500
+--- no_error_log
+[error]