You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@casbin.apache.org by hs...@apache.org on 2026/08/31 03:17:07 UTC
(casbin-gateway) branch master updated: fix(proxy): fail over from a rejected provider and record why
This is an automated email from the ASF dual-hosted git repository.
hsluoyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/casbin-gateway.git
The following commit(s) were added to refs/heads/master by this push:
new 9f0a3bb fix(proxy): fail over from a rejected provider and record why
9f0a3bb is described below
commit 9f0a3bba7605af635f9ff4a8355b4c0376fd0bab
Author: Yang Luo <hs...@gmail.com>
AuthorDate: Mon Aug 31 11:16:59 2026 +0800
fix(proxy): fail over from a rejected provider and record why
---
controllers/proxy.go | 52 ++++++++++++--------
controllers/proxy_record.go | 39 ++++++++++++++-
controllers/proxy_test.go | 6 +--
object/llm_record.go | 18 ++++++-
web/src/locales/en/data.json | 6 +++
web/src/locales/zh/data.json | 6 +++
web/src/pages/LlmRecordsPage.tsx | 101 +++++++++++++++++++++++++++++++++++----
web/src/types.ts | 10 ++++
8 files changed, 205 insertions(+), 33 deletions(-)
diff --git a/controllers/proxy.go b/controllers/proxy.go
index c765507..5a3947a 100644
--- a/controllers/proxy.go
+++ b/controllers/proxy.go
@@ -390,6 +390,7 @@ func (c *ApiController) forwardToProviders(providers []*object.Provider, route *
if written {
return
}
+ route.recordFailure(provider.GetId(), status, message)
lastStatus, lastMessage = status, message
}
@@ -473,9 +474,14 @@ func (c *ApiController) forwardToProvider(provider *object.Provider, route *prox
if !isLast && isRetryableStatus(upstreamResp.StatusCode) {
beego.Error("provider", provider.GetId(), "returned a retryable status:", upstreamResp.Status)
- // Drain a bounded amount so the connection can be pooled and reused.
- _, _ = io.Copy(io.Discard, io.LimitReader(upstreamResp.Body, 4096))
- return http.StatusBadGateway, fmt.Sprintf("upstream returned %s", upstreamResp.Status), false
+ // Read a bounded amount rather than discarding it: the connection can
+ // still be pooled, and the reason this provider was skipped is in there.
+ skipped, _ := io.ReadAll(io.LimitReader(upstreamResp.Body, maxErrorBodyBytes))
+ message := fmt.Sprintf("upstream returned %s", upstreamResp.Status)
+ if _, detail := protocol.ReadError(skipped, ""); detail != "" {
+ message += ": " + detail
+ }
+ return upstreamResp.StatusCode, message, false
}
// Abort the request when the upstream goes silent, rather than capping the
@@ -490,10 +496,15 @@ func (c *ApiController) forwardToProvider(provider *object.Provider, route *prox
}
// The tap reads the counters out of the upstream answer as it passes, so it
- // sees them in the provider's own spelling, translated or not.
+ // sees them in the provider's own spelling, translated or not. On a failure
+ // the same bytes are the error the client was handed.
tap := &usageTap{reader: body}
c.relayResponse(route, upstream, upstreamResp, tap)
- route.recordUsage(tap.tail)
+ if isSuccessStatus(upstreamResp.StatusCode) {
+ route.recordUsage(tap.tail)
+ } else {
+ route.recordErrorBody(tap.tail)
+ }
return 0, "", true
}
@@ -514,20 +525,16 @@ func (c *ApiController) answerCountTokens(route *proxyRoute) (int, string, bool)
return 0, "", true
}
-// reportProviderStatus feeds the breaker that decides in which order providers are
-// tried. A status the upstream itself rejected the request with counts as a
-// provider failure: a wrong key or an exhausted quota is not something the next
-// request will do better.
+// reportProviderStatus feeds the breaker that decides in which order providers
+// are tried. Every status worth failing over from counts as a provider failure:
+// a wrong key or an exhausted quota is not something the next request will do
+// better.
func reportProviderStatus(provider *object.Provider, statusCode int) {
- switch {
- case isRetryableStatus(statusCode):
- object.ReportProviderFailure(provider.GetId(), fmt.Sprintf("upstream returned %d", statusCode))
- case statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden ||
- statusCode == http.StatusPaymentRequired:
- object.ReportProviderFailure(provider.GetId(), fmt.Sprintf("upstream rejected the credentials with %d", statusCode))
- default:
+ if !isRetryableStatus(statusCode) {
object.ReportProviderSuccess(provider.GetId())
+ return
}
+ object.ReportProviderFailure(provider.GetId(), fmt.Sprintf("upstream returned %d", statusCode))
}
// allowRelay decides whether a request may use the providers stored here. A
@@ -782,10 +789,17 @@ func isSuccessStatus(statusCode int) bool {
}
// isRetryableStatus reports whether another provider is worth trying. A rate
-// limit or an upstream-side error is transient or specific to that provider,
-// while a 4xx caused by the request itself would fail the same way everywhere.
+// limit, an upstream-side error, a credential this provider rejected, a quota it
+// has run out of and a model it does not serve are all specific to that
+// provider, while a 4xx caused by the request itself would fail the same way
+// everywhere.
func isRetryableStatus(statusCode int) bool {
- return statusCode == http.StatusTooManyRequests || statusCode >= 500
+ switch statusCode {
+ case http.StatusUnauthorized, http.StatusPaymentRequired, http.StatusForbidden,
+ http.StatusNotFound, http.StatusRequestTimeout, http.StatusTooManyRequests:
+ return true
+ }
+ return statusCode >= 500
}
// providerUnusableReason reports why the proxy cannot forward to a provider, or
diff --git a/controllers/proxy_record.go b/controllers/proxy_record.go
index 4e97df3..0774d2c 100644
--- a/controllers/proxy_record.go
+++ b/controllers/proxy_record.go
@@ -20,13 +20,22 @@ import (
"io"
"time"
+ "github.com/apache/casbin-gateway/auditutil"
"github.com/apache/casbin-gateway/object"
+ "github.com/apache/casbin-gateway/protocol"
)
// usageTailBytes is how much of the end of a response is kept to read the token
// usage out of. Both APIs report it last, so the tail is enough.
const usageTailBytes = 16 * 1024
+// llmErrorBodyBytes bounds the upstream failure kept on a record, and
+// llmErrorMessageBytes the one-line reason shown beside it.
+const (
+ llmErrorBodyBytes = 8 * 1024
+ llmErrorMessageBytes = 500
+)
+
// finishLlmRecord queues the record of one relayed request. Every path out of
// forwardToProviders ends the client request, so this is the only place it is
// called from.
@@ -54,7 +63,35 @@ func (route *proxyRoute) recordOutcome(status int, message string) {
return
}
route.record.Status = status
- route.record.Error = message
+ route.record.Error = auditutil.BoundString(message, llmErrorMessageBytes)
+}
+
+// recordErrorBody keeps what the upstream answered a failed request with. It is
+// the only place the record learns why a status code was what it was, so it
+// runs whichever mode recording is in.
+func (route *proxyRoute) recordErrorBody(raw []byte) {
+ if route.record == nil || len(raw) == 0 {
+ return
+ }
+
+ route.record.ErrorBody = auditutil.SanitizeJSON(string(raw), llmErrorBodyBytes)
+ if route.record.Error == "" {
+ if _, message := protocol.ReadError(raw, ""); message != "" {
+ route.record.Error = auditutil.BoundString(message, llmErrorMessageBytes)
+ }
+ }
+}
+
+// recordFailure keeps one attempt the chain failed over from.
+func (route *proxyRoute) recordFailure(providerId string, status int, message string) {
+ if route.record == nil {
+ return
+ }
+ route.record.Failures = append(route.record.Failures, object.LlmFailure{
+ Provider: providerId,
+ Status: status,
+ Error: auditutil.BoundString(message, llmErrorMessageBytes),
+ })
}
func (route *proxyRoute) recordUsage(tail []byte) {
diff --git a/controllers/proxy_test.go b/controllers/proxy_test.go
index 5667629..c5af929 100644
--- a/controllers/proxy_test.go
+++ b/controllers/proxy_test.go
@@ -73,12 +73,12 @@ func TestIsEventStream(t *testing.T) {
}
func TestIsRetryableStatus(t *testing.T) {
- for _, statusCode := range []int{429, 500, 502, 503} {
+ for _, statusCode := range []int{401, 402, 403, 404, 429, 500, 502, 503} {
if !isRetryableStatus(statusCode) {
t.Errorf("status %d should be retryable", statusCode)
}
}
- for _, statusCode := range []int{200, 400, 401, 404} {
+ for _, statusCode := range []int{200, 400, 422} {
if isRetryableStatus(statusCode) {
t.Errorf("status %d should not be retryable", statusCode)
}
@@ -268,7 +268,7 @@ func TestForwardToProvider(t *testing.T) {
if written {
t.Fatal("a retryable status was relayed instead of failing over")
}
- if statusCode != http.StatusBadGateway || !strings.Contains(message, "503") {
+ if statusCode != http.StatusServiceUnavailable || !strings.Contains(message, "overloaded") {
t.Errorf("statusCode = %d, message = %s", statusCode, message)
}
if recorder.Body.Len() != 0 {
diff --git a/object/llm_record.go b/object/llm_record.go
index c65d10d..810c22f 100644
--- a/object/llm_record.go
+++ b/object/llm_record.go
@@ -38,6 +38,14 @@ const llmOversizeFactor = 4
// message and tool stays visible where dropping the body would keep none.
var llmStringCaps = []int{16384, 4096, 1024, 256}
+// LlmFailure is one provider attempt that did not answer, kept on the record of
+// the request it was made for.
+type LlmFailure struct {
+ Provider string `json:"provider"`
+ Status int `json:"status"`
+ Error string `json:"error"`
+}
+
// LlmRecord is one client request relayed by the LLM proxy, together with the
// outcome it ended in. Payload is filled only in "full" mode, and only after
// auditutil has removed the credentials the body may carry.
@@ -58,6 +66,13 @@ type LlmRecord struct {
Attempts int `xorm:"int" json:"attempts"`
Error string `xorm:"varchar(500)" json:"error"`
+ // ErrorBody is what the upstream itself answered with, which is where the
+ // reason for a failure normally is: the status code alone rarely says.
+ ErrorBody string `xorm:"mediumtext" json:"errorBody"`
+ // Failures are the attempts the chain failed over from, so a record shows
+ // which providers were tried and why each of them did not answer.
+ Failures []LlmFailure `xorm:"mediumtext" json:"failures"`
+
// PromptTokens counts only the input billed as fresh: the cached part is
// reported separately, so the counters can be added up without counting a
// token twice.
@@ -524,7 +539,7 @@ func GetLlmRecords(filter LlmRecordFilter, offset int, limit int) ([]*LlmRecord,
session := llmRecordSession(filter)
defer session.Close()
records := []*LlmRecord{}
- err = session.Omit("payload").Desc("id").Limit(limit, offset).Find(&records)
+ err = session.Omit("payload", "error_body").Desc("id").Limit(limit, offset).Find(&records)
return records, count, err
}
@@ -734,6 +749,7 @@ func publishLlmRecord(record *LlmRecord) {
}
summary := *record
summary.Payload = ""
+ summary.ErrorBody = ""
for _, feed := range llmHub.subscribers {
select {
case feed <- &summary:
diff --git a/web/src/locales/en/data.json b/web/src/locales/en/data.json
index 61db768..5a6fb88 100644
--- a/web/src/locales/en/data.json
+++ b/web/src/locales/en/data.json
@@ -376,6 +376,8 @@
"Attempts": "Attempts",
"Bodies are not stored": "Request bodies are not stored",
"Bodies are not stored detail": "llmRecordMode is \"metadata\", so the prompts, tool schemas and messages behind these rows were never kept. Store them from now on to inspect them.",
+ "Bodies are stored": "Prompts and responses are being stored",
+ "Bodies are stored detail": "llmRecordMode is \"full\", so the prompts, tool schemas and messages of every relayed request are kept, with the credentials removed. Record metadata only, or stop recording, to keep less.",
"Body is not JSON": "The stored body is not valid JSON, so it is shown as it was captured.",
"Body not stored": "The request body was not stored: llmRecordMode is \"metadata\".",
"Body too large to store": "The request body was larger than the capture limit and was not stored.",
@@ -408,6 +410,7 @@
"Mode metadata": "metadata only",
"Mode off": "off",
"Mode {mode}, kept for {days} days, up to {max} records": "Mode {mode}, kept for {days} days, up to {max} records",
+ "No error message": "The upstream gave no error message",
"No messages": "This request carried no messages.",
"No parameters": "This request carried no parameters of its own.",
"No price for this model": "No price is configured for this model",
@@ -419,6 +422,7 @@
"Output tokens": "Output tokens",
"Parameters": "Parameters",
"Provider": "Provider",
+ "Providers tried first": "Providers tried first",
"Raw": "Raw",
"Record metadata": "Record metadata",
"Record metadata and bodies": "Record metadata and bodies",
@@ -429,6 +433,7 @@
"Records were dropped": "{count} records were dropped because the write queue was full. Raise llmRecordQueueCapacity if this keeps happening.",
"Redacted values": "Redacted values",
"Request": "Request",
+ "Request failed": "Request failed",
"Request size": "Request size",
"Requests": "Requests",
"Show all {chars}": "Show all {chars}",
@@ -442,6 +447,7 @@
"Tool result": "tool result",
"Tools": "Tools",
"Unknown role": "unknown role",
+ "Upstream response": "Upstream response",
"all succeeded": "all succeeded",
"blocks": "blocks",
"chars": "chars",
diff --git a/web/src/locales/zh/data.json b/web/src/locales/zh/data.json
index 6d2b1ea..68ceb2e 100644
--- a/web/src/locales/zh/data.json
+++ b/web/src/locales/zh/data.json
@@ -376,6 +376,8 @@
"Attempts": "尝试次数",
"Bodies are not stored": "未保存请求体",
"Bodies are not stored detail": "llmRecordMode 为 \"metadata\"，因此这些记录背后的提示词、工具 schema 和消息都没有保存。开启保存后即可查看。",
+ "Bodies are stored": "正在保存请求内容",
+ "Bodies are stored detail": "llmRecordMode 为 \"full\"，转发的每一条请求，其提示词、工具 schema 和消息都会在去除凭据后保存下来。可改为只记录元数据，或直接关闭记录。",
"Body is not JSON": "保存的请求体不是合法 JSON，这里按捕获时的原样显示。",
"Body not stored": "未保存请求正文：llmRecordMode 为 \"metadata\"。",
"Body too large to store": "请求正文超过采集上限，未予保存。",
@@ -408,6 +410,7 @@
"Mode metadata": "仅元数据",
"Mode off": "关闭",
"Mode {mode}, kept for {days} days, up to {max} records": "模式：{mode}，保留 {days} 天，最多 {max} 条",
+ "No error message": "上游没有给出错误信息",
"No messages": "该请求没有消息。",
"No parameters": "该请求没有额外参数。",
"No price for this model": "该模型没有配置价格",
@@ -419,6 +422,7 @@
"Output tokens": "输出 Token",
"Parameters": "参数",
"Provider": "供应商",
+ "Providers tried first": "先尝试过的供应商",
"Raw": "原始 JSON",
"Record metadata": "只记录元数据",
"Record metadata and bodies": "记录元数据和正文",
@@ -429,6 +433,7 @@
"Records were dropped": "有 {count} 条记录因写入队列已满而被丢弃。如持续出现，请调大 llmRecordQueueCapacity。",
"Redacted values": "已脱敏字段数",
"Request": "请求内容",
+ "Request failed": "请求失败",
"Request size": "请求大小",
"Requests": "请求数",
"Show all {chars}": "展开全部 {chars}",
@@ -442,6 +447,7 @@
"Tool result": "工具结果",
"Tools": "工具",
"Unknown role": "未知角色",
+ "Upstream response": "上游返回的内容",
"all succeeded": "全部成功",
"blocks": "块",
"chars": "字符",
diff --git a/web/src/pages/LlmRecordsPage.tsx b/web/src/pages/LlmRecordsPage.tsx
index 7a039e3..38a6e79 100644
--- a/web/src/pages/LlmRecordsPage.tsx
+++ b/web/src/pages/LlmRecordsPage.tsx
@@ -13,7 +13,7 @@
// limitations under the License.
import * as React from "react";
-import {Activity, CircleDollarSign, Database, Hash, Logs, Play, RefreshCw, Trash2} from "lucide-react";
+import {Activity, CircleDollarSign, Database, Hash, Logs, Play, RefreshCw, Square, Trash2} from "lucide-react";
import i18next from "i18next";
import * as LlmRecordBackend from "@/backend/LlmRecordBackend";
@@ -23,7 +23,7 @@ import {RequestInspector} from "@/components/llm/request-inspector";
import {DataTable, type Column} from "@/components/shared/data-table";
import {ConfirmDialog} from "@/components/shared/confirm-dialog";
import {Loading} from "@/components/shared/loading";
-import {CodeText, DescriptionList, UnauthorizedResult} from "@/components/shared/misc";
+import {CodeBlock, CodeText, DescriptionList, UnauthorizedResult} from "@/components/shared/misc";
import {PageContainer, PageHeader} from "@/components/shared/page-header";
import {SimpleSelect} from "@/components/shared/simple-select";
import {StatCard} from "@/components/shared/stat-card";
@@ -64,6 +64,56 @@ function tokenBreakdown(record: LlmRecord) {
].join("\n");
}
+function isFailed(record: LlmRecord) {
+ return record.status < 200 || record.status >= 300;
+}
+
+/** The upstream error is stored as it arrived, so JSON is laid out to be read. */
+function formatErrorBody(body: string) {
+ try {
+ return JSON.stringify(JSON.parse(body), null, 2);
+ } catch {
+ return body;
+ }
+}
+
+/** What went wrong, which is the part a status code on its own never says. A
+ * request that failed over and then succeeded still lists what it failed over
+ * from. */
+function RecordFailure({record, errorBody}: {record: LlmRecord; errorBody: string}) {
+ return (
+ <div className="flex flex-col gap-2">
+ {isFailed(record) || record.error ? (
+ <MessageAlert
+ variant="destructive"
+ title={i18next.t("llm:Request failed")}
+ description={record.error || i18next.t("llm:No error message")}
+ />
+ ) : null}
+ {record.failures?.length ? (
+ <div className="flex flex-col gap-1">
+ <span className="text-muted-foreground text-xs">{i18next.t("llm:Providers tried first")}</span>
+ {record.failures.map((failure, index) => (
+ <div key={index} className="flex min-w-0 flex-wrap items-center gap-2 text-xs">
+ <Badge variant={statusVariant(failure.status)}>{failure.status || i18next.t("llm:No response")}</Badge>
+ <CodeText>{failure.provider}</CodeText>
+ <span className="text-muted-foreground min-w-0 break-all">{failure.error}</span>
+ </div>
+ ))}
+ </div>
+ ) : null}
+ {errorBody ? (
+ <div className="flex flex-col gap-1">
+ <span className="text-muted-foreground text-xs">{i18next.t("llm:Upstream response")}</span>
+ <CodeBlock copyable maxHeight="16rem">
+ {formatErrorBody(errorBody)}
+ </CodeBlock>
+ </div>
+ ) : null}
+ </div>
+ );
+}
+
/** Mounted only once a row is expanded, which is when the body is fetched. */
function RecordDetail({record, onDelete}: {record: LlmRecord; onDelete: () => void}) {
const [detail, setDetail] = React.useState<LlmRecord | null>(null);
@@ -120,10 +170,13 @@ function RecordDetail({record, onDelete}: {record: LlmRecord; onDelete: () => vo
},
{label: i18next.t("llm:Request size"), value: `${record.bytes.toLocaleString()} B`},
{label: i18next.t("llm:Redacted values"), value: record.redactions},
- record.error && {label: i18next.t("llm:Error"), value: record.error},
]}
/>
+ {isFailed(record) || record.error || record.failures?.length ? (
+ <RecordFailure record={record} errorBody={detail?.errorBody ?? ""} />
+ ) : null}
+
{error ? <MessageAlert title={error} /> : null}
{record.truncated ? <MessageAlert variant="warning" title={i18next.t("llm:Body was shortened")} /> : null}
@@ -442,7 +495,11 @@ export default function LlmRecordsPage({account}: {account: Account}) {
<span className="truncate text-xs" title={record.summary || record.error}>
{record.summary || record.error}
</span>
- {record.messageCount > 0 || record.toolCount > 0 ? (
+ {record.summary && record.error ? (
+ <span className="text-destructive truncate text-xs" title={record.error}>
+ {record.error}
+ </span>
+ ) : record.messageCount > 0 || record.toolCount > 0 ? (
<span className="text-muted-foreground truncate text-xs">
{record.messageCount} {i18next.t("llm:messages")} · {record.toolCount} {i18next.t("llm:tools")}
{record.systemBytes > 0
@@ -457,6 +514,13 @@ export default function LlmRecordsPage({account}: {account: Account}) {
const modeOff = status !== null && status.mode === "off";
const modeMetadata = status !== null && status.mode === "metadata";
+ const modeFull = status !== null && status.mode === "full";
+ const stopRecording = (
+ <Button size="sm" variant="outline" onClick={() => setRecordMode("off")} loading={changingMode}>
+ <Square />
+ {i18next.t("llm:Recording off")}
+ </Button>
+ );
const description = status
? i18next
.t("llm:Mode {mode}, kept for {days} days, up to {max} records")
@@ -543,14 +607,33 @@ export default function LlmRecordsPage({account}: {account: Account}) {
) : null}
{modeMetadata ? (
<MessageAlert
- variant="info"
+ variant="warning"
title={i18next.t("llm:Bodies are not stored")}
description={i18next.t("llm:Bodies are not stored detail")}
action={
- <Button size="sm" variant="outline" onClick={() => setRecordMode("full")} loading={changingMode}>
- <Play />
- {i18next.t("llm:Record metadata and bodies")}
- </Button>
+ <div className="flex flex-wrap gap-2">
+ <Button size="sm" variant="outline" onClick={() => setRecordMode("full")} loading={changingMode}>
+ <Play />
+ {i18next.t("llm:Record metadata and bodies")}
+ </Button>
+ {stopRecording}
+ </div>
+ }
+ />
+ ) : null}
+ {modeFull ? (
+ <MessageAlert
+ variant="warning"
+ title={i18next.t("llm:Bodies are stored")}
+ description={i18next.t("llm:Bodies are stored detail")}
+ action={
+ <div className="flex flex-wrap gap-2">
+ <Button size="sm" variant="outline" onClick={() => setRecordMode("metadata")} loading={changingMode}>
+ <Logs />
+ {i18next.t("llm:Record metadata")}
+ </Button>
+ {stopRecording}
+ </div>
}
/>
) : null}
diff --git a/web/src/types.ts b/web/src/types.ts
index 210867d..3953d7b 100644
--- a/web/src/types.ts
+++ b/web/src/types.ts
@@ -238,6 +238,12 @@ export interface AgentTranscript {
truncated?: boolean;
}
+export interface LlmFailure {
+ provider: string;
+ status: number;
+ error: string;
+}
+
export interface LlmRecord {
id: number;
createdTime: string;
@@ -252,6 +258,10 @@ export interface LlmRecord {
durationMs: number;
attempts: number;
error: string;
+ /** What the upstream answered a failed request with. Only returned by getLlmRecord. */
+ errorBody: string;
+ /** The attempts the provider chain failed over from, oldest first. */
+ failures: LlmFailure[];
/** Input billed as fresh: the cached part is counted separately. */
promptTokens: number;
completionTokens: number;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@casbin.apache.org
For additional commands, e-mail: commits-help@casbin.apache.org