You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2026/10/14 13:34:41 UTC
(skywalking) branch master updated: Make AI evaluation judge HTTP timeout cover response body reads (#14134)
This is an automated email from the ASF dual-hosted git repository.
wu-sheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 1092e512e9 Make AI evaluation judge HTTP timeout cover response body reads (#14134)
1092e512e9 is described below
commit 1092e512e963fad27fa867a6689206cf3656c475
Author: songzhendong <xi...@qq.com>
AuthorDate: Wed Oct 14 21:34:33 2026 +0800
Make AI evaluation judge HTTP timeout cover response body reads (#14134)
---
docs/en/setup/backend/ai-evaluation.md | 2 +-
.../judge/provider/OpenAICompatibleProvider.java | 31 +++++++-
.../provider/OpenAICompatibleProviderTest.java | 85 +++++++++++++++++++++-
.../src/main/resources/ai-evaluation.yml | 1 +
4 files changed, 113 insertions(+), 6 deletions(-)
diff --git a/docs/en/setup/backend/ai-evaluation.md b/docs/en/setup/backend/ai-evaluation.md
index 8e7d5a79cd..13b8a956c9 100644
--- a/docs/en/setup/backend/ai-evaluation.md
+++ b/docs/en/setup/backend/ai-evaluation.md
@@ -54,7 +54,7 @@ tasks:
`provider` must currently be `openai`. The endpoint must accept the OpenAI Chat Completions request shape and return a response with `choices[0].message.content`. The API key is sent as a Bearer token.
-`request-timeout-seconds` defaults to `30`. `max-retries` defaults to `2` and is limited to `5`; timeout, HTTP 429, and HTTP 5xx responses may be retried. `temperature` must be between `0` and `1`. `max_tokens` must be a positive integer.
+`request-timeout-seconds` defaults to `30`. It is the deadline of each judge HTTP request, covering connecting, waiting for the response and reading the response body; a retry gets a deadline of its own. `max-retries` defaults to `2` and is limited to `5`; timeout, HTTP 429, and HTTP 5xx responses may be retried. `temperature` must be between `0` and `1`. `max_tokens` must be a positive integer.
## Sampling and queue settings
diff --git a/oap-server/analyzer/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProvider.java b/oap-server/analyzer/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProvider.java
index 09f084f758..dcf2625cab 100644
--- a/oap-server/analyzer/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProvider.java
+++ b/oap-server/analyzer/ai-evaluation/src/main/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProvider.java
@@ -31,6 +31,7 @@ import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Optional;
import java.util.Properties;
+import java.util.concurrent.TimeUnit;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
@@ -43,12 +44,18 @@ import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelProvider;
import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelRequest;
import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelResponse;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
+import org.apache.skywalking.oap.server.library.util.JdkHttpClientUtils;
import org.apache.skywalking.oap.server.library.util.PropertyPlaceholderHelper;
import org.apache.skywalking.oap.server.library.util.StringUtil;
public class OpenAICompatibleProvider implements JudgeModelProvider {
private static final Gson GSON = new Gson();
private static final long DEFAULT_REQUEST_TIMEOUT_SECONDS = 30L;
+ /**
+ * Keeps the JDK HTTP client's own timer arithmetic from overflowing: {@code Long.MAX_VALUE} seconds is accepted
+ * by {@code HttpRequest.Builder#timeout} and then fails every send.
+ */
+ private static final long MAX_REQUEST_TIMEOUT_SECONDS = TimeUnit.NANOSECONDS.toSeconds(Long.MAX_VALUE);
private static final int DEFAULT_MAX_RETRIES = 2;
private static final int MAX_CONFIGURED_RETRIES = 5;
private static final long DEFAULT_RETRY_DELAY_MILLIS = 200L;
@@ -66,7 +73,7 @@ public class OpenAICompatibleProvider implements JudgeModelProvider {
private final Integer maxTokens;
public OpenAICompatibleProvider(final Properties config) throws ModuleStartException {
- this(HttpClient.newHttpClient(), config);
+ this(newHttpClient(config), config);
}
OpenAICompatibleProvider(final HttpClient httpClient, final Properties config) throws ModuleStartException {
@@ -81,11 +88,22 @@ public class OpenAICompatibleProvider implements JudgeModelProvider {
this.maxTokens = getInteger(config, "max_tokens");
}
+ static HttpClient newHttpClient(final Properties config) throws ModuleStartException {
+ // Validate first, so a non-positive timeout fails as a ModuleStartException, not connectTimeout's
+ // IllegalArgumentException.
+ validate(config);
+ return HttpClient.newBuilder()
+ .connectTimeout(Duration.ofSeconds(getRequestTimeoutSeconds(config)))
+ .build();
+ }
+
@Override
public JudgeModelResponse judge(final JudgeModelRequest request)
throws IOException, InterruptedException {
final HttpRequest httpRequest = HttpRequest.newBuilder()
.uri(URI.create(endpoint))
+ // JdkHttpClientUtils requires it: only the request's own timeout aborts an exchange still waiting
+ // for the response headers.
.timeout(requestTimeout)
.header("Authorization", "Bearer " + apiKey)
.header("Content-Type", "application/json")
@@ -94,7 +112,7 @@ public class OpenAICompatibleProvider implements JudgeModelProvider {
for (int retry = 0; ; retry++) {
final HttpResponse<String> response;
try {
- response = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString());
+ response = JdkHttpClientUtils.sendStringWithTimeout(httpClient, httpRequest, requestTimeout);
} catch (HttpTimeoutException e) {
if (retry >= maxRetries) {
throw new JudgeModelException(Reason.TIMEOUT, "OpenAI compatible judge API request timed out.", e);
@@ -312,19 +330,24 @@ public class OpenAICompatibleProvider implements JudgeModelProvider {
}
}
- private static long getRequestTimeoutSeconds(final Properties properties) throws ModuleStartException {
+ static long getRequestTimeoutSeconds(final Properties properties) throws ModuleStartException {
final String value = getString(properties, "request-timeout-seconds");
if (StringUtil.isBlank(value)) {
return DEFAULT_REQUEST_TIMEOUT_SECONDS;
}
+ final long parsed;
try {
- return Long.parseLong(value);
+ parsed = Long.parseLong(value);
} catch (NumberFormatException e) {
throw new ModuleStartException(
"AI evaluation judge config [request-timeout-seconds] must be an integer.",
e
);
}
+ if (parsed > MAX_REQUEST_TIMEOUT_SECONDS) {
+ return MAX_REQUEST_TIMEOUT_SECONDS;
+ }
+ return parsed;
}
private static int getMaxRetries(final Properties properties) throws ModuleStartException {
diff --git a/oap-server/analyzer/ai-evaluation/src/test/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProviderTest.java b/oap-server/analyzer/ai-evaluation/src/test/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProviderTest.java
index cc6a3ffc07..a77ed454ac 100644
--- a/oap-server/analyzer/ai-evaluation/src/test/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProviderTest.java
+++ b/oap-server/analyzer/ai-evaluation/src/test/java/org/apache/skywalking/oap/server/ai/evaluation/judge/provider/OpenAICompatibleProviderTest.java
@@ -25,7 +25,10 @@ import java.io.IOException;
import java.lang.reflect.Method;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
+import java.time.Duration;
import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import com.sun.net.httpserver.HttpExchange;
@@ -36,7 +39,9 @@ import org.apache.skywalking.oap.server.ai.evaluation.judge.JudgeModelRequest;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+@Timeout(value = 20, unit = TimeUnit.SECONDS)
class OpenAICompatibleProviderTest {
private static final String SUCCESS_RESPONSE =
"{\"choices\":[{\"message\":{\"content\":\"valid evaluation\"}}]}";
@@ -47,9 +52,42 @@ class OpenAICompatibleProviderTest {
void tearDown() {
if (server != null) {
server.stop(0);
+ server = null;
}
}
+ @Test
+ void shouldDefaultRequestTimeoutTo30SecondsAndWireConnectTimeout() throws Exception {
+ final Properties config = validConfig();
+ assertEquals(30L, OpenAICompatibleProvider.getRequestTimeoutSeconds(config));
+ assertEquals(
+ Duration.ofSeconds(30),
+ OpenAICompatibleProvider.newHttpClient(config).connectTimeout().orElseThrow()
+ );
+ }
+
+ @Test
+ void shouldApplyConfiguredRequestTimeoutToConnectTimeout() throws Exception {
+ final Properties config = validConfig();
+ config.setProperty("request-timeout-seconds", "7");
+ assertEquals(7L, OpenAICompatibleProvider.getRequestTimeoutSeconds(config));
+ assertEquals(
+ Duration.ofSeconds(7),
+ OpenAICompatibleProvider.newHttpClient(config).connectTimeout().orElseThrow()
+ );
+ }
+
+ @Test
+ void shouldCapExtremeRequestTimeoutSeconds() throws Exception {
+ final Properties config = validConfig();
+ config.setProperty("request-timeout-seconds", "10000000000");
+ assertEquals(9_223_372_036L, OpenAICompatibleProvider.getRequestTimeoutSeconds(config));
+ assertEquals(
+ Duration.ofSeconds(9_223_372_036L),
+ OpenAICompatibleProvider.newHttpClient(config).connectTimeout().orElseThrow()
+ );
+ }
+
@Test
void shouldRejectInvalidTemperatureOnStartup() {
final Properties config = validConfig();
@@ -170,7 +208,7 @@ class OpenAICompatibleProviderTest {
}
@Test
- void shouldClassifyRequestTimeout() throws Exception {
+ void shouldTimeoutWhileWaitingForResponseHeaders() throws Exception {
startServer(exchange -> {
try {
Thread.sleep(1_500L);
@@ -184,12 +222,57 @@ class OpenAICompatibleProviderTest {
config.setProperty("max-retries", "0");
final OpenAICompatibleProvider provider = new OpenAICompatibleProvider(config);
+ final long started = System.nanoTime();
final JudgeModelException exception = assertThrows(
JudgeModelException.class,
() -> provider.judge(request())
);
+ final long elapsedMillis = (System.nanoTime() - started) / 1_000_000L;
assertEquals(JudgeModelException.Reason.TIMEOUT, exception.getReason());
+ assertTrue(
+ elapsedMillis >= 700L && elapsedMillis < 2_500L,
+ "delayed headers should fail near the 1s budget, took " + elapsedMillis + "ms"
+ );
+ }
+
+ @Test
+ void shouldTimeoutWhileReadingStalledResponseBody() throws Exception {
+ final CountDownLatch release = new CountDownLatch(1);
+ startServer(exchange -> {
+ // Advertise more bytes than are sent, then stall the body.
+ exchange.sendResponseHeaders(200, 64);
+ exchange.getResponseBody().write("ping".getBytes(StandardCharsets.UTF_8));
+ exchange.getResponseBody().flush();
+ try {
+ release.await(10, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ exchange.close();
+ });
+ final Properties config = serverConfig();
+ config.setProperty("request-timeout-seconds", "1");
+ config.setProperty("max-retries", "0");
+ final OpenAICompatibleProvider provider = new OpenAICompatibleProvider(config);
+
+ try {
+ final long started = System.nanoTime();
+ final JudgeModelException exception = assertThrows(
+ JudgeModelException.class,
+ () -> provider.judge(request())
+ );
+ final long elapsedMillis = (System.nanoTime() - started) / 1_000_000L;
+
+ assertEquals(JudgeModelException.Reason.TIMEOUT, exception.getReason());
+ assertTrue(
+ elapsedMillis >= 700L && elapsedMillis < 2_500L,
+ "stalled body should fail near the 1s budget, took " + elapsedMillis + "ms"
+ );
+ } finally {
+ // The handler runs on the server's dispatcher thread, which HttpServer.stop in tearDown joins.
+ release.countDown();
+ }
}
@Test
diff --git a/oap-server/server-starter/src/main/resources/ai-evaluation.yml b/oap-server/server-starter/src/main/resources/ai-evaluation.yml
index fb89c8499c..73173286f4 100644
--- a/oap-server/server-starter/src/main/resources/ai-evaluation.yml
+++ b/oap-server/server-starter/src/main/resources/ai-evaluation.yml
@@ -21,6 +21,7 @@ judge:
endpoint: ${AI_EVALUATION_ENDPOINT:}
model: ${AI_EVALUATION_MODEL:gpt-5.5}
api-key: ${AI_EVALUATION_API_KEY:}
+ # Deadline (seconds) of each judge HTTP request, covering connect, response wait and body read; each retry gets its own.
request-timeout-seconds: 30
max-retries: 2
temperature: 0.2