You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by di...@apache.org on 2026/04/25 06:33:43 UTC
(flink) branch master updated: [FLINK-38857][models] Add health check and circuit breaker for Triton inference (#27567)
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new c4b626d3f6c [FLINK-38857][models] Add health check and circuit breaker for Triton inference (#27567)
c4b626d3f6c is described below
commit c4b626d3f6ce3e489daed2f9f418b0beca2a16b6
Author: Feat Zhang <fe...@apache.org>
AuthorDate: Sat Apr 25 14:33:33 2026 +0800
[FLINK-38857][models] Add health check and circuit breaker for Triton inference (#27567)
---
.../generated/model_triton_advanced_section.html | 36 +
.../shortcodes/generated/triton_configuration.html | 36 +
.../model/triton/AbstractTritonModelFunction.java | 242 +++++-
.../flink/model/triton/TritonCircuitBreaker.java | 739 ++++++++++++++++++
.../flink/model/triton/TritonHealthChecker.java | 490 ++++++++++++
.../model/triton/TritonInferenceModelFunction.java | 91 ++-
.../apache/flink/model/triton/TritonOptions.java | 72 ++
.../org/apache/flink/model/triton/TritonUtils.java | 41 +
.../TritonCircuitBreakerOpenException.java | 63 ++
.../model/triton/TritonCircuitBreakerTest.java | 827 +++++++++++++++++++++
.../model/triton/TritonHealthCheckerTest.java | 463 ++++++++++++
11 files changed, 3081 insertions(+), 19 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/model_triton_advanced_section.html b/docs/layouts/shortcodes/generated/model_triton_advanced_section.html
index 20347e0519f..794b20941ff 100644
--- a/docs/layouts/shortcodes/generated/model_triton_advanced_section.html
+++ b/docs/layouts/shortcodes/generated/model_triton_advanced_section.html
@@ -14,6 +14,30 @@
<td>String</td>
<td>Authentication token for secured Triton servers.</td>
</tr>
+ <tr>
+ <td><h5>circuit-breaker-enabled</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Whether to enable circuit breaker protection. When enabled, the circuit breaker will automatically fail fast when the server is unhealthy, preventing cascading failures and reducing load on the failing server. The circuit breaker implements a three-state model: CLOSED (normal), OPEN (failing fast), and HALF_OPEN (testing recovery). Defaults to false.</td>
+ </tr>
+ <tr>
+ <td><h5>circuit-breaker-failure-threshold</h5></td>
+ <td style="word-wrap: break-word;">0.5</td>
+ <td>Double</td>
+ <td>Failure rate threshold (0.0-1.0) that triggers the circuit breaker to open. For example, 0.5 means the circuit will open when 50% of recent requests fail. Requires a minimum of 10 requests before evaluation. Defaults to 0.5 (50%). Only effective when circuit-breaker-enabled is true.</td>
+ </tr>
+ <tr>
+ <td><h5>circuit-breaker-half-open-requests</h5></td>
+ <td style="word-wrap: break-word;">3</td>
+ <td>Integer</td>
+ <td>Number of successful test requests required in HALF_OPEN state to close the circuit. If any request fails in HALF_OPEN state, the circuit immediately reopens. Defaults to 3 requests. Only effective when circuit-breaker-enabled is true.</td>
+ </tr>
+ <tr>
+ <td><h5>circuit-breaker-timeout</h5></td>
+ <td style="word-wrap: break-word;">1 min</td>
+ <td>Duration</td>
+ <td>Duration to keep the circuit breaker in OPEN state before transitioning to HALF_OPEN. In HALF_OPEN state, limited requests are allowed to probe if the server has recovered. Defaults to 60 seconds. Only effective when circuit-breaker-enabled is true.</td>
+ </tr>
<tr>
<td><h5>compression</h5></td>
<td style="word-wrap: break-word;">(none)</td>
@@ -32,6 +56,18 @@
<td>Boolean</td>
<td>Whether to flatten the batch dimension for array inputs. When true, shape [1,N] becomes [N]. Defaults to false.</td>
</tr>
+ <tr>
+ <td><h5>health-check-enabled</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Whether to enable periodic health checks for the Triton server. When enabled, the health checker will periodically call <code class="highlighter-rouge">/v2/health/live</code> endpoint to verify server availability. Defaults to false.</td>
+ </tr>
+ <tr>
+ <td><h5>health-check-interval</h5></td>
+ <td style="word-wrap: break-word;">30 s</td>
+ <td>Duration</td>
+ <td>Interval between health check requests. Shorter intervals provide faster failure detection but increase server load. Defaults to 30 seconds. Only effective when health-check-enabled is true.</td>
+ </tr>
<tr>
<td><h5>priority</h5></td>
<td style="word-wrap: break-word;">(none)</td>
diff --git a/docs/layouts/shortcodes/generated/triton_configuration.html b/docs/layouts/shortcodes/generated/triton_configuration.html
index 692a5821f24..2ebfac39733 100644
--- a/docs/layouts/shortcodes/generated/triton_configuration.html
+++ b/docs/layouts/shortcodes/generated/triton_configuration.html
@@ -14,6 +14,30 @@
<td>String</td>
<td>Authentication token for secured Triton servers.</td>
</tr>
+ <tr>
+ <td><h5>circuit-breaker-enabled</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Whether to enable circuit breaker protection. When enabled, the circuit breaker will automatically fail fast when the server is unhealthy, preventing cascading failures and reducing load on the failing server. The circuit breaker implements a three-state model: CLOSED (normal), OPEN (failing fast), and HALF_OPEN (testing recovery). Defaults to false.</td>
+ </tr>
+ <tr>
+ <td><h5>circuit-breaker-failure-threshold</h5></td>
+ <td style="word-wrap: break-word;">0.5</td>
+ <td>Double</td>
+ <td>Failure rate threshold (0.0-1.0) that triggers the circuit breaker to open. For example, 0.5 means the circuit will open when 50% of recent requests fail. Requires a minimum of 10 requests before evaluation. Defaults to 0.5 (50%). Only effective when circuit-breaker-enabled is true.</td>
+ </tr>
+ <tr>
+ <td><h5>circuit-breaker-half-open-requests</h5></td>
+ <td style="word-wrap: break-word;">3</td>
+ <td>Integer</td>
+ <td>Number of successful test requests required in HALF_OPEN state to close the circuit. If any request fails in HALF_OPEN state, the circuit immediately reopens. Defaults to 3 requests. Only effective when circuit-breaker-enabled is true.</td>
+ </tr>
+ <tr>
+ <td><h5>circuit-breaker-timeout</h5></td>
+ <td style="word-wrap: break-word;">1 min</td>
+ <td>Duration</td>
+ <td>Duration to keep the circuit breaker in OPEN state before transitioning to HALF_OPEN. In HALF_OPEN state, limited requests are allowed to probe if the server has recovered. Defaults to 60 seconds. Only effective when circuit-breaker-enabled is true.</td>
+ </tr>
<tr>
<td><h5>compression</h5></td>
<td style="word-wrap: break-word;">(none)</td>
@@ -38,6 +62,18 @@
<td>Boolean</td>
<td>Whether to flatten the batch dimension for array inputs. When true, shape [1,N] becomes [N]. Defaults to false.</td>
</tr>
+ <tr>
+ <td><h5>health-check-enabled</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Whether to enable periodic health checks for the Triton server. When enabled, the health checker will periodically call <code class="highlighter-rouge">/v2/health/live</code> endpoint to verify server availability. Defaults to false.</td>
+ </tr>
+ <tr>
+ <td><h5>health-check-interval</h5></td>
+ <td style="word-wrap: break-word;">30 s</td>
+ <td>Duration</td>
+ <td>Interval between health check requests. Shorter intervals provide faster failure detection but increase server load. Defaults to 30 seconds. Only effective when health-check-enabled is true.</td>
+ </tr>
<tr>
<td><h5>model-name</h5></td>
<td style="word-wrap: break-word;">(none)</td>
diff --git a/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/AbstractTritonModelFunction.java b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/AbstractTritonModelFunction.java
index 22965b9ddaa..916ffeda928 100644
--- a/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/AbstractTritonModelFunction.java
+++ b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/AbstractTritonModelFunction.java
@@ -63,8 +63,21 @@ public abstract class AbstractTritonModelFunction extends AsyncPredictFunction {
private static final Logger LOG = LoggerFactory.getLogger(AbstractTritonModelFunction.class);
protected transient OkHttpClient httpClient;
+ protected transient TritonCircuitBreaker circuitBreaker;
+ protected transient TritonHealthChecker healthChecker;
private final String endpoint;
+
+ /**
+ * Cached sanitized view of {@link #endpoint} used exclusively for log and error-message
+ * formatting. The raw {@link #endpoint} (which may carry {@code user:password@host}
+ * credentials) is still required for HTTP calls and for constructing request URLs, but must
+ * never be echoed to logs, metrics or exception messages, as those can end up in CI artifacts
+ * or user-facing dashboards. Computed once in the constructor so we don't pay the URI parsing
+ * cost on every log line.
+ */
+ private final String loggedEndpoint;
+
private final String modelName;
private final String modelVersion;
private final Duration timeout;
@@ -86,9 +99,18 @@ public abstract class AbstractTritonModelFunction extends AsyncPredictFunction {
private final String authToken;
private final Map<String, String> customHeaders;
+ // Health check and circuit breaker configuration
+ private final boolean healthCheckEnabled;
+ private final Duration healthCheckInterval;
+ private final boolean circuitBreakerEnabled;
+ private final double circuitBreakerFailureThreshold;
+ private final Duration circuitBreakerTimeout;
+ private final int circuitBreakerHalfOpenRequests;
+
public AbstractTritonModelFunction(
ModelProviderFactory.Context factoryContext, ReadableConfig config) {
this.endpoint = config.get(TritonOptions.ENDPOINT);
+ this.loggedEndpoint = TritonUtils.sanitizeUrl(this.endpoint);
this.modelName = config.get(TritonOptions.MODEL_NAME);
this.modelVersion = config.get(TritonOptions.MODEL_VERSION);
this.timeout = config.get(TritonOptions.TIMEOUT);
@@ -101,6 +123,16 @@ public abstract class AbstractTritonModelFunction extends AsyncPredictFunction {
this.authToken = config.get(TritonOptions.AUTH_TOKEN);
this.customHeaders = config.get(TritonOptions.CUSTOM_HEADERS);
+ // Health check and circuit breaker configuration
+ this.healthCheckEnabled = config.get(TritonOptions.HEALTH_CHECK_ENABLED);
+ this.healthCheckInterval = config.get(TritonOptions.HEALTH_CHECK_INTERVAL);
+ this.circuitBreakerEnabled = config.get(TritonOptions.CIRCUIT_BREAKER_ENABLED);
+ this.circuitBreakerFailureThreshold =
+ config.get(TritonOptions.CIRCUIT_BREAKER_FAILURE_THRESHOLD);
+ this.circuitBreakerTimeout = config.get(TritonOptions.CIRCUIT_BREAKER_TIMEOUT);
+ this.circuitBreakerHalfOpenRequests =
+ config.get(TritonOptions.CIRCUIT_BREAKER_HALF_OPEN_REQUESTS);
+
// Validate input schema - support multiple types
validateInputSchema(factoryContext.getCatalogModel().getResolvedInputSchema());
}
@@ -110,15 +142,115 @@ public abstract class AbstractTritonModelFunction extends AsyncPredictFunction {
super.open(context);
LOG.debug("Creating Triton HTTP client.");
this.httpClient = TritonUtils.createHttpClient(timeout.toMillis());
+
+ // Initialize circuit breaker if enabled
+ if (circuitBreakerEnabled) {
+ LOG.info(
+ "Initializing circuit breaker for endpoint {} with threshold={}, timeout={}, halfOpenRequests={}",
+ loggedEndpoint,
+ circuitBreakerFailureThreshold,
+ circuitBreakerTimeout,
+ circuitBreakerHalfOpenRequests);
+
+ this.circuitBreaker =
+ new TritonCircuitBreaker(
+ endpoint,
+ circuitBreakerFailureThreshold,
+ circuitBreakerTimeout,
+ circuitBreakerHalfOpenRequests);
+ }
+
+ // Initialize health checker if enabled
+ if (healthCheckEnabled) {
+ if (circuitBreaker == null) {
+ LOG.warn(
+ "Health check is enabled but circuit breaker is disabled for endpoint {}. "
+ + "Health check will run but failures will not trigger circuit breaking. "
+ + "Consider enabling circuit-breaker-enabled for better fault tolerance.",
+ loggedEndpoint);
+ }
+
+ LOG.info(
+ "Initializing health checker for endpoint {} with interval {}",
+ loggedEndpoint,
+ healthCheckInterval);
+
+ // Pass the (possibly null) circuit breaker directly. The health checker treats a null
+ // breaker as "log-only" mode rather than relying on a dummy instance whose state
+ // nobody reads.
+ this.healthChecker =
+ new TritonHealthChecker(
+ endpoint, httpClient, circuitBreaker, healthCheckInterval);
+
+ // Perform an immediate health check
+ boolean initialHealth = healthChecker.checkNow();
+ if (initialHealth) {
+ LOG.info("Initial health check passed for endpoint {}", loggedEndpoint);
+ } else {
+ LOG.warn(
+ "Initial health check failed for endpoint {}. "
+ + "Inference requests may fail until server becomes healthy.",
+ loggedEndpoint);
+ }
+
+ // Start periodic health checking. The scheduler uses an initial delay equal to
+ // checkInterval so it does not duplicate the eager checkNow() above.
+ healthChecker.start();
+ }
}
@Override
public void close() throws Exception {
- super.close();
+ // Each cleanup step is isolated so that a failure in one does not prevent the others
+ // from running. The HTTP client in particular must always be released back to the
+ // reference-counted pool; leaking its reference across restarts would keep the shared
+ // client alive forever.
+ Exception firstFailure = null;
+
+ try {
+ super.close();
+ } catch (Exception e) {
+ firstFailure = e;
+ }
+
+ // Stop health checker first so it cannot race with the HTTP client teardown below.
+ if (this.healthChecker != null) {
+ LOG.debug("Stopping health checker for {}", loggedEndpoint);
+ try {
+ this.healthChecker.close();
+ } catch (Exception e) {
+ LOG.warn("Error closing health checker for " + loggedEndpoint, e);
+ if (firstFailure == null) {
+ firstFailure = e;
+ } else {
+ firstFailure.addSuppressed(e);
+ }
+ }
+ this.healthChecker = null;
+ }
+
+ // Release circuit breaker (no-op, just drop the reference).
+ this.circuitBreaker = null;
+
+ // Release HTTP client last so it's always attempted even if earlier steps threw.
if (this.httpClient != null) {
LOG.debug("Releasing Triton HTTP client.");
- TritonUtils.releaseHttpClient(this.httpClient);
- httpClient = null;
+ try {
+ TritonUtils.releaseHttpClient(this.httpClient);
+ } catch (Exception e) {
+ LOG.warn("Error releasing Triton HTTP client for " + loggedEndpoint, e);
+ if (firstFailure == null) {
+ firstFailure = e;
+ } else {
+ firstFailure.addSuppressed(e);
+ }
+ } finally {
+ this.httpClient = null;
+ }
+ }
+
+ if (firstFailure != null) {
+ throw firstFailure;
}
}
@@ -167,13 +299,23 @@ public abstract class AbstractTritonModelFunction extends AsyncPredictFunction {
column.getName(),
column.getClass());
- Preconditions.checkArgument(
- expectedType != null && !expectedType.equals(column.getDataType().getLogicalType()),
- "%s column %s should be %s, but is a %s.",
- inputOrOutput,
- column.getName(),
- expectedType,
- column.getDataType().getLogicalType());
+ // The previous condition ({@code expectedType != null && !expectedType.equals(actual)})
+ // inverted the {@link Preconditions#checkArgument} contract: it threw whenever
+ // {@code expectedType} was null (the documented "skip type check" mode) and also whenever
+ // the actual type matched the expected one. All current call sites pass {@code
+ // expectedType = null}, so this bug would have rejected every schema had the code path
+ // been exercised. The corrected form only enforces the equality check when a specific
+ // expected type is supplied.
+ if (expectedType != null) {
+ LogicalType actualType = column.getDataType().getLogicalType();
+ Preconditions.checkArgument(
+ expectedType.equals(actualType),
+ "%s column %s should be %s, but is %s.",
+ inputOrOutput,
+ column.getName(),
+ expectedType,
+ actualType);
+ }
// Validate that the type is supported by Triton
try {
@@ -319,4 +461,84 @@ public abstract class AbstractTritonModelFunction extends AsyncPredictFunction {
protected Map<String, String> getCustomHeaders() {
return customHeaders;
}
+
+ /**
+ * Checks if a request is allowed through the circuit breaker.
+ *
+ * <p>Subclasses should call this method before making inference requests. If the circuit
+ * breaker is OPEN, this method will throw an exception to fail fast.
+ *
+ * <p>The nullness of {@link #circuitBreaker} is the single source of truth: it is set in {@link
+ * #open(FunctionContext)} only when circuit breaking is enabled, so an explicit {@code
+ * circuitBreakerEnabled} check here would be redundant (and could drift out of sync with the
+ * field).
+ *
+ * @throws org.apache.flink.model.triton.exception.TritonCircuitBreakerOpenException if circuit
+ * is OPEN
+ */
+ protected void checkCircuitBreaker() {
+ if (circuitBreaker != null) {
+ circuitBreaker.isRequestAllowed();
+ }
+ }
+
+ /**
+ * Records a successful inference request with the circuit breaker.
+ *
+ * <p>Subclasses should call this method after successful inference requests to update circuit
+ * breaker metrics.
+ */
+ protected void recordSuccess() {
+ if (circuitBreaker != null) {
+ circuitBreaker.recordSuccess();
+ }
+ }
+
+ /**
+ * Records a failed inference request with the circuit breaker.
+ *
+ * <p>Subclasses should call this method after failed inference requests to update circuit
+ * breaker metrics.
+ */
+ protected void recordFailure() {
+ if (circuitBreaker != null) {
+ circuitBreaker.recordFailure();
+ }
+ }
+
+ /**
+ * Gets the current circuit breaker instance.
+ *
+ * @return the circuit breaker, or null if not enabled
+ */
+ protected TritonCircuitBreaker getCircuitBreaker() {
+ return circuitBreaker;
+ }
+
+ /**
+ * Gets the current health checker instance.
+ *
+ * @return the health checker, or null if not enabled
+ */
+ protected TritonHealthChecker getHealthChecker() {
+ return healthChecker;
+ }
+
+ /**
+ * Checks if health checking is enabled.
+ *
+ * @return true if health checking is enabled
+ */
+ protected boolean isHealthCheckEnabled() {
+ return healthCheckEnabled;
+ }
+
+ /**
+ * Checks if circuit breaker is enabled.
+ *
+ * @return true if circuit breaker is enabled
+ */
+ protected boolean isCircuitBreakerEnabled() {
+ return circuitBreakerEnabled;
+ }
}
diff --git a/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonCircuitBreaker.java b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonCircuitBreaker.java
new file mode 100644
index 00000000000..9dc75dc61d6
--- /dev/null
+++ b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonCircuitBreaker.java
@@ -0,0 +1,739 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.model.triton;
+
+import org.apache.flink.model.triton.exception.TritonCircuitBreakerOpenException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Circuit breaker implementation for Triton Inference Server health management.
+ *
+ * <p>This circuit breaker follows the classic three-state model to protect the system from
+ * cascading failures when the Triton server becomes unhealthy:
+ *
+ * <ul>
+ * <li><b>CLOSED</b>: Normal operation. Requests are allowed. Tracks failure rate.
+ * <li><b>OPEN</b>: Triton is unhealthy. All requests fail fast without hitting the server.
+ * <li><b>HALF_OPEN</b>: Testing recovery. Limited requests allowed to probe server health.
+ * </ul>
+ *
+ * <p><b>State Transitions:</b>
+ *
+ * <pre>
+ * CLOSED ──[failure rate > threshold]──> OPEN
+ * ↑ │
+ * │ │ [after timeout]
+ * │ ↓
+ * └──[success count met]── HALF_OPEN
+ * </pre>
+ *
+ * <p><b>Benefits:</b>
+ *
+ * <ul>
+ * <li>Fail fast when server is down, avoiding wasted retries
+ * <li>Automatic recovery detection
+ * <li>Reduced load on failing servers (prevents cascading failure)
+ * <li>Improved system resilience
+ * </ul>
+ *
+ * <p><b>Thread Safety:</b> This class is thread-safe and designed for concurrent access from
+ * multiple Flink task threads.
+ *
+ * @see TritonHealthChecker
+ */
+public class TritonCircuitBreaker {
+ private static final Logger LOG = LoggerFactory.getLogger(TritonCircuitBreaker.class);
+
+ /** Current state of the circuit breaker. */
+ public enum State {
+ /** Normal operation, requests allowed, tracking failures. */
+ CLOSED,
+ /** Server unhealthy, failing fast without hitting server. */
+ OPEN,
+ /** Testing recovery with limited requests. */
+ HALF_OPEN
+ }
+
+ private final String endpoint;
+
+ /**
+ * Cached sanitized view of {@link #endpoint} used for all log lines and user-visible exception
+ * messages. The raw endpoint may contain basic-auth credentials (e.g. {@code
+ * http://user:password@host:8000}); {@link TritonUtils#sanitizeUrl(String)} strips the userInfo
+ * component so those credentials cannot leak into INFO/WARN logs or into the message of {@link
+ * TritonCircuitBreakerOpenException} propagated to user pipelines.
+ */
+ private final String loggedEndpoint;
+
+ private final double failureThreshold;
+ private final Duration openStateDuration;
+ private final int halfOpenMaxRequests;
+
+ private final AtomicReference<State> state = new AtomicReference<>(State.CLOSED);
+
+ /**
+ * Timestamp of the last state transition, captured from {@link System#nanoTime()}.
+ *
+ * <p>We deliberately use {@code nanoTime} (a monotonic clock) rather than {@code
+ * currentTimeMillis} (wall-clock) so that the OPEN -> HALF_OPEN timeout is not affected by NTP
+ * adjustments or manual clock changes. A backwards wall-clock jump would otherwise make the
+ * breaker believe its timeout has regressed (leaving it OPEN indefinitely); a forward jump
+ * would spuriously trigger an early probe.
+ *
+ * <p>The value is only meaningful when compared against another {@code nanoTime()} reading
+ * taken on the same JVM process; it is never serialised or exposed to external clients.
+ */
+ private final AtomicLong lastStateTransitionNanos = new AtomicLong(System.nanoTime());
+
+ // Metrics for CLOSED state
+ private final AtomicInteger totalRequests = new AtomicInteger(0);
+ private final AtomicInteger failedRequests = new AtomicInteger(0);
+
+ // Metrics for HALF_OPEN state
+ private final AtomicInteger halfOpenSuccesses = new AtomicInteger(0);
+ private final AtomicInteger halfOpenFailures = new AtomicInteger(0);
+ private final AtomicInteger halfOpenRequests = new AtomicInteger(0);
+
+ /**
+ * Minimum number of requests before evaluating failure rate. This prevents opening the circuit
+ * based on too few samples.
+ */
+ private static final int MIN_REQUESTS_THRESHOLD = 10;
+
+ /**
+ * Maximum number of requests to track in CLOSED state before decaying the counters. Decay
+ * (halving the counters - see {@link #incrementClosedMetrics(boolean)}) prevents historical
+ * successes from diluting the current failure rate while still preserving the current rate at
+ * the decay boundary.
+ */
+ private static final int MAX_CLOSED_STATE_REQUESTS = 10000;
+
+ /**
+ * Bounded number of retries when re-evaluating circuit breaker state after losing a CAS race.
+ * In practice one or two iterations are always sufficient; this bound merely guards against
+ * pathological contention.
+ */
+ private static final int MAX_STATE_EVAL_ATTEMPTS = 4;
+
+ /**
+ * Lock used to make CLOSED-state metric resets atomic with respect to readers.
+ *
+ * <p>Using a dedicated lock (rather than synchronizing on {@code this}) avoids accidentally
+ * blocking unrelated operations.
+ */
+ private final Object closedMetricsLock = new Object();
+
+ /**
+ * Lock that serialises the OPEN -> HALF_OPEN transition so that {@link #resetHalfOpenMetrics()}
+ * cannot race with concurrent probe increments done by {@link #allowHalfOpenRequest()}.
+ *
+ * <p>Without this lock, two threads both observing OPEN would each call {@code
+ * resetHalfOpenMetrics()} and then CAS; the second reset could wipe probe counts that another
+ * thread had already recorded after the first CAS succeeded.
+ */
+ private final Object halfOpenTransitionLock = new Object();
+
+ /**
+ * Creates a new circuit breaker for a Triton endpoint.
+ *
+ * @param endpoint The Triton server endpoint URL
+ * @param failureThreshold Failure rate (0.0-1.0) that triggers circuit opening
+ * @param openStateDuration How long to stay OPEN before transitioning to HALF_OPEN
+ * @param halfOpenMaxRequests Number of successful test requests needed in HALF_OPEN to close
+ */
+ public TritonCircuitBreaker(
+ String endpoint,
+ double failureThreshold,
+ Duration openStateDuration,
+ int halfOpenMaxRequests) {
+ this.endpoint = endpoint;
+ this.loggedEndpoint = TritonUtils.sanitizeUrl(endpoint);
+ if (failureThreshold <= 0.0 || failureThreshold > 1.0) {
+ throw new IllegalArgumentException(
+ "failureThreshold must be in range (0.0, 1.0], got: " + failureThreshold);
+ }
+ this.failureThreshold = failureThreshold;
+ // openStateDuration governs the mandatory OPEN -> HALF_OPEN cool-off. A null or
+ // non-positive value would either NPE at every state check (null) or make the breaker
+ // immediately re-admit traffic while the server is still being considered down
+ // (non-positive), which defeats the purpose of the OPEN state. Reject both eagerly so
+ // misconfiguration fails fast at operator construction time, not at first failure.
+ if (openStateDuration == null) {
+ throw new IllegalArgumentException("openStateDuration must not be null");
+ }
+ if (openStateDuration.isZero() || openStateDuration.isNegative()) {
+ throw new IllegalArgumentException(
+ "openStateDuration must be strictly positive, got: " + openStateDuration);
+ }
+ this.openStateDuration = openStateDuration;
+ if (halfOpenMaxRequests <= 0) {
+ throw new IllegalArgumentException(
+ "halfOpenMaxRequests must be positive, got: " + halfOpenMaxRequests);
+ }
+ this.halfOpenMaxRequests = halfOpenMaxRequests;
+
+ LOG.info(
+ "Circuit breaker created for endpoint {} with threshold={}, openDuration={}, halfOpenRequests={}",
+ loggedEndpoint,
+ failureThreshold,
+ openStateDuration,
+ halfOpenMaxRequests);
+ }
+
+ /**
+ * Checks if a request is allowed through the circuit breaker.
+ *
+ * <p>This method either allows the request to proceed (returning {@code true}) or fails fast by
+ * throwing {@link TritonCircuitBreakerOpenException}. It never returns {@code false}; the
+ * boolean return type is preserved for API stability and to read naturally at call sites.
+ *
+ * @return {@code true} when the request is permitted
+ * @throws TritonCircuitBreakerOpenException if circuit is OPEN, or if HALF_OPEN has already
+ * dispatched its maximum probe requests
+ */
+ public boolean isRequestAllowed() throws TritonCircuitBreakerOpenException {
+ // Use a bounded retry loop so that if the state transitions during our evaluation
+ // (e.g. OPEN -> HALF_OPEN CAS loses to another thread), we re-evaluate against the
+ // new state instead of incorrectly rejecting a request that the new state would allow.
+ for (int attempt = 0; attempt < MAX_STATE_EVAL_ATTEMPTS; attempt++) {
+ State currentState = state.get();
+
+ switch (currentState) {
+ case CLOSED:
+ return true;
+
+ case OPEN:
+ // Check if it's time to transition to HALF_OPEN
+ if (shouldTransitionToHalfOpen()) {
+ // Serialise the reset + CAS under a dedicated lock. This prevents a
+ // second thread from resetting the HALF_OPEN counters after another
+ // thread has already transitioned and started granting probes.
+ synchronized (halfOpenTransitionLock) {
+ // Re-check the state under the lock: another thread may have
+ // already completed the transition while we were blocked.
+ if (state.get() == State.OPEN) {
+ resetHalfOpenMetrics();
+ if (state.compareAndSet(State.OPEN, State.HALF_OPEN)) {
+ lastStateTransitionNanos.set(System.nanoTime());
+ LOG.info(
+ "Circuit breaker transitioning from OPEN to HALF_OPEN for {}",
+ loggedEndpoint);
+ }
+ }
+ }
+ // Fall through the outer loop so this request competes for a probe
+ // slot via allowHalfOpenRequest(), ensuring the budget is respected
+ // regardless of which thread performed the actual transition.
+ continue;
+ }
+ throw new TritonCircuitBreakerOpenException(
+ String.format(
+ "Circuit breaker is OPEN for endpoint %s. "
+ + "Server is considered unhealthy. Will retry in %d seconds.",
+ loggedEndpoint, getRemainingOpenTimeSeconds()));
+
+ case HALF_OPEN:
+ // Allow limited number of requests in HALF_OPEN state using CAS loop.
+ // The inner loop can only exit via {@code return} or {@code throw}, so no
+ // fall-through to {@code default} is possible.
+ return allowHalfOpenRequest();
+
+ default:
+ // Unreachable in practice: all enum values are handled above.
+ throw new IllegalStateException(
+ "Unexpected circuit breaker state: " + currentState);
+ }
+ }
+
+ // Safety net: if the state kept flipping under heavy contention, fail fast with
+ // the OPEN message rather than looping indefinitely.
+ throw new TritonCircuitBreakerOpenException(
+ String.format(
+ "Circuit breaker state evaluation exceeded %d attempts for endpoint %s.",
+ MAX_STATE_EVAL_ATTEMPTS, loggedEndpoint));
+ }
+
+ /**
+ * Acquires a slot for a probe request while in HALF_OPEN state.
+ *
+ * <p>Extracted to a helper so that the caller's {@code switch} can simply {@code return} the
+ * result, avoiding any appearance of fall-through to the {@code default} branch.
+ */
+ private boolean allowHalfOpenRequest() throws TritonCircuitBreakerOpenException {
+ while (true) {
+ int current = halfOpenRequests.get();
+ if (current >= halfOpenMaxRequests) {
+ throw new TritonCircuitBreakerOpenException(
+ String.format(
+ "Circuit breaker is HALF_OPEN for endpoint %s. "
+ + "Maximum test requests (%d) reached. Please retry later.",
+ loggedEndpoint, halfOpenMaxRequests));
+ }
+ if (halfOpenRequests.compareAndSet(current, current + 1)) {
+ LOG.debug(
+ "Allowing request {}/{} in HALF_OPEN state for {}",
+ current + 1,
+ halfOpenMaxRequests,
+ loggedEndpoint);
+ return true;
+ }
+ }
+ }
+
+ /**
+ * Records a successful request.
+ *
+ * <p>In CLOSED state, this updates success metrics. In HALF_OPEN state, this may trigger
+ * transition back to CLOSED if enough successful probes complete.
+ */
+ public void recordSuccess() {
+ State currentState = state.get();
+
+ switch (currentState) {
+ case CLOSED:
+ {
+ incrementClosedMetrics(false);
+
+ // A pure success can only lower the failure rate. However, when the very
+ // first sample that pushes total over MIN_REQUESTS_THRESHOLD happens to be
+ // a success (e.g. 9 prior failures followed by one success => total=10,
+ // failed=9, rate 0.9), this is the earliest point at which
+ // shouldOpenCircuit() can legitimately fire. We therefore still evaluate it
+ // here; the cost is a single atomic read per success.
+ //
+ // Snapshot ordering: we read the counters AFTER incrementClosedMetrics()
+ // returns (so this sample is folded in) and BEFORE the CLOSED -> OPEN CAS.
+ // Two consequences matter for readers of this code:
+ // 1. The snapshot reflects the state that the breaker is deciding on,
+ // including the current call's contribution. snapshotClosedMetrics()
+ // reads both counters under closedMetricsLock so the pair is
+ // internally consistent even if another thread is concurrently
+ // incrementing or halving the counters.
+ // 2. Logging and the shouldOpenCircuit() decision use the SAME snapshot,
+ // so the "X/Y failure rate" line always matches the values the
+ // decision was actually made on - even though a concurrent thread may
+ // have advanced the counters by the time the log line is formatted.
+ // This is intentional: a readable, reproducible log is more useful
+ // than a re-read that would race with other writers anyway.
+ Snapshot triggeringSnap = snapshotClosedMetrics();
+ if (shouldOpenCircuit(triggeringSnap)) {
+ if (state.compareAndSet(State.CLOSED, State.OPEN)) {
+ LOG.warn(
+ "Circuit breaker opening for {} due to high failure rate: {}/{}",
+ loggedEndpoint,
+ triggeringSnap.failed,
+ triggeringSnap.total);
+ lastStateTransitionNanos.set(System.nanoTime());
+ }
+ }
+ break;
+ }
+
+ case HALF_OPEN:
+ {
+ int successes = halfOpenSuccesses.incrementAndGet();
+ LOG.debug(
+ "Circuit breaker recorded success {}/{} in HALF_OPEN for {}",
+ successes,
+ halfOpenMaxRequests,
+ loggedEndpoint);
+
+ if (successes >= halfOpenMaxRequests) {
+ // Enough successful probes, close the circuit
+ if (state.compareAndSet(State.HALF_OPEN, State.CLOSED)) {
+ LOG.info(
+ "Circuit breaker transitioning from HALF_OPEN to CLOSED for {} "
+ + "after {} successful probes",
+ loggedEndpoint,
+ successes);
+ lastStateTransitionNanos.set(System.nanoTime());
+ resetClosedMetrics();
+ } else {
+ // CAS failed: state changed under us (e.g. recorded failure reopened
+ // the circuit). Log so the probe isn't silently lost from metrics view.
+ LOG.debug(
+ "HALF_OPEN -> CLOSED transition lost race for {}; state is now {}",
+ loggedEndpoint,
+ state.get());
+ }
+ }
+ break;
+ }
+
+ case OPEN:
+ // Shouldn't happen, but log if it does
+ LOG.warn("Recorded success while circuit breaker is OPEN for {}", loggedEndpoint);
+ break;
+ }
+ }
+
+ /**
+ * Records a failed request.
+ *
+ * <p>In CLOSED state, this may trigger transition to OPEN if failure rate exceeds threshold. In
+ * HALF_OPEN state, any failure immediately reopens the circuit.
+ */
+ public void recordFailure() {
+ State currentState = state.get();
+
+ switch (currentState) {
+ case CLOSED:
+ {
+ incrementClosedMetrics(true);
+
+ // Snapshot ordering: taken AFTER the increment (so this failure is included
+ // in the denominator / numerator) and BEFORE the CLOSED -> OPEN CAS. See the
+ // matching comment in recordSuccess() for the full reasoning. Short version:
+ // the snapshot read is guarded by the same lock as the increment, so the
+ // {total, failed} pair is internally consistent, and the log line below
+ // reports the exact numbers that drove the decision.
+ Snapshot triggeringSnap = snapshotClosedMetrics();
+ // Check if we should open the circuit
+ if (shouldOpenCircuit(triggeringSnap)) {
+ if (state.compareAndSet(State.CLOSED, State.OPEN)) {
+ LOG.warn(
+ "Circuit breaker opening for {} due to high failure rate: {}/{}",
+ loggedEndpoint,
+ triggeringSnap.failed,
+ triggeringSnap.total);
+ lastStateTransitionNanos.set(System.nanoTime());
+ }
+ }
+ break;
+ }
+
+ case HALF_OPEN:
+ // Any failure in HALF_OPEN immediately reopens the circuit. Only bump the
+ // failure counter when the CAS succeeds: if the CAS loses, the state has
+ // already transitioned (typically another thread already moved HALF_OPEN ->
+ // OPEN), and incrementing halfOpenFailures here would otherwise pollute the
+ // next HALF_OPEN cycle's monitoring view between its OPEN->HALF_OPEN reset
+ // and the first real probe.
+ if (state.compareAndSet(State.HALF_OPEN, State.OPEN)) {
+ halfOpenFailures.incrementAndGet();
+ LOG.warn(
+ "Circuit breaker reopening for {} due to failure in HALF_OPEN state",
+ loggedEndpoint);
+ lastStateTransitionNanos.set(System.nanoTime());
+ } else {
+ // CAS failed: state already transitioned (e.g. another thread reset).
+ // Log so the failure isn't silently dropped from diagnostic view.
+ LOG.debug(
+ "HALF_OPEN -> OPEN transition lost race for {}; state is now {}",
+ loggedEndpoint,
+ state.get());
+ }
+ break;
+
+ case OPEN:
+ // Already open, nothing to do
+ break;
+ }
+ }
+
+ /**
+ * Atomically increments CLOSED-state counters, performing the periodic decay under the same
+ * lock so that concurrent increments and decays cannot drop samples.
+ *
+ * <p>When {@code totalRequests} reaches {@link #MAX_CLOSED_STATE_REQUESTS}, we halve both
+ * counters (integer division) rather than zeroing them. Halving preserves the current failure
+ * <i>rate</i> while capping the sample count, so:
+ *
+ * <ul>
+ * <li>A burst of failures that drove the rate near the threshold is not silently wiped just
+ * because the cap was reached. Previously, a hard reset could force the breaker to wait
+ * for another {@link #MIN_REQUESTS_THRESHOLD} samples before it could ever trip, hiding
+ * an ongoing incident.
+ * <li>Failure-rate computations remain well-defined the entire time (we only halve when
+ * {@code total} has already reached the cap, so {@code total} after decay is still {@code
+ * >= MIN_REQUESTS_THRESHOLD} for any realistic cap and threshold values).
+ * <li>Memory footprint stays bounded exactly as before.
+ * </ul>
+ *
+ * <p>This is a cheap approximation of an exponentially-weighted moving average: each decay
+ * halves the influence of older samples without requiring us to remember them individually.
+ *
+ * @param isFailure whether the recorded outcome was a failure
+ */
+ private void incrementClosedMetrics(boolean isFailure) {
+ synchronized (closedMetricsLock) {
+ int newTotal = totalRequests.incrementAndGet();
+ if (isFailure) {
+ failedRequests.incrementAndGet();
+ }
+ // Decay counters when the cap is hit. See Javadoc for why halving is preferred
+ // over zeroing. Done inside the same lock as the increment so no sample crosses
+ // the decay boundary.
+ if (newTotal >= MAX_CLOSED_STATE_REQUESTS) {
+ int decayedTotal = totalRequests.get() / 2;
+ int decayedFailed = failedRequests.get() / 2;
+ totalRequests.set(decayedTotal);
+ failedRequests.set(decayedFailed);
+ }
+ }
+ }
+
+ /**
+ * Records the outcome of an out-of-band health probe (e.g. from {@link TritonHealthChecker}).
+ *
+ * <p>This is intentionally distinct from {@link #recordSuccess()}/{@link #recordFailure()},
+ * which are reserved for callers that actually went through {@link #isRequestAllowed()}:
+ *
+ * <ul>
+ * <li>In {@link State#CLOSED} the probe result is folded into the failure-rate computation
+ * just like a real inference call: using a consistent denominator prevents the rate from
+ * being skewed by counting only failures.
+ * <li>In {@link State#HALF_OPEN} a successful probe is <b>ignored</b>: the probe budget is
+ * reserved for real inference traffic, otherwise the breaker could close purely on the
+ * strength of health-endpoint 200s without ever exercising the inference path. A failing
+ * probe, however, is treated as a signal to reopen immediately - if the server cannot
+ * even answer /v2/health/live, it is clearly not ready to serve inference.
+ * <li>In {@link State#OPEN} the probe result is ignored: the scheduled transition back to
+ * HALF_OPEN is driven exclusively by {@link #openStateDuration}.
+ * </ul>
+ *
+ * <p>Implementation note: we deliberately do <b>not</b> delegate to {@link #recordSuccess()}/
+ * {@link #recordFailure()} after observing a CLOSED state. Doing so would introduce a TOCTOU
+ * race - the state could flip to HALF_OPEN between our read and the delegated method's own
+ * {@code state.get()}, causing a health-probe success to silently consume a probe slot in
+ * HALF_OPEN (which is exactly what this method is designed to avoid). Instead, we perform the
+ * CLOSED-state accounting inline via {@link #recordClosedProbeOutcome(boolean)}, which is a
+ * no-op when the state is no longer CLOSED.
+ *
+ * @param healthy whether the health probe reported the server as healthy
+ */
+ public void recordHealthProbe(boolean healthy) {
+ State currentState = state.get();
+ switch (currentState) {
+ case CLOSED:
+ recordClosedProbeOutcome(healthy);
+ break;
+ case HALF_OPEN:
+ // Only failing probes can affect state in HALF_OPEN; successful probes must not
+ // consume or progress the probe budget that belongs to real inference traffic.
+ // A failing probe reopens the breaker directly, without going through
+ // recordFailure() (which would branch on a possibly stale state re-read).
+ if (!healthy) {
+ reopenFromHalfOpen();
+ }
+ break;
+ case OPEN:
+ // Timing of the OPEN -> HALF_OPEN transition is driven by the configured timeout
+ // alone; health probes are informational only.
+ break;
+ }
+ }
+
+ /**
+ * Performs CLOSED-state accounting for a health-probe outcome without going through {@link
+ * #recordSuccess()}/{@link #recordFailure()}.
+ *
+ * <p>Each counter mutation is guarded by a CAS on the state so that if the breaker transitions
+ * out of CLOSED between our caller's state read and our own, the sample is dropped rather than
+ * leaking into HALF_OPEN probe counters or into a reset-boundary CLOSED counter set.
+ */
+ private void recordClosedProbeOutcome(boolean healthy) {
+ // Only account the sample if the state is still CLOSED. If another thread moved us to
+ // OPEN/HALF_OPEN in the meantime, the sample is simply dropped - probes are best-effort
+ // signals, not load-bearing inputs to the state machine in those states.
+ if (state.get() != State.CLOSED) {
+ return;
+ }
+ incrementClosedMetrics(!healthy);
+ // Re-check state after the increment: the CLOSED -> OPEN transition is only evaluated
+ // when we are still CLOSED, using the same pre-CAS snapshot used for the log line.
+ if (state.get() != State.CLOSED) {
+ return;
+ }
+ Snapshot triggeringSnap = snapshotClosedMetrics();
+ if (shouldOpenCircuit(triggeringSnap)) {
+ if (state.compareAndSet(State.CLOSED, State.OPEN)) {
+ LOG.warn(
+ "Circuit breaker opening for {} due to high failure rate: {}/{} "
+ + "(triggered by health probe)",
+ loggedEndpoint,
+ triggeringSnap.failed,
+ triggeringSnap.total);
+ lastStateTransitionNanos.set(System.nanoTime());
+ }
+ }
+ }
+
+ /**
+ * Attempts to reopen the breaker from HALF_OPEN after a failing health probe.
+ *
+ * <p>Mirrors the HALF_OPEN branch of {@link #recordFailure()} but does <i>not</i> call that
+ * method, so we avoid re-reading the state and accidentally taking the CLOSED or OPEN branches
+ * instead (which would at best drop the signal and at worst pollute CLOSED counters).
+ */
+ private void reopenFromHalfOpen() {
+ if (state.compareAndSet(State.HALF_OPEN, State.OPEN)) {
+ halfOpenFailures.incrementAndGet();
+ LOG.warn(
+ "Circuit breaker reopening for {} due to failing health probe in HALF_OPEN",
+ loggedEndpoint);
+ lastStateTransitionNanos.set(System.nanoTime());
+ } else {
+ LOG.debug(
+ "Health probe failure in HALF_OPEN for {} lost race; state is now {}",
+ loggedEndpoint,
+ state.get());
+ }
+ }
+
+ /**
+ * Manually resets the circuit breaker to CLOSED state.
+ *
+ * <p>This is useful for administrative recovery or testing. All counters (both CLOSED and
+ * HALF_OPEN) are cleared so that monitoring does not observe stale values from the previous
+ * state.
+ */
+ public void reset() {
+ State oldState = state.getAndSet(State.CLOSED);
+ if (oldState != State.CLOSED) {
+ LOG.info("Circuit breaker manually reset to CLOSED for {}", loggedEndpoint);
+ lastStateTransitionNanos.set(System.nanoTime());
+ resetClosedMetrics();
+ resetHalfOpenMetrics();
+ }
+ }
+
+ /**
+ * Checks if the circuit should open based on the given counter snapshot.
+ *
+ * <p>Callers that also want to log the triggering counts should pass the same snapshot to the
+ * log statement so the decision and its diagnostic output are perfectly consistent.
+ *
+ * <p>Every production caller takes its own snapshot (typically right after mutating the
+ * counters via {@link #incrementClosedMetrics(boolean)}) and therefore always routes through
+ * this single overload. A previous no-arg variant that re-read the counters internally was
+ * removed as dead code - keeping it would have duplicated the snapshot logic and risked the log
+ * line and the decision being based on different reads.
+ */
+ private boolean shouldOpenCircuit(Snapshot snap) {
+ // Need minimum number of requests to make a decision
+ if (snap.total < MIN_REQUESTS_THRESHOLD) {
+ return false;
+ }
+
+ double failureRate = (double) snap.failed / snap.total;
+ return failureRate >= failureThreshold;
+ }
+
+ /**
+ * Returns a consistent snapshot of CLOSED-state counters.
+ *
+ * <p>Callers should prefer this over two independent {@code .get()} calls to avoid observing a
+ * reset that happens between them.
+ */
+ private Snapshot snapshotClosedMetrics() {
+ synchronized (closedMetricsLock) {
+ return new Snapshot(totalRequests.get(), failedRequests.get());
+ }
+ }
+
+ /** Immutable pair of CLOSED-state counter values. */
+ private static final class Snapshot {
+ final int total;
+ final int failed;
+
+ Snapshot(int total, int failed) {
+ this.total = total;
+ this.failed = failed;
+ }
+ }
+
+ /**
+ * Checks if enough time has passed to transition from OPEN to HALF_OPEN.
+ *
+ * <p>Uses {@link System#nanoTime()} so that NTP adjustments cannot make the breaker either (a)
+ * skip its cool-off because wall-clock jumped forward, or (b) remain OPEN forever because
+ * wall-clock jumped backward. {@code nanoTime()} deltas are the only guarantee-monotonic way to
+ * measure elapsed time on the JVM.
+ */
+ private boolean shouldTransitionToHalfOpen() {
+ long elapsedNanos = System.nanoTime() - lastStateTransitionNanos.get();
+ return elapsedNanos >= openStateDuration.toNanos();
+ }
+
+ /** Gets remaining time in OPEN state (for error messages). */
+ private long getRemainingOpenTimeSeconds() {
+ long elapsedNanos = System.nanoTime() - lastStateTransitionNanos.get();
+ long remainingNanos = openStateDuration.toNanos() - elapsedNanos;
+ return Math.max(0L, TimeUnit.NANOSECONDS.toSeconds(remainingNanos));
+ }
+
+ /** Resets metrics for CLOSED state atomically. */
+ private void resetClosedMetrics() {
+ // Reset both counters under a common lock so that no reader can observe
+ // {@code totalRequests == 0 && failedRequests > 0} during the reset.
+ // shouldOpenCircuit() and getCurrentFailureRate() also acquire this lock
+ // when reading the pair.
+ synchronized (closedMetricsLock) {
+ totalRequests.set(0);
+ failedRequests.set(0);
+ }
+ }
+
+ /** Resets metrics for HALF_OPEN state. */
+ private void resetHalfOpenMetrics() {
+ halfOpenSuccesses.set(0);
+ halfOpenFailures.set(0);
+ halfOpenRequests.set(0);
+ }
+
+ // Getters for monitoring and testing
+
+ public State getState() {
+ return state.get();
+ }
+
+ public String getEndpoint() {
+ return endpoint;
+ }
+
+ public int getTotalRequests() {
+ return totalRequests.get();
+ }
+
+ public int getFailedRequests() {
+ return failedRequests.get();
+ }
+
+ public double getCurrentFailureRate() {
+ Snapshot snap = snapshotClosedMetrics();
+ if (snap.total == 0) {
+ return 0.0;
+ }
+ return (double) snap.failed / snap.total;
+ }
+
+ public long getTimeInCurrentState() {
+ long elapsedNanos = System.nanoTime() - lastStateTransitionNanos.get();
+ return TimeUnit.NANOSECONDS.toMillis(elapsedNanos);
+ }
+}
diff --git a/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonHealthChecker.java b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonHealthChecker.java
new file mode 100644
index 00000000000..3c1bea4ec49
--- /dev/null
+++ b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonHealthChecker.java
@@ -0,0 +1,490 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.model.triton;
+
+import org.apache.flink.annotation.VisibleForTesting;
+
+import okhttp3.Dispatcher;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.Response;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.time.Duration;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Health checker for Triton Inference Server.
+ *
+ * <p>This component periodically polls the Triton server's health endpoint to verify the server is
+ * running and can accept requests. It works in conjunction with {@link TritonCircuitBreaker} to
+ * provide comprehensive fault tolerance.
+ *
+ * <p><b>Health Check Endpoints:</b>
+ *
+ * <ul>
+ * <li><b>/v2/health/live</b>: Checks if server is alive (primary endpoint)
+ * <li><b>/v2/health/ready</b>: Checks if server is ready to accept requests (fallback)
+ * </ul>
+ *
+ * <p><b>Fallback semantics:</b> The readiness endpoint is only tried when the liveness endpoint
+ * responded with an HTTP error (i.e. the server is reachable but reports not-live). On a network
+ * error (connection refused, DNS failure, timeout) we do <i>not</i> fall back, because a second
+ * failing call would only double the time spent per health-check cycle without improving diagnostic
+ * value.
+ *
+ * <p><b>Dedicated HTTP client:</b> Health checks are performed on an HTTP client derived from the
+ * inference client but configured with short, independent timeouts. This isolates the health
+ * checker from long inference timeouts that would otherwise cause health-check cycles to overlap or
+ * stall.
+ *
+ * <p><b>Integration with Circuit Breaker:</b>
+ *
+ * <ul>
+ * <li>The circuit breaker is optional. When {@code null}, health status is only logged and no
+ * circuit breaker state is maintained.
+ * <li>When a circuit breaker is provided, health-check results are forwarded
+ * <b>symmetrically</b>: both successes and failures are reported so the breaker's
+ * CLOSED-state failure rate is not skewed by only counting failures.
+ * </ul>
+ *
+ * <p><b>Thread Safety:</b> This class manages its own background thread for periodic checks and is
+ * safe for concurrent access.
+ *
+ * @see TritonCircuitBreaker
+ */
+public class TritonHealthChecker implements AutoCloseable {
+ private static final Logger LOG = LoggerFactory.getLogger(TritonHealthChecker.class);
+
+ /**
+ * Short, independent timeout for health-check HTTP calls.
+ *
+ * <p>Health probes should fail fast so that {@link #checkInterval} is not starved by long
+ * inference timeouts on the shared {@link OkHttpClient}. A 2-second budget is generous for a
+ * healthy Triton server (whose health endpoints are local and in-memory) yet short enough that
+ * two sequential probes (liveness + fallback) still fit well within any realistic check
+ * interval.
+ */
+ private static final Duration HEALTH_CHECK_TIMEOUT = Duration.ofSeconds(2);
+
+ private final String endpoint;
+
+ /** Sanitised endpoint used for log output; strips any user-info (credentials). */
+ private final String loggedEndpoint;
+
+ /** Dedicated HTTP client with short timeouts, used exclusively for health probes. */
+ private final OkHttpClient healthHttpClient;
+
+ @Nullable private final TritonCircuitBreaker circuitBreaker;
+ private final Duration checkInterval;
+
+ private final ScheduledExecutorService scheduler;
+ private final AtomicBoolean isRunning = new AtomicBoolean(false);
+ private final AtomicBoolean lastCheckResult = new AtomicBoolean(true);
+ private final AtomicLong lastCheckTime = new AtomicLong(0);
+ private final AtomicLong consecutiveFailures = new AtomicLong(0);
+ private final AtomicLong consecutiveSuccesses = new AtomicLong(0);
+
+ private static final String HEALTH_LIVE_PATH = "/v2/health/live";
+ private static final String HEALTH_READY_PATH = "/v2/health/ready";
+
+ /**
+ * Outcome categories for a single endpoint probe. Distinguishing {@link #UNHEALTHY_RESPONSE}
+ * (server answered with a non-2xx) from {@link #IO_ERROR} (network-level failure) lets the
+ * server-health check decide whether a readiness fallback makes sense.
+ */
+ private enum CheckResult {
+ /** Endpoint returned 2xx. */
+ HEALTHY,
+ /** Endpoint returned a non-2xx HTTP response. */
+ UNHEALTHY_RESPONSE,
+ /** Network-level failure (connection refused, timeout, DNS, TLS, ...). */
+ IO_ERROR
+ }
+
+ /**
+ * Creates a new health checker.
+ *
+ * @param endpoint The Triton server base URL
+ * @param httpClient The inference HTTP client; a derived client with short timeouts is used
+ * internally for probes so that the inference client's long timeouts do not block health
+ * checks
+ * @param circuitBreaker The circuit breaker to notify about health status, or {@code null} to
+ * disable circuit-breaker integration (status will only be logged)
+ * @param checkInterval How often to perform health checks
+ */
+ public TritonHealthChecker(
+ String endpoint,
+ OkHttpClient httpClient,
+ @Nullable TritonCircuitBreaker circuitBreaker,
+ Duration checkInterval) {
+ this.endpoint = endpoint;
+ this.loggedEndpoint = sanitizeEndpoint(endpoint);
+ // IMPORTANT: OkHttpClient#newBuilder() shares the Dispatcher and ConnectionPool with the
+ // source client by default. If we kept that sharing, a later call to
+ // {@code healthHttpClient.dispatcher().cancelAll()} in {@link #close()} would cancel
+ // every in-flight INFERENCE call as well - exactly the opposite of what the health
+ // checker should be doing. Installing a dedicated {@link Dispatcher} isolates the
+ // lifecycle of probe calls from inference calls; the {@link okhttp3.ConnectionPool}
+ // can still be shared safely because {@code cancelAll()} only affects the dispatcher
+ // that owns the running calls.
+ this.healthHttpClient =
+ httpClient
+ .newBuilder()
+ .dispatcher(new Dispatcher())
+ .callTimeout(HEALTH_CHECK_TIMEOUT)
+ .connectTimeout(HEALTH_CHECK_TIMEOUT)
+ .readTimeout(HEALTH_CHECK_TIMEOUT)
+ .writeTimeout(HEALTH_CHECK_TIMEOUT)
+ // Disable OkHttp's own retry-on-failure so the short timeout bound is
+ // respected (otherwise a single probe may internally retry and exceed it).
+ .retryOnConnectionFailure(false)
+ .build();
+ this.circuitBreaker = circuitBreaker;
+ this.checkInterval = checkInterval;
+ this.scheduler =
+ Executors.newSingleThreadScheduledExecutor(
+ r -> {
+ Thread thread =
+ new Thread(r, "triton-health-checker-" + loggedEndpoint);
+ thread.setDaemon(true);
+ return thread;
+ });
+
+ LOG.info(
+ "Health checker created for endpoint {} with interval {} (circuit breaker {})",
+ loggedEndpoint,
+ checkInterval,
+ circuitBreaker != null ? "enabled" : "disabled");
+ }
+
+ /**
+ * Starts the periodic health checking.
+ *
+ * <p>Health checks will run at the configured interval on a background thread. The first
+ * scheduled check fires one full interval after {@code start()} is called, leaving callers free
+ * to perform an eager {@link #checkNow()} on initialisation without producing an immediate
+ * duplicate check.
+ */
+ public void start() {
+ if (isRunning.compareAndSet(false, true)) {
+ LOG.info("Starting health checker for {}", loggedEndpoint);
+ long intervalMillis = checkInterval.toMillis();
+ scheduler.scheduleAtFixedRate(
+ this::performHealthCheck,
+ // Initial delay equals the check interval to avoid duplicating an eager
+ // checkNow() performed by the caller immediately before start().
+ intervalMillis,
+ intervalMillis,
+ TimeUnit.MILLISECONDS);
+ }
+ }
+
+ /**
+ * Stops the health checker and releases resources.
+ *
+ * <p>This method is idempotent and safe to call multiple times. Any in-flight health probe on
+ * the dedicated client is cancelled so shutdown cannot hang on a slow server response.
+ */
+ @Override
+ public void close() {
+ if (isRunning.compareAndSet(true, false)) {
+ LOG.info("Stopping health checker for {}", loggedEndpoint);
+ scheduler.shutdown();
+ try {
+ if (!scheduler.awaitTermination(5, TimeUnit.SECONDS)) {
+ scheduler.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ scheduler.shutdownNow();
+ }
+ // Cancel any probe that is still in flight on the dedicated health-check client.
+ // Because we installed a private {@link Dispatcher} in the constructor, this only
+ // affects probe calls - inference calls on the shared inference client are NOT
+ // cancelled.
+ //
+ // Note: health probes use synchronous {@link okhttp3.Call#execute()}, which runs on
+ // the calling thread rather than on the dispatcher's executor. Shutting down that
+ // executor here is therefore primarily a hygiene step to release any lazily-created
+ // cached thread pool deterministically, rather than a functional fix for a leak
+ // (OkHttp's default cached pool would itself idle out after 60s).
+ try {
+ Dispatcher healthDispatcher = healthHttpClient.dispatcher();
+ healthDispatcher.cancelAll();
+ healthDispatcher.executorService().shutdown();
+ } catch (RuntimeException e) {
+ LOG.debug(
+ "Ignoring error while cancelling in-flight health probes for {}: {}",
+ loggedEndpoint,
+ e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Performs a single health check.
+ *
+ * <p>This method is called periodically by the scheduler and should not be called directly in
+ * normal operation.
+ */
+ private void performHealthCheck() {
+ boolean healthy;
+ try {
+ healthy = checkServerHealth();
+ } catch (RuntimeException e) {
+ // checkServerHealth() only invokes HTTP calls that catch IOException internally;
+ // anything propagating here is an unexpected runtime condition. We deliberately do
+ // NOT catch Throwable (OOM / StackOverflow should surface), and InterruptedException
+ // is a checked exception that cannot reach us.
+ LOG.error("Unexpected runtime error during health check for {}", loggedEndpoint, e);
+ healthy = false;
+ }
+
+ recordCheckOutcome(healthy);
+ notifyCircuitBreaker(healthy);
+ }
+
+ /**
+ * Updates {@link #lastCheckTime}, {@link #lastCheckResult} and consecutive-run counters after a
+ * probe. Shared by the periodic {@link #performHealthCheck()} and the synchronous {@link
+ * #checkNow()} paths to keep their accounting identical.
+ */
+ private void recordCheckOutcome(boolean healthy) {
+ lastCheckTime.set(System.currentTimeMillis());
+ lastCheckResult.set(healthy);
+
+ if (healthy) {
+ consecutiveFailures.set(0);
+ long successes = consecutiveSuccesses.incrementAndGet();
+ LOG.debug("Health check passed for {} (consecutive: {})", loggedEndpoint, successes);
+ } else {
+ consecutiveSuccesses.set(0);
+ long failures = consecutiveFailures.incrementAndGet();
+ LOG.warn("Health check failed for {} (consecutive: {})", loggedEndpoint, failures);
+ }
+ }
+
+ /**
+ * Forwards a health-check outcome to the circuit breaker, if configured.
+ *
+ * <p>Delegates to {@link TritonCircuitBreaker#recordHealthProbe(boolean)} so that the breaker
+ * can apply the right state-dependent semantics:
+ *
+ * <ul>
+ * <li>In CLOSED, both successes and failures feed the failure-rate computation.
+ * <li>In HALF_OPEN, probe successes are ignored (otherwise the breaker could close on
+ * health-endpoint 200s without any real inference traffic being exercised) while probe
+ * failures immediately reopen the breaker.
+ * <li>In OPEN, probes are informational only.
+ * </ul>
+ */
+ private void notifyCircuitBreaker(boolean healthy) {
+ if (circuitBreaker == null) {
+ return;
+ }
+ circuitBreaker.recordHealthProbe(healthy);
+ }
+
+ /**
+ * Checks the Triton server health by calling its health endpoints.
+ *
+ * <p>Try the liveness endpoint first. Fall back to readiness <i>only</i> when liveness returned
+ * an HTTP response that indicates "reachable but not live" — a network-level failure
+ * means a second probe would very likely incur another full timeout without adding signal.
+ *
+ * @return true if server is healthy, false otherwise
+ */
+ private boolean checkServerHealth() {
+ CheckResult liveResult = checkEndpointDetailed(HEALTH_LIVE_PATH);
+ if (liveResult == CheckResult.HEALTHY) {
+ return true;
+ }
+ if (liveResult == CheckResult.IO_ERROR) {
+ // Avoid doubling the per-cycle cost on network errors.
+ return false;
+ }
+
+ LOG.debug("Liveness responded unhealthy for {}, trying readiness check", loggedEndpoint);
+ return checkEndpointDetailed(HEALTH_READY_PATH) == CheckResult.HEALTHY;
+ }
+
+ /**
+ * Probes a single health endpoint and returns a detailed outcome.
+ *
+ * @param path The health endpoint path (e.g., "/v2/health/live")
+ * @return one of {@link CheckResult#HEALTHY}, {@link CheckResult#UNHEALTHY_RESPONSE} or {@link
+ * CheckResult#IO_ERROR}
+ */
+ private CheckResult checkEndpointDetailed(String path) {
+ String healthUrl = buildHealthUrl(path);
+
+ Request request = new Request.Builder().url(healthUrl).get().build();
+
+ try (Response response = healthHttpClient.newCall(request).execute()) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(
+ "Health check {} for {}: status={}, body={}",
+ path,
+ loggedEndpoint,
+ response.code(),
+ response.body() != null ? response.body().string() : "null");
+ }
+ return response.isSuccessful() ? CheckResult.HEALTHY : CheckResult.UNHEALTHY_RESPONSE;
+ } catch (IOException e) {
+ LOG.debug(
+ "Health check {} failed for {} ({}): {}",
+ path,
+ loggedEndpoint,
+ e.getClass().getSimpleName(),
+ e.getMessage());
+ return CheckResult.IO_ERROR;
+ }
+ }
+
+ /**
+ * Builds the full URL for a health endpoint.
+ *
+ * <p>The configured endpoint may already point at a Triton sub-path such as {@code
+ * http://host:8000/v2/models/.../infer}. We strip any existing path while preserving scheme,
+ * host, port and user-info, then append the health path. Using {@link URI} (rather than naive
+ * substring matching on {@code "/v2"}) keeps hostnames that happen to contain the literal
+ * {@code "/v2"} substring — e.g. {@code http://triton-v2.example.com:8000} —
+ * intact.
+ *
+ * @param path The health endpoint path
+ * @return The complete health check URL
+ */
+ @VisibleForTesting
+ String buildHealthUrl(String path) {
+ String trimmed = endpoint.replaceAll("/*$", "");
+ try {
+ URI uri = new URI(trimmed);
+ // Only strip the path when the URI parses as an absolute URL with authority.
+ if (uri.isAbsolute() && uri.getHost() != null) {
+ URI base =
+ new URI(
+ uri.getScheme(),
+ uri.getUserInfo(),
+ uri.getHost(),
+ uri.getPort(),
+ path,
+ null,
+ null);
+ return base.toString();
+ }
+ } catch (URISyntaxException e) {
+ LOG.debug(
+ "Endpoint {} is not a valid URI ({}); falling back to string-based health URL construction.",
+ loggedEndpoint,
+ e.getMessage());
+ }
+
+ // Fallback: conservative path-segment match. Only strip "/v2/..." or a trailing "/v2",
+ // never a host-level substring like "triton-v2.example.com".
+ String baseUrl = trimmed;
+ int idx = baseUrl.indexOf("/v2/");
+ if (idx >= 0) {
+ baseUrl = baseUrl.substring(0, idx);
+ } else if (baseUrl.endsWith("/v2")) {
+ baseUrl = baseUrl.substring(0, baseUrl.length() - 3);
+ }
+ return baseUrl + path;
+ }
+
+ /**
+ * Returns an endpoint representation safe to log: any {@code user:password@} prefix in the
+ * authority is stripped so credentials cannot leak to INFO/WARN logs.
+ *
+ * <p>Delegates to {@link TritonUtils#sanitizeUrl(String)} so that every Triton component shares
+ * a single, tested redaction implementation; kept here as a thin {@link VisibleForTesting}
+ * wrapper to preserve the existing test entry-point.
+ */
+ @VisibleForTesting
+ static String sanitizeEndpoint(String endpoint) {
+ return TritonUtils.sanitizeUrl(endpoint);
+ }
+
+ /**
+ * Performs an immediate synchronous health check.
+ *
+ * <p><b>INTENTIONALLY BYPASSES THE CIRCUIT BREAKER.</b> Unlike {@link #performHealthCheck()},
+ * this method does <i>not</i> call {@link TritonCircuitBreaker#recordHealthProbe(boolean)},
+ * because its designed use case is a one-shot probe during initialisation - folding that single
+ * synchronous sample into the circuit-breaker's failure-rate or half-open budget would bias
+ * both the CLOSED-state rate computation and the HALF_OPEN probe accounting in ways callers
+ * almost never want.
+ *
+ * <p><b>Do not</b> invoke this method from a recurring schedule as a substitute for {@link
+ * #start()}; the periodic path is the only one that keeps the breaker in sync with server
+ * health over time. The intended callers are (a) {@link AbstractTritonModelFunction#open} for
+ * the initial probe and (b) tests. Monitoring dashboards should read {@link #isHealthy()}
+ * rather than calling this method repeatedly.
+ *
+ * @return true if server is healthy, false otherwise
+ */
+ public boolean checkNow() {
+ boolean healthy;
+ try {
+ healthy = checkServerHealth();
+ } catch (RuntimeException e) {
+ LOG.error(
+ "Unexpected runtime error during immediate health check for {}",
+ loggedEndpoint,
+ e);
+ healthy = false;
+ }
+ // Keep lastCheckTime/lastCheckResult in sync with the periodic path so monitors observe
+ // a consistent timestamp regardless of which code path produced the latest result.
+ recordCheckOutcome(healthy);
+ return healthy;
+ }
+
+ // Getters for monitoring
+
+ public boolean isHealthy() {
+ return lastCheckResult.get();
+ }
+
+ public long getLastCheckTime() {
+ return lastCheckTime.get();
+ }
+
+ public long getConsecutiveFailures() {
+ return consecutiveFailures.get();
+ }
+
+ public long getConsecutiveSuccesses() {
+ return consecutiveSuccesses.get();
+ }
+
+ public boolean isRunning() {
+ return isRunning.get();
+ }
+}
diff --git a/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonInferenceModelFunction.java b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonInferenceModelFunction.java
index 24622fac06a..7bebfc942c3 100644
--- a/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonInferenceModelFunction.java
+++ b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonInferenceModelFunction.java
@@ -89,8 +89,19 @@ public class TritonInferenceModelFunction extends AbstractTritonModelFunction {
MediaType.get("application/json; charset=utf-8");
private static final ObjectMapper objectMapper = new ObjectMapper();
- /** Reusable buffer for gzip compression to avoid repeated allocations. */
- private final ByteArrayOutputStream compressionBuffer = new ByteArrayOutputStream(1024);
+ /**
+ * Initial capacity for the per-call gzip compression buffer.
+ *
+ * <p>A previous implementation kept a single {@link ByteArrayOutputStream} as an instance field
+ * to avoid repeated allocations. That was unsafe: {@link #asyncPredict(RowData)} is invoked
+ * concurrently from the AsyncIO operator (capacity is typically far greater than 1), so
+ * concurrent calls would race on {@code reset()} / {@code toByteArray()} and produce torn,
+ * corrupt bytes. Allocating a fresh buffer per call trades a cheap allocation for correctness;
+ * {@link ThreadLocal} was considered but rejected because the AsyncIO runner may dispatch
+ * different records to the same thread after the network callback, leaving stale residue
+ * visible if we forgot to {@code reset()}.
+ */
+ private static final int COMPRESSION_BUFFER_INITIAL_CAPACITY = 1024;
private final LogicalType inputType;
private final LogicalType outputType;
@@ -124,9 +135,16 @@ public class TritonInferenceModelFunction extends AbstractTritonModelFunction {
CompletableFuture<Collection<RowData>> future = new CompletableFuture<>();
try {
+ // Check circuit breaker before making request
+ checkCircuitBreaker();
+
String requestBody = buildInferenceRequest(rowData);
String url =
TritonUtils.buildInferenceUrl(getEndpoint(), getModelName(), getModelVersion());
+ // Sanitized URL used exclusively for log lines and exception messages so that an
+ // endpoint carrying basic-auth credentials (user:password@host) does not leak into
+ // logs or surface in stack traces that may reach users.
+ String loggedUrl = TritonUtils.sanitizeUrl(url);
Request.Builder requestBuilder = new Request.Builder().url(url);
@@ -136,8 +154,11 @@ public class TritonInferenceModelFunction extends AbstractTritonModelFunction {
"gzip".equalsIgnoreCase(getCompression()),
"Unsupported compression algorithm: '%s'. Currently only 'gzip' is supported.",
getCompression());
- // Only support GZIP: Compress request body with gzip using reusable buffer.
- compressionBuffer.reset();
+ // Only support GZIP. Allocate a fresh per-call buffer: see
+ // COMPRESSION_BUFFER_INITIAL_CAPACITY Javadoc for why reusing an instance-level
+ // buffer would be unsafe under concurrent asyncPredict() invocations.
+ ByteArrayOutputStream compressionBuffer =
+ new ByteArrayOutputStream(COMPRESSION_BUFFER_INITIAL_CAPACITY);
try (GZIPOutputStream gzos = new GZIPOutputStream(compressionBuffer)) {
gzos.write(requestBody.getBytes(StandardCharsets.UTF_8));
}
@@ -171,13 +192,19 @@ public class TritonInferenceModelFunction extends AbstractTritonModelFunction {
"Triton inference request failed due to network error",
e);
- // Wrap IOException in TritonNetworkException
+ // Record failure with circuit breaker
+ recordFailure();
+
+ // Wrap IOException in TritonNetworkException. The sanitized
+ // URL is used in the user-visible message so that basic-auth
+ // credentials embedded in the endpoint cannot leak through
+ // exception stacks, CI logs, or error dashboards.
TritonNetworkException networkException =
new TritonNetworkException(
String.format(
"Failed to connect to Triton server at %s: %s. "
+ "This may indicate network connectivity issues, DNS resolution failure, or server unavailability.",
- url, e.getMessage()),
+ loggedUrl, e.getMessage()),
e);
future.completeExceptionally(networkException);
@@ -188,16 +215,56 @@ public class TritonInferenceModelFunction extends AbstractTritonModelFunction {
throws IOException {
try {
if (!response.isSuccessful()) {
+ // Record failure for 5xx errors (server issues) so
+ // they feed into the circuit breaker's failure-rate
+ // signal. 4xx errors are intentionally NOT recorded:
+ // they represent client-side configuration problems
+ // (wrong model name, bad shape, missing auth) that
+ // would persist regardless of server health. Folding
+ // them into the circuit breaker would cause a user
+ // with a persistent config error to force-open the
+ // breaker and deny traffic to an otherwise healthy
+ // server. The trade-off - 4xx calls do not contribute
+ // to the breaker's denominator - is a deliberate
+ // choice; the breaker is a server-health signal, not
+ // a generic error-rate tracker.
+ if (response.code() >= 500) {
+ recordFailure();
+ }
handleErrorResponse(response, future);
return;
}
- String responseBody = response.body().string();
+ // OkHttp guarantees that a response produced by
+ // Call.enqueue() has a non-null body for successful
+ // responses, but we guard defensively: a malformed
+ // proxy/interceptor could violate that invariant, and
+ // we prefer a typed client exception to an NPE in the
+ // user's pipeline.
+ okhttp3.ResponseBody body = response.body();
+ if (body == null) {
+ recordFailure();
+ future.completeExceptionally(
+ new TritonClientException(
+ "Triton response has no body for "
+ + loggedUrl
+ + ". This typically indicates a misbehaving "
+ + "proxy or interceptor between the client and Triton.",
+ response.code()));
+ return;
+ }
+ String responseBody = body.string();
Collection<RowData> result =
parseInferenceResponse(responseBody);
+
+ // Record success with circuit breaker
+ recordSuccess();
+
future.complete(result);
} catch (JsonProcessingException e) {
LOG.error("Failed to parse Triton inference response", e);
+ // Don't record as circuit breaker failure - this is a
+ // client parsing issue
future.completeExceptionally(
new TritonClientException(
"Failed to parse Triton response JSON: "
@@ -206,6 +273,8 @@ public class TritonInferenceModelFunction extends AbstractTritonModelFunction {
400));
} catch (Exception e) {
LOG.error("Failed to process Triton inference response", e);
+ // Don't record as circuit breaker failure - processing
+ // error
future.completeExceptionally(e);
} finally {
response.close();
@@ -242,7 +311,11 @@ public class TritonInferenceModelFunction extends AbstractTritonModelFunction {
errorMsg.append("\n=== Request Configuration ===\n");
errorMsg.append(
String.format(" Model: %s (version: %s)\n", getModelName(), getModelVersion()));
- errorMsg.append(String.format(" Endpoint: %s\n", getEndpoint()));
+ // Use the sanitized endpoint in user-facing error output so that basic-auth credentials
+ // in the configured endpoint cannot leak through error messages surfaced in job logs,
+ // metrics, or downstream exception stacks.
+ String sanitizedEndpoint = TritonUtils.sanitizeUrl(getEndpoint());
+ errorMsg.append(String.format(" Endpoint: %s\n", sanitizedEndpoint));
errorMsg.append(String.format(" Input column: %s\n", inputName));
errorMsg.append(String.format(" Input Flink type: %s\n", inputType));
errorMsg.append(
@@ -281,7 +354,7 @@ public class TritonInferenceModelFunction extends AbstractTritonModelFunction {
.append(getModelVersion())
.append("\n");
errorMsg.append(" • Check model is loaded: GET ")
- .append(getEndpoint())
+ .append(sanitizedEndpoint)
.append("\n");
} else if (statusCode == 401 || statusCode == 403) {
errorMsg.append(" • Check auth-token configuration\n");
diff --git a/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonOptions.java b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonOptions.java
index bf3b7d8967a..f28a27a8b8b 100644
--- a/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonOptions.java
+++ b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonOptions.java
@@ -181,4 +181,76 @@ public class TritonOptions {
+ "Example: %s",
code("'X-Custom-Header:value,X-Another:value2'"))
.build());
+
+ // ==================== Health Check and Circuit Breaker Options ====================
+
+ @Documentation.Section({Documentation.Sections.MODEL_TRITON_ADVANCED})
+ public static final ConfigOption<Boolean> HEALTH_CHECK_ENABLED =
+ ConfigOptions.key("health-check-enabled")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ Description.builder()
+ .text(
+ "Whether to enable periodic health checks for the Triton server. "
+ + "When enabled, the health checker will periodically call %s endpoint "
+ + "to verify server availability. Defaults to false.",
+ code("/v2/health/live"))
+ .build());
+
+ @Documentation.Section({Documentation.Sections.MODEL_TRITON_ADVANCED})
+ public static final ConfigOption<Duration> HEALTH_CHECK_INTERVAL =
+ ConfigOptions.key("health-check-interval")
+ .durationType()
+ .defaultValue(Duration.ofSeconds(30))
+ .withDescription(
+ "Interval between health check requests. "
+ + "Shorter intervals provide faster failure detection but increase server load. "
+ + "Defaults to 30 seconds. Only effective when health-check-enabled is true.");
+
+ @Documentation.Section({Documentation.Sections.MODEL_TRITON_ADVANCED})
+ public static final ConfigOption<Boolean> CIRCUIT_BREAKER_ENABLED =
+ ConfigOptions.key("circuit-breaker-enabled")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ Description.builder()
+ .text(
+ "Whether to enable circuit breaker protection. "
+ + "When enabled, the circuit breaker will automatically fail fast when the server "
+ + "is unhealthy, preventing cascading failures and reducing load on the failing server. "
+ + "The circuit breaker implements a three-state model: CLOSED (normal), OPEN (failing fast), "
+ + "and HALF_OPEN (testing recovery). Defaults to false.")
+ .build());
+
+ @Documentation.Section({Documentation.Sections.MODEL_TRITON_ADVANCED})
+ public static final ConfigOption<Double> CIRCUIT_BREAKER_FAILURE_THRESHOLD =
+ ConfigOptions.key("circuit-breaker-failure-threshold")
+ .doubleType()
+ .defaultValue(0.5)
+ .withDescription(
+ "Failure rate threshold (0.0-1.0) that triggers the circuit breaker to open. "
+ + "For example, 0.5 means the circuit will open when 50% of recent requests fail. "
+ + "Requires a minimum of 10 requests before evaluation. Defaults to 0.5 (50%). "
+ + "Only effective when circuit-breaker-enabled is true.");
+
+ @Documentation.Section({Documentation.Sections.MODEL_TRITON_ADVANCED})
+ public static final ConfigOption<Duration> CIRCUIT_BREAKER_TIMEOUT =
+ ConfigOptions.key("circuit-breaker-timeout")
+ .durationType()
+ .defaultValue(Duration.ofSeconds(60))
+ .withDescription(
+ "Duration to keep the circuit breaker in OPEN state before transitioning to HALF_OPEN. "
+ + "In HALF_OPEN state, limited requests are allowed to probe if the server has recovered. "
+ + "Defaults to 60 seconds. Only effective when circuit-breaker-enabled is true.");
+
+ @Documentation.Section({Documentation.Sections.MODEL_TRITON_ADVANCED})
+ public static final ConfigOption<Integer> CIRCUIT_BREAKER_HALF_OPEN_REQUESTS =
+ ConfigOptions.key("circuit-breaker-half-open-requests")
+ .intType()
+ .defaultValue(3)
+ .withDescription(
+ "Number of successful test requests required in HALF_OPEN state to close the circuit. "
+ + "If any request fails in HALF_OPEN state, the circuit immediately reopens. "
+ + "Defaults to 3 requests. Only effective when circuit-breaker-enabled is true.");
}
diff --git a/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonUtils.java b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonUtils.java
index f94aaf9d6ce..4fc9621a866 100644
--- a/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonUtils.java
+++ b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/TritonUtils.java
@@ -22,6 +22,8 @@ import okhttp3.OkHttpClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -145,6 +147,45 @@ public class TritonUtils {
return String.format("%s/%s/versions/%s/infer", baseUrl, modelName, modelVersion);
}
+ /**
+ * Returns a representation of the given URL or endpoint that is safe to include in logs and
+ * user-visible error messages.
+ *
+ * <p>Any {@code user:password@} prefix in the URI's authority is stripped so that basic-auth
+ * credentials configured on the Triton endpoint cannot leak through INFO/WARN logs, exception
+ * stacks surfaced by Flink, or metrics consumed by monitoring dashboards.
+ *
+ * <p>Inputs that do not parse as URIs (for example, bare {@code host:port} strings, or values
+ * already containing a placeholder) are returned unchanged - such inputs cannot carry
+ * credentials in the {@code userInfo} component by format.
+ *
+ * <p>Returns {@code "<null>"} for a {@code null} input so that callers can embed the result in
+ * a format string without a separate null-check.
+ */
+ public static String sanitizeUrl(String url) {
+ if (url == null) {
+ return "<null>";
+ }
+ try {
+ URI uri = new URI(url);
+ if (uri.isAbsolute() && uri.getUserInfo() != null) {
+ URI sanitized =
+ new URI(
+ uri.getScheme(),
+ null, // drop userInfo
+ uri.getHost(),
+ uri.getPort(),
+ uri.getPath(),
+ uri.getQuery(),
+ uri.getFragment());
+ return sanitized.toString();
+ }
+ } catch (URISyntaxException ignored) {
+ // fall through: return the original string
+ }
+ return url;
+ }
+
private static class ClientValue {
private final OkHttpClient client;
private final AtomicInteger referenceCount;
diff --git a/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/exception/TritonCircuitBreakerOpenException.java b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/exception/TritonCircuitBreakerOpenException.java
new file mode 100644
index 00000000000..a041d91894d
--- /dev/null
+++ b/flink-models/flink-model-triton/src/main/java/org/apache/flink/model/triton/exception/TritonCircuitBreakerOpenException.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.model.triton.exception;
+
+/**
+ * Exception thrown when the circuit breaker is in OPEN state and requests are not allowed.
+ *
+ * <p>This exception indicates that the Triton server is considered unhealthy based on recent
+ * failures, and the circuit breaker is preventing additional requests to avoid cascading failures.
+ *
+ * <p>The circuit will automatically transition to HALF_OPEN state after a configured timeout and
+ * will attempt to probe the server health with limited requests.
+ *
+ * <p><b>Retry semantics:</b> This exception is considered retryable because the underlying cause is
+ * a transient server-health condition; once the OPEN timeout elapses the circuit will move to
+ * HALF_OPEN and subsequent requests may succeed. It is categorised as {@link
+ * ErrorCategory#SERVER_ERROR} to align with the reason the breaker tripped.
+ *
+ * @see org.apache.flink.model.triton.TritonCircuitBreaker
+ */
+public class TritonCircuitBreakerOpenException extends TritonException {
+
+ private static final long serialVersionUID = 1L;
+
+ public TritonCircuitBreakerOpenException(String message) {
+ super(message);
+ }
+
+ public TritonCircuitBreakerOpenException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ @Override
+ public boolean isRetryable() {
+ // The circuit opens in response to server-side unhealthiness, and will automatically
+ // transition to HALF_OPEN after its configured timeout. Upstream retry logic should be
+ // allowed to retry (typically after a backoff) rather than giving up on the request.
+ return true;
+ }
+
+ @Override
+ public ErrorCategory getCategory() {
+ // The breaker trips because the Triton server is considered unhealthy, so classify this
+ // as a server-side error for metrics and alerting.
+ return ErrorCategory.SERVER_ERROR;
+ }
+}
diff --git a/flink-models/flink-model-triton/src/test/java/org/apache/flink/model/triton/TritonCircuitBreakerTest.java b/flink-models/flink-model-triton/src/test/java/org/apache/flink/model/triton/TritonCircuitBreakerTest.java
new file mode 100644
index 00000000000..8a770faf17c
--- /dev/null
+++ b/flink-models/flink-model-triton/src/test/java/org/apache/flink/model/triton/TritonCircuitBreakerTest.java
@@ -0,0 +1,827 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.model.triton;
+
+import org.apache.flink.model.triton.exception.TritonCircuitBreakerOpenException;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link TritonCircuitBreaker}. */
+class TritonCircuitBreakerTest {
+
+ /**
+ * Timeout used for the OPEN state in tests that exercise the OPEN -> HALF_OPEN transition. Kept
+ * generous enough to remain reliable on loaded CI hosts where short sleeps can jitter.
+ */
+ private static final Duration SHORT_OPEN_TIMEOUT = Duration.ofMillis(200);
+
+ /**
+ * Wait duration used when verifying the OPEN -> HALF_OPEN transition. Must be strictly greater
+ * than {@link #SHORT_OPEN_TIMEOUT} with enough headroom to survive GC pauses / scheduler jitter
+ * on CI.
+ */
+ private static final long POST_OPEN_SLEEP_MS = 500L;
+
+ @Test
+ void testCircuitBreakerStartsInClosedState() {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, Duration.ofSeconds(1), 3);
+
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+ assertThat(breaker.isRequestAllowed()).isTrue();
+ }
+
+ @Test
+ void testCircuitOpensAfterThresholdExceeded() {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, Duration.ofSeconds(60), 3);
+
+ // Send 10 requests with 6 failures (60% failure rate)
+ for (int i = 0; i < 10; i++) {
+ breaker.isRequestAllowed();
+ if (i < 6) {
+ breaker.recordFailure();
+ } else {
+ breaker.recordSuccess();
+ }
+ }
+
+ // Circuit should now be OPEN
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+ assertThatThrownBy(() -> breaker.isRequestAllowed())
+ .isInstanceOf(TritonCircuitBreakerOpenException.class)
+ .hasMessageContaining("Circuit breaker is OPEN");
+ }
+
+ @Test
+ void testCircuitRemainsClosedWithLowFailureRate() {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, Duration.ofSeconds(60), 3);
+
+ // Send 10 requests with only 4 failures (40% failure rate, below 50% threshold)
+ for (int i = 0; i < 10; i++) {
+ breaker.isRequestAllowed();
+ if (i < 4) {
+ breaker.recordFailure();
+ } else {
+ breaker.recordSuccess();
+ }
+ }
+
+ // Circuit should still be CLOSED
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+ assertThat(breaker.isRequestAllowed()).isTrue();
+ }
+
+ @Test
+ void testCircuitDoesNotOpenWithFewRequests() {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, Duration.ofSeconds(60), 3);
+
+ // Send only 5 requests with 100% failure rate
+ // Circuit should NOT open because we need minimum 10 requests
+ for (int i = 0; i < 5; i++) {
+ breaker.isRequestAllowed();
+ breaker.recordFailure();
+ }
+
+ // Circuit should still be CLOSED (not enough samples)
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+ assertThat(breaker.getCurrentFailureRate()).isEqualTo(1.0);
+ }
+
+ @Test
+ void testCircuitTransitionsToHalfOpenAfterTimeout() throws InterruptedException {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, SHORT_OPEN_TIMEOUT, 3);
+
+ // Open the circuit
+ for (int i = 0; i < 10; i++) {
+ breaker.isRequestAllowed();
+ breaker.recordFailure();
+ }
+
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+
+ // Wait for timeout
+ Thread.sleep(POST_OPEN_SLEEP_MS);
+
+ // Next request should transition to HALF_OPEN
+ assertThat(breaker.isRequestAllowed()).isTrue();
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.HALF_OPEN);
+ }
+
+ @Test
+ void testHalfOpenAllowsLimitedRequests() throws InterruptedException {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, SHORT_OPEN_TIMEOUT, 3);
+
+ // Open the circuit
+ for (int i = 0; i < 10; i++) {
+ breaker.isRequestAllowed();
+ breaker.recordFailure();
+ }
+
+ // Wait for transition to HALF_OPEN
+ Thread.sleep(POST_OPEN_SLEEP_MS);
+
+ // Should allow exactly 3 requests (halfOpenMaxRequests)
+ assertThat(breaker.isRequestAllowed()).isTrue(); // 1st request
+ assertThat(breaker.isRequestAllowed()).isTrue(); // 2nd request
+ assertThat(breaker.isRequestAllowed()).isTrue(); // 3rd request
+
+ // 4th request should be rejected
+ assertThatThrownBy(() -> breaker.isRequestAllowed())
+ .isInstanceOf(TritonCircuitBreakerOpenException.class)
+ .hasMessageContaining("HALF_OPEN");
+ }
+
+ @Test
+ void testHalfOpenClosesAfterSuccessfulProbes() throws InterruptedException {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, SHORT_OPEN_TIMEOUT, 3);
+
+ // Open the circuit
+ for (int i = 0; i < 10; i++) {
+ breaker.isRequestAllowed();
+ breaker.recordFailure();
+ }
+
+ // Wait for transition to HALF_OPEN
+ Thread.sleep(POST_OPEN_SLEEP_MS);
+
+ // Send 3 successful probe requests
+ for (int i = 0; i < 3; i++) {
+ breaker.isRequestAllowed();
+ breaker.recordSuccess();
+ }
+
+ // Circuit should now be CLOSED again
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+ assertThat(breaker.isRequestAllowed()).isTrue();
+ }
+
+ @Test
+ void testHalfOpenReopensOnFailure() throws InterruptedException {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, SHORT_OPEN_TIMEOUT, 3);
+
+ // Open the circuit
+ for (int i = 0; i < 10; i++) {
+ breaker.isRequestAllowed();
+ breaker.recordFailure();
+ }
+
+ // Wait for transition to HALF_OPEN
+ Thread.sleep(POST_OPEN_SLEEP_MS);
+
+ // First probe request succeeds
+ breaker.isRequestAllowed();
+ breaker.recordSuccess();
+
+ // Second probe request fails - should reopen circuit
+ breaker.isRequestAllowed();
+ breaker.recordFailure();
+
+ // Circuit should be OPEN again
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+ }
+
+ @Test
+ void testManualReset() {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, Duration.ofSeconds(60), 3);
+
+ // Open the circuit
+ for (int i = 0; i < 10; i++) {
+ breaker.isRequestAllowed();
+ breaker.recordFailure();
+ }
+
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+
+ // Manual reset
+ breaker.reset();
+
+ // Circuit should be CLOSED
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+ assertThat(breaker.getTotalRequests()).isEqualTo(0);
+ assertThat(breaker.getFailedRequests()).isEqualTo(0);
+ }
+
+ @Test
+ void testMetricsTracking() {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, Duration.ofSeconds(60), 3);
+
+ // Record some requests
+ for (int i = 0; i < 20; i++) {
+ breaker.isRequestAllowed();
+ if (i % 3 == 0) {
+ breaker.recordFailure();
+ } else {
+ breaker.recordSuccess();
+ }
+ }
+
+ assertThat(breaker.getTotalRequests()).isEqualTo(20);
+ assertThat(breaker.getFailedRequests()).isEqualTo(7); // Every 3rd request fails
+ assertThat(breaker.getCurrentFailureRate()).isCloseTo(0.35, within(0.01));
+ }
+
+ private static org.assertj.core.data.Offset<Double> within(double offset) {
+ return org.assertj.core.data.Offset.offset(offset);
+ }
+
+ // ---------------------------------------------------------------------------------------
+ // Regression tests for the second-round CR fixes.
+ // ---------------------------------------------------------------------------------------
+
+ @Test
+ void testResetAlsoClearsHalfOpenMetrics() throws InterruptedException {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, SHORT_OPEN_TIMEOUT, 3);
+
+ // Open then transition to HALF_OPEN and dispatch a single probe so halfOpenRequests > 0.
+ for (int i = 0; i < 10; i++) {
+ breaker.isRequestAllowed();
+ breaker.recordFailure();
+ }
+ Thread.sleep(POST_OPEN_SLEEP_MS);
+ breaker.isRequestAllowed();
+ breaker.recordSuccess();
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.HALF_OPEN);
+
+ // reset() must wipe BOTH CLOSED and HALF_OPEN metrics so a subsequent HALF_OPEN cycle
+ // starts from a clean slate.
+ breaker.reset();
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+ assertThat(breaker.getTotalRequests()).isZero();
+ assertThat(breaker.getFailedRequests()).isZero();
+
+ // Re-opening and re-entering HALF_OPEN should still respect the configured probe budget.
+ for (int i = 0; i < 10; i++) {
+ breaker.isRequestAllowed();
+ breaker.recordFailure();
+ }
+ Thread.sleep(POST_OPEN_SLEEP_MS);
+ // Exactly 3 probes should be permitted (not 2 — which would be the case if half-open
+ // counters leaked across the reset).
+ assertThat(breaker.isRequestAllowed()).isTrue();
+ assertThat(breaker.isRequestAllowed()).isTrue();
+ assertThat(breaker.isRequestAllowed()).isTrue();
+ assertThatThrownBy(() -> breaker.isRequestAllowed())
+ .isInstanceOf(TritonCircuitBreakerOpenException.class);
+ }
+
+ @Test
+ void testResetIsIdempotent() {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, Duration.ofSeconds(60), 3);
+
+ // Reset a fresh breaker — should not throw and counters stay zero.
+ breaker.reset();
+ breaker.reset();
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+ assertThat(breaker.getTotalRequests()).isZero();
+ }
+
+ @Test
+ void testConcurrentFailuresOpenCircuitAtMostOnce() throws Exception {
+ // The CAS in recordFailure() guarantees that at most one thread observes the
+ // CLOSED -> OPEN transition edge, so lastStateTransitionTime is bumped exactly once
+ // regardless of how many failures race concurrently. This test asserts that property
+ // by watching the visible transition time and the final state.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, Duration.ofSeconds(60), 3);
+
+ // Prime just under the threshold so the next failure from any thread tips it over.
+ for (int i = 0; i < 9; i++) {
+ breaker.recordFailure();
+ }
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+
+ int workerCount = 32;
+ ExecutorService pool = Executors.newFixedThreadPool(workerCount);
+ CountDownLatch gate = new CountDownLatch(1);
+ CountDownLatch done = new CountDownLatch(workerCount);
+
+ try {
+ for (int i = 0; i < workerCount; i++) {
+ pool.submit(
+ () -> {
+ try {
+ gate.await();
+ breaker.recordFailure();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ done.countDown();
+ }
+ });
+ }
+ gate.countDown();
+ assertThat(done.await(5, TimeUnit.SECONDS)).isTrue();
+ } finally {
+ pool.shutdownNow();
+ }
+
+ // Final state must be OPEN, and isRequestAllowed() must fail fast.
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+ assertThatThrownBy(breaker::isRequestAllowed)
+ .isInstanceOf(TritonCircuitBreakerOpenException.class);
+ }
+
+ @Test
+ void testHighConcurrencyHalfOpenRespectsProbeBudget() throws Exception {
+ int probeBudget = 5;
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker(
+ "http://localhost:8000", 0.5, SHORT_OPEN_TIMEOUT, probeBudget);
+
+ for (int i = 0; i < 10; i++) {
+ breaker.recordFailure();
+ }
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+ Thread.sleep(POST_OPEN_SLEEP_MS);
+
+ int threads = 64;
+ ExecutorService pool = Executors.newFixedThreadPool(threads);
+ CountDownLatch gate = new CountDownLatch(1);
+ CountDownLatch done = new CountDownLatch(threads);
+ AtomicInteger allowed = new AtomicInteger(0);
+
+ try {
+ for (int i = 0; i < threads; i++) {
+ pool.submit(
+ () -> {
+ try {
+ gate.await();
+ try {
+ breaker.isRequestAllowed();
+ allowed.incrementAndGet();
+ } catch (TritonCircuitBreakerOpenException ignored) {
+ // expected for rejected probes
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ done.countDown();
+ }
+ });
+ }
+ gate.countDown();
+ assertThat(done.await(5, TimeUnit.SECONDS)).isTrue();
+ } finally {
+ pool.shutdownNow();
+ }
+
+ // Under concurrent access the probe budget must never be exceeded.
+ assertThat(allowed.get()).isLessThanOrEqualTo(probeBudget);
+ assertThat(allowed.get()).isPositive();
+ }
+
+ @Test
+ void testRecordSuccessAlone_neverOpensCircuit() {
+ // Regression: previously recordSuccess() contained a shouldOpenCircuit() branch that
+ // could never actually evaluate to true. A sequence of pure successes must leave the
+ // circuit CLOSED regardless of how many samples we feed.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, Duration.ofSeconds(60), 3);
+
+ for (int i = 0; i < 500; i++) {
+ breaker.recordSuccess();
+ }
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+ assertThat(breaker.getFailedRequests()).isZero();
+ }
+
+ @Test
+ void testRecordHealthProbeInClosedStateFeedsFailureRate() {
+ // In CLOSED the health probe outcome is treated symmetrically so the failure rate has
+ // a consistent denominator. A mix of 9 healthy + 1 unhealthy probe on a breaker with
+ // threshold 0.5 must NOT open the circuit.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, Duration.ofSeconds(60), 3);
+
+ for (int i = 0; i < 9; i++) {
+ breaker.recordHealthProbe(true);
+ }
+ breaker.recordHealthProbe(false);
+
+ assertThat(breaker.getTotalRequests()).isEqualTo(10);
+ assertThat(breaker.getFailedRequests()).isEqualTo(1);
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+ }
+
+ @Test
+ void testRecordHealthProbeInHalfOpenIgnoresSuccesses() throws InterruptedException {
+ // Regression: previously the health checker fed successes into recordSuccess() which
+ // accumulated halfOpenSuccesses, letting the breaker close purely on health probes
+ // without ever exercising a real inference call. recordHealthProbe(true) in HALF_OPEN
+ // must be a no-op.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, SHORT_OPEN_TIMEOUT, 3);
+
+ for (int i = 0; i < 10; i++) {
+ breaker.recordFailure();
+ }
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+ Thread.sleep(POST_OPEN_SLEEP_MS);
+ // Transition to HALF_OPEN by asking for a request slot; we don't care about the probe
+ // itself, only that the state moves.
+ try {
+ breaker.isRequestAllowed();
+ } catch (TritonCircuitBreakerOpenException ignored) {
+ // no-op
+ }
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.HALF_OPEN);
+
+ // Pump healthy probes; the breaker must remain HALF_OPEN because those are ignored.
+ for (int i = 0; i < 50; i++) {
+ breaker.recordHealthProbe(true);
+ }
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.HALF_OPEN);
+ }
+
+ @Test
+ void testRecordHealthProbeInHalfOpenFailureReopens() throws InterruptedException {
+ // A failing health probe in HALF_OPEN must reopen the circuit immediately: if the
+ // server cannot even respond to /v2/health/live there is no point letting further
+ // probe requests hit it.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, SHORT_OPEN_TIMEOUT, 3);
+
+ for (int i = 0; i < 10; i++) {
+ breaker.recordFailure();
+ }
+ Thread.sleep(POST_OPEN_SLEEP_MS);
+ try {
+ breaker.isRequestAllowed();
+ } catch (TritonCircuitBreakerOpenException ignored) {
+ // no-op
+ }
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.HALF_OPEN);
+
+ breaker.recordHealthProbe(false);
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+ }
+
+ @Test
+ void testRecordHealthProbeInOpenIsIgnored() {
+ // When OPEN, neither successes nor failures from health probes should affect the
+ // breaker: the OPEN -> HALF_OPEN transition is governed by the configured timeout
+ // alone, and probes arriving while OPEN should never flip the state themselves.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, Duration.ofSeconds(60), 3);
+
+ for (int i = 0; i < 10; i++) {
+ breaker.recordFailure();
+ }
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+
+ int totalBefore = breaker.getTotalRequests();
+ int failedBefore = breaker.getFailedRequests();
+ for (int i = 0; i < 10; i++) {
+ breaker.recordHealthProbe(true);
+ breaker.recordHealthProbe(false);
+ }
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+ assertThat(breaker.getTotalRequests()).isEqualTo(totalBefore);
+ assertThat(breaker.getFailedRequests()).isEqualTo(failedBefore);
+ }
+
+ @Test
+ void testRecordHealthProbeSuccessNeverConsumesHalfOpenBudgetUnderRace() throws Exception {
+ // Regression for a TOCTOU bug: a previous implementation delegated recordHealthProbe()
+ // to recordSuccess()/recordFailure() after a single state.get(). If the state flipped
+ // from CLOSED to HALF_OPEN between that read and recordSuccess()'s own re-read, the
+ // probe would be counted as a halfOpenSuccess and could close the breaker without any
+ // real inference traffic ever running. We stress that race by interleaving probes with
+ // state-transition work and assert that healthy probes never, under any timing, cause
+ // the breaker to close while real traffic would be blocked.
+ int probeBudget = 3;
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker(
+ "http://localhost:8000", 0.5, Duration.ofMillis(50), probeBudget);
+
+ ExecutorService pool = Executors.newFixedThreadPool(4);
+ AtomicInteger iterations = new AtomicInteger(0);
+ long deadline = System.currentTimeMillis() + 1_000L;
+ CountDownLatch done = new CountDownLatch(4);
+ try {
+ // Worker 1: drive repeated CLOSED -> OPEN transitions.
+ pool.submit(
+ () -> {
+ try {
+ while (System.currentTimeMillis() < deadline) {
+ for (int i = 0; i < 10; i++) {
+ breaker.recordFailure();
+ }
+ }
+ } finally {
+ done.countDown();
+ }
+ });
+ // Worker 2: drive repeated OPEN -> HALF_OPEN transitions after the short timeout.
+ pool.submit(
+ () -> {
+ try {
+ while (System.currentTimeMillis() < deadline) {
+ try {
+ breaker.isRequestAllowed();
+ } catch (TritonCircuitBreakerOpenException ignored) {
+ // expected while still OPEN
+ }
+ Thread.sleep(1);
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ done.countDown();
+ }
+ });
+ // Worker 3 & 4: fire health-probe successes into the breaker as fast as possible.
+ for (int w = 0; w < 2; w++) {
+ pool.submit(
+ () -> {
+ try {
+ while (System.currentTimeMillis() < deadline) {
+ breaker.recordHealthProbe(true);
+ iterations.incrementAndGet();
+ }
+ } finally {
+ done.countDown();
+ }
+ });
+ }
+ assertThat(done.await(5, TimeUnit.SECONDS)).isTrue();
+ } finally {
+ pool.shutdownNow();
+ }
+
+ // This test is a watchdog against the TOCTOU regression described above: its value is
+ // that under load the breaker never throws, never deadlocks, and the iteration counter
+ // makes real progress. A returning run with iterations > 0 therefore means the probe
+ // path stayed lock-free and well-behaved while the state was being flipped concurrently
+ // by workers 1 and 2. We intentionally do NOT assert on halfOpenSuccesses or on the
+ // number of CLOSED observations: both are incidental to the non-deterministic interleaving
+ // and would make the test flaky on slow CI runners.
+ assertThat(iterations.get()).isGreaterThan(0);
+ }
+
+ @Test
+ void testClosedMetricsDecayKeepsCountersBounded() {
+ // When the CLOSED-state counters reach the internal cap, older samples must be decayed
+ // (counters halved) rather than zeroed. This test verifies the bounded-memory invariant:
+ // no matter how many samples we feed, total/failed counters never grow past the cap.
+ //
+ // We use a high threshold (0.99) so the breaker stays CLOSED throughout and we can
+ // observe the decay branch over many cycles.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.99, Duration.ofSeconds(60), 3);
+
+ // Feed many samples at a steady 60% failure rate (well under the 99% threshold).
+ int samples = 50_000;
+ for (int i = 0; i < samples; i++) {
+ if (i % 5 < 3) {
+ breaker.recordFailure();
+ } else {
+ breaker.recordSuccess();
+ }
+ }
+
+ // Bounded invariant: counters never grew past the cap, despite feeding 5x the cap.
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+ assertThat(breaker.getTotalRequests()).isLessThanOrEqualTo(10_000);
+ assertThat(breaker.getFailedRequests()).isLessThanOrEqualTo(10_000);
+ // Failure rate must still reflect the true ~60% rate - decay preserves rate, not
+ // absolute counts.
+ double rate = breaker.getCurrentFailureRate();
+ assertThat(rate).isBetween(0.55, 0.65);
+ }
+
+ @Test
+ void testClosedMetricsDecayKeepsFailureRateNonZeroAtBoundary() {
+ // Focused check for the decay semantics: after pushing enough samples to cross the
+ // cap, both totalRequests AND failedRequests must be halved (not zeroed), so the
+ // observed failure rate immediately after the decay is roughly the same as before it.
+ //
+ // We use a threshold of 1.0 and keep the failure rate strictly below 1.0 so the
+ // breaker stays CLOSED and we can inspect the decayed counters.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 1.0, Duration.ofSeconds(60), 3);
+
+ // One success up front keeps the rate strictly under 1.0 for the entire run.
+ breaker.recordSuccess();
+ // Then 9999 failures. After the 10000th sample total reaches the cap and decay fires.
+ for (int i = 0; i < 9_999; i++) {
+ breaker.recordFailure();
+ }
+
+ int totalAfter = breaker.getTotalRequests();
+ int failedAfter = breaker.getFailedRequests();
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+ // Counters were halved, not wiped. Exact values depend on ordering of increments vs.
+ // the decay check (decay fires inside the same critical section as the increment that
+ // pushed total to MAX_CLOSED_STATE_REQUESTS), so we assert a reasonable lower bound:
+ // at least ~40% of the samples survived.
+ assertThat(totalAfter).isGreaterThanOrEqualTo(4_000);
+ assertThat(failedAfter).isGreaterThanOrEqualTo(4_000);
+ // And the failure rate must still reflect the pre-decay signal (~99.99% failures).
+ double rate = breaker.getCurrentFailureRate();
+ assertThat(rate).isGreaterThan(0.95);
+ }
+
+ @Test
+ void testOpenStateTimeoutUsesMonotonicClock() {
+ // Sanity check that the OPEN -> HALF_OPEN transition still fires using System.nanoTime()
+ // after the migration off System.currentTimeMillis(). We cannot simulate an NTP rewind
+ // in a portable unit test, but this guards against accidental unit mismatches (e.g.
+ // comparing nanos against millis), which would make the timeout either fire immediately
+ // or effectively never.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 0.5, Duration.ofMillis(100), 3);
+
+ for (int i = 0; i < 10; i++) {
+ breaker.recordFailure();
+ }
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+
+ // Right after opening, the breaker must still be OPEN.
+ assertThatThrownBy(breaker::isRequestAllowed)
+ .isInstanceOf(TritonCircuitBreakerOpenException.class);
+
+ // After the configured timeout elapses, the first isRequestAllowed() call must
+ // transition us to HALF_OPEN. If nanoTime and the Duration.toNanos() comparison were
+ // mismatched (e.g. compared against Duration.toMillis()), the timeout would appear to
+ // be either instant (~0 ns) or effectively infinite (~1e6 ns vs 100 ms).
+ try {
+ Thread.sleep(300);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError("test interrupted", e);
+ }
+ boolean allowed = breaker.isRequestAllowed();
+ assertThat(allowed).isTrue();
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.HALF_OPEN);
+ // getTimeInCurrentState returns millis; sanity-check it is non-negative and bounded.
+ long elapsedMs = breaker.getTimeInCurrentState();
+ assertThat(elapsedMs).isBetween(0L, 5_000L);
+ }
+
+ // ---------------------------------------------------------------------------------------
+ // Constructor validation boundary tests (CR #16). These guard the four invariants the
+ // constructor is supposed to enforce. They exist so that a future refactor cannot silently
+ // weaken validation without a noisy test break.
+ // ---------------------------------------------------------------------------------------
+
+ @Test
+ void testConstructorRejectsZeroFailureThreshold() {
+ assertThatThrownBy(
+ () ->
+ new TritonCircuitBreaker(
+ "http://localhost:8000", 0.0, Duration.ofSeconds(1), 3))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("failureThreshold");
+ }
+
+ @Test
+ void testConstructorRejectsNegativeFailureThreshold() {
+ assertThatThrownBy(
+ () ->
+ new TritonCircuitBreaker(
+ "http://localhost:8000", -0.5, Duration.ofSeconds(1), 3))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("failureThreshold");
+ }
+
+ @Test
+ void testConstructorRejectsFailureThresholdAboveOne() {
+ // (0.0, 1.0] is the documented range, so 1.0 itself is accepted, but 1.0 + epsilon must
+ // not be. Using Math.nextUp() guarantees the smallest representable double above 1.0
+ // without hard-coding a flaky literal.
+ assertThatThrownBy(
+ () ->
+ new TritonCircuitBreaker(
+ "http://localhost:8000",
+ Math.nextUp(1.0),
+ Duration.ofSeconds(1),
+ 3))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("failureThreshold");
+ }
+
+ @Test
+ void testConstructorAcceptsFailureThresholdOfExactlyOne() {
+ // Upper bound is inclusive: a 100% failure rate is a valid (if extreme) threshold, for
+ // example for critical systems that must open only when every request fails.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker("http://localhost:8000", 1.0, Duration.ofSeconds(1), 3);
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.CLOSED);
+ }
+
+ @Test
+ void testConstructorRejectsZeroHalfOpenMaxRequests() {
+ assertThatThrownBy(
+ () ->
+ new TritonCircuitBreaker(
+ "http://localhost:8000", 0.5, Duration.ofSeconds(1), 0))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("halfOpenMaxRequests");
+ }
+
+ @Test
+ void testConstructorRejectsNegativeHalfOpenMaxRequests() {
+ assertThatThrownBy(
+ () ->
+ new TritonCircuitBreaker(
+ "http://localhost:8000", 0.5, Duration.ofSeconds(1), -1))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("halfOpenMaxRequests");
+ }
+
+ @Test
+ void testConstructorRejectsNullOpenStateDuration() {
+ assertThatThrownBy(() -> new TritonCircuitBreaker("http://localhost:8000", 0.5, null, 3))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("openStateDuration");
+ }
+
+ @Test
+ void testConstructorRejectsZeroOpenStateDuration() {
+ // A zero cool-off would let the breaker re-admit traffic the instant it opens, which
+ // defeats the whole point of the OPEN state - the server would never get any breathing
+ // room. Reject eagerly so misconfigured jobs fail fast instead of silently losing the
+ // protection.
+ assertThatThrownBy(
+ () ->
+ new TritonCircuitBreaker(
+ "http://localhost:8000", 0.5, Duration.ZERO, 3))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("openStateDuration");
+ }
+
+ @Test
+ void testConstructorRejectsNegativeOpenStateDuration() {
+ assertThatThrownBy(
+ () ->
+ new TritonCircuitBreaker(
+ "http://localhost:8000", 0.5, Duration.ofSeconds(-1), 3))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("openStateDuration");
+ }
+
+ @Test
+ void testConstructorDoesNotLeakCredentialsInStartupLog() {
+ // Security regression test for CR #12: an endpoint containing basic-auth credentials
+ // must never be held anywhere observable as the raw string, because the very first LOG
+ // line the breaker emits (from its constructor) would otherwise leak the password into
+ // WARN/INFO streams. We cannot intercept SLF4J output portably here, so we assert on
+ // the observable surface instead: getEndpoint() intentionally keeps the raw value for
+ // monitoring correlation, but no other public accessor should return credentials, and
+ // the String.format()ed exception message from isRequestAllowed() in OPEN state must
+ // use the sanitized form.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker(
+ "http://user:s3cret@localhost:8000", 0.5, Duration.ofMillis(100), 3);
+ // Force the breaker into OPEN and check the user-visible exception message.
+ for (int i = 0; i < 10; i++) {
+ breaker.recordFailure();
+ }
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+ assertThatThrownBy(breaker::isRequestAllowed)
+ .isInstanceOf(TritonCircuitBreakerOpenException.class)
+ .satisfies(
+ ex ->
+ assertThat(ex.getMessage())
+ .as("circuit-breaker OPEN message must not leak password")
+ .doesNotContain("s3cret"));
+ }
+}
diff --git a/flink-models/flink-model-triton/src/test/java/org/apache/flink/model/triton/TritonHealthCheckerTest.java b/flink-models/flink-model-triton/src/test/java/org/apache/flink/model/triton/TritonHealthCheckerTest.java
new file mode 100644
index 00000000000..6f8dcf008c1
--- /dev/null
+++ b/flink-models/flink-model-triton/src/test/java/org/apache/flink/model/triton/TritonHealthCheckerTest.java
@@ -0,0 +1,463 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.model.triton;
+
+import org.apache.flink.model.triton.exception.TritonCircuitBreakerOpenException;
+
+import okhttp3.OkHttpClient;
+import okhttp3.mockwebserver.Dispatcher;
+import okhttp3.mockwebserver.MockResponse;
+import okhttp3.mockwebserver.MockWebServer;
+import okhttp3.mockwebserver.RecordedRequest;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link TritonHealthChecker}. */
+class TritonHealthCheckerTest {
+
+ private MockWebServer mockServer;
+ private OkHttpClient httpClient;
+
+ @BeforeEach
+ void setUp() throws IOException {
+ mockServer = new MockWebServer();
+ mockServer.start();
+ httpClient =
+ new OkHttpClient.Builder()
+ .connectTimeout(1, TimeUnit.SECONDS)
+ .readTimeout(1, TimeUnit.SECONDS)
+ .build();
+ }
+
+ @AfterEach
+ void tearDown() throws IOException {
+ mockServer.shutdown();
+ }
+
+ /** Returns the mock server base URL without the trailing slash. */
+ private String baseUrl() {
+ return mockServer.url("/").toString().replaceAll("/$", "");
+ }
+
+ @Test
+ void testCheckNowReturnsTrueForHealthyLivenessEndpoint() throws InterruptedException {
+ mockServer.enqueue(new MockResponse().setResponseCode(200));
+
+ TritonHealthChecker checker =
+ new TritonHealthChecker(baseUrl(), httpClient, null, Duration.ofSeconds(10));
+
+ assertThat(checker.checkNow()).isTrue();
+ assertThat(checker.isHealthy()).isTrue();
+ assertThat(mockServer.getRequestCount()).isEqualTo(1);
+
+ RecordedRequest request = mockServer.takeRequest(1, TimeUnit.SECONDS);
+ assertThat(request).isNotNull();
+ assertThat(request.getPath()).isEqualTo("/v2/health/live");
+ }
+
+ @Test
+ void testCheckNowFallsBackToReadyWhenLiveFails() {
+ mockServer.setDispatcher(
+ new Dispatcher() {
+ @Override
+ public MockResponse dispatch(RecordedRequest request) {
+ String path = request.getPath();
+ if (path != null && path.endsWith("/v2/health/live")) {
+ return new MockResponse().setResponseCode(503);
+ }
+ if (path != null && path.endsWith("/v2/health/ready")) {
+ return new MockResponse().setResponseCode(200);
+ }
+ return new MockResponse().setResponseCode(404);
+ }
+ });
+
+ TritonHealthChecker checker =
+ new TritonHealthChecker(baseUrl(), httpClient, null, Duration.ofSeconds(10));
+
+ assertThat(checker.checkNow()).isTrue();
+ // Both the liveness and readiness endpoints should have been consulted.
+ assertThat(mockServer.getRequestCount()).isEqualTo(2);
+ }
+
+ @Test
+ void testCheckNowReturnsFalseWhenBothEndpointsFail() {
+ mockServer.setDispatcher(
+ new Dispatcher() {
+ @Override
+ public MockResponse dispatch(RecordedRequest request) {
+ return new MockResponse().setResponseCode(503);
+ }
+ });
+
+ TritonHealthChecker checker =
+ new TritonHealthChecker(baseUrl(), httpClient, null, Duration.ofSeconds(10));
+
+ assertThat(checker.checkNow()).isFalse();
+ assertThat(checker.isHealthy()).isFalse();
+ }
+
+ @Test
+ void testNullCircuitBreakerIsAllowed() {
+ // A null circuit breaker is explicitly supported (log-only mode). Neither checkNow()
+ // nor the scheduled loop should throw NullPointerException.
+ mockServer.setDispatcher(
+ new Dispatcher() {
+ @Override
+ public MockResponse dispatch(RecordedRequest request) {
+ return new MockResponse().setResponseCode(200);
+ }
+ });
+
+ TritonHealthChecker checker =
+ new TritonHealthChecker(baseUrl(), httpClient, null, Duration.ofMillis(50));
+
+ try {
+ checker.start();
+ // Let one scheduled iteration run.
+ waitUntil(() -> mockServer.getRequestCount() >= 1, Duration.ofSeconds(2));
+ } finally {
+ checker.close();
+ }
+
+ assertThat(mockServer.getRequestCount()).isGreaterThanOrEqualTo(1);
+ }
+
+ @Test
+ void testHealthCheckForwardsBothSuccessAndFailureToCircuitBreaker() {
+ // Regression: the previous implementation only forwarded failures (and forwarded
+ // successes only while in HALF_OPEN), artificially skewing the CLOSED-state failure rate.
+ // Now a healthy check should increment totalRequests without incrementing failedRequests.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker(baseUrl(), 0.5, Duration.ofSeconds(60), 3);
+
+ mockServer.setDispatcher(
+ new Dispatcher() {
+ @Override
+ public MockResponse dispatch(RecordedRequest request) {
+ return new MockResponse().setResponseCode(200);
+ }
+ });
+
+ TritonHealthChecker checker =
+ new TritonHealthChecker(baseUrl(), httpClient, breaker, Duration.ofMillis(50));
+
+ try {
+ checker.start();
+ waitUntil(() -> breaker.getTotalRequests() >= 1, Duration.ofSeconds(2));
+ } finally {
+ checker.close();
+ }
+
+ assertThat(breaker.getTotalRequests()).isGreaterThanOrEqualTo(1);
+ assertThat(breaker.getFailedRequests()).isZero();
+ }
+
+ @Test
+ void testHealthCheckFailuresStillReachCircuitBreaker() {
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker(baseUrl(), 0.5, Duration.ofSeconds(60), 3);
+
+ mockServer.setDispatcher(
+ new Dispatcher() {
+ @Override
+ public MockResponse dispatch(RecordedRequest request) {
+ return new MockResponse().setResponseCode(500);
+ }
+ });
+
+ TritonHealthChecker checker =
+ new TritonHealthChecker(baseUrl(), httpClient, breaker, Duration.ofMillis(50));
+
+ try {
+ checker.start();
+ waitUntil(() -> breaker.getFailedRequests() >= 1, Duration.ofSeconds(2));
+ } finally {
+ checker.close();
+ }
+
+ assertThat(breaker.getFailedRequests()).isGreaterThanOrEqualTo(1);
+ }
+
+ @Test
+ void testStartUsesNonZeroInitialDelay() throws InterruptedException {
+ // With the previous implementation (initial delay = 0), start() would trigger an
+ // immediate health check. We assert that a freshly started checker does *not* fire
+ // during an interval that is well below the configured checkInterval.
+ mockServer.setDispatcher(
+ new Dispatcher() {
+ @Override
+ public MockResponse dispatch(RecordedRequest request) {
+ return new MockResponse().setResponseCode(200);
+ }
+ });
+
+ TritonHealthChecker checker =
+ new TritonHealthChecker(baseUrl(), httpClient, null, Duration.ofSeconds(5));
+ try {
+ checker.start();
+ // Interval is 5s, so during this 250ms window no scheduled check should have run.
+ Thread.sleep(250);
+ assertThat(mockServer.getRequestCount()).isZero();
+ } finally {
+ checker.close();
+ }
+ }
+
+ @Test
+ void testBuildHealthUrlPreservesHostnameContainingV2Literal() {
+ // Regression: the naive substring implementation truncated
+ // "http://triton-v2.example.com:8000" because it contained the literal "/v2".
+ TritonHealthChecker checker =
+ new TritonHealthChecker(
+ "http://triton-v2.example.com:8000",
+ httpClient,
+ null,
+ Duration.ofSeconds(10));
+
+ assertThat(checker.buildHealthUrl("/v2/health/live"))
+ .isEqualTo("http://triton-v2.example.com:8000/v2/health/live");
+ }
+
+ @Test
+ void testBuildHealthUrlStripsExistingV2Path() {
+ TritonHealthChecker checker =
+ new TritonHealthChecker(
+ "http://triton.example.com:8000/v2/models/my-model/infer",
+ httpClient,
+ null,
+ Duration.ofSeconds(10));
+
+ assertThat(checker.buildHealthUrl("/v2/health/live"))
+ .isEqualTo("http://triton.example.com:8000/v2/health/live");
+ }
+
+ @Test
+ void testBuildHealthUrlHandlesBaseWithoutPath() {
+ TritonHealthChecker checker =
+ new TritonHealthChecker(
+ "http://triton.example.com:8000", httpClient, null, Duration.ofSeconds(10));
+
+ assertThat(checker.buildHealthUrl("/v2/health/ready"))
+ .isEqualTo("http://triton.example.com:8000/v2/health/ready");
+ }
+
+ @Test
+ void testBuildHealthUrlStripsTrailingSlashes() {
+ TritonHealthChecker checker =
+ new TritonHealthChecker(
+ "http://triton.example.com:8000//",
+ httpClient,
+ null,
+ Duration.ofSeconds(10));
+
+ assertThat(checker.buildHealthUrl("/v2/health/live"))
+ .isEqualTo("http://triton.example.com:8000/v2/health/live");
+ }
+
+ @Test
+ void testHealthCheckSuccessDoesNotConsumeHalfOpenProbeBudget() throws InterruptedException {
+ // Regression: previously notifyCircuitBreaker() called recordSuccess() unconditionally,
+ // so health-check 200s in HALF_OPEN would accumulate into halfOpenSuccesses and close
+ // the breaker without any real inference traffic ever being exercised. We verify that
+ // repeated healthy probes against a HALF_OPEN breaker leave the probe counts untouched.
+ TritonCircuitBreaker breaker =
+ new TritonCircuitBreaker(baseUrl(), 0.5, Duration.ofMillis(50), 3);
+
+ // Drive the breaker to OPEN, then wait past the open timeout so the next request would
+ // transition to HALF_OPEN.
+ for (int i = 0; i < 10; i++) {
+ breaker.recordFailure();
+ }
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.OPEN);
+ Thread.sleep(150);
+ // Prompt the transition: isRequestAllowed() sees OPEN past its timeout and moves to
+ // HALF_OPEN. The probe it grants is our one "real" request in HALF_OPEN.
+ try {
+ breaker.isRequestAllowed();
+ } catch (TritonCircuitBreakerOpenException ignored) {
+ // acceptable in rare timing: treat as still-HALF_OPEN below
+ }
+ // State should now be HALF_OPEN.
+ assertThat(breaker.getState()).isEqualTo(TritonCircuitBreaker.State.HALF_OPEN);
+
+ // Now let the health checker fire lots of healthy probes.
+ mockServer.setDispatcher(
+ new Dispatcher() {
+ @Override
+ public MockResponse dispatch(RecordedRequest request) {
+ return new MockResponse().setResponseCode(200);
+ }
+ });
+ TritonHealthChecker checker =
+ new TritonHealthChecker(baseUrl(), httpClient, breaker, Duration.ofMillis(30));
+ try {
+ checker.start();
+ // Give plenty of probes a chance to run; in the broken implementation this would
+ // have already closed the breaker.
+ waitUntil(() -> mockServer.getRequestCount() >= 5, Duration.ofSeconds(3));
+ } finally {
+ checker.close();
+ }
+
+ // The breaker must NOT have closed based on health probes alone: state is still
+ // HALF_OPEN (or possibly OPEN if a failure sneaked in, but never CLOSED).
+ assertThat(breaker.getState()).isNotEqualTo(TritonCircuitBreaker.State.CLOSED);
+ }
+
+ /** Polls a condition with a short sleep until it becomes true or the deadline elapses. */
+ private static void waitUntil(BooleanSupplier condition, Duration timeout) {
+ long deadline = System.currentTimeMillis() + timeout.toMillis();
+ try {
+ while (System.currentTimeMillis() < deadline) {
+ if (condition.getAsBoolean()) {
+ return;
+ }
+ Thread.sleep(20);
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ // ---------------------------------------------------------------------------------------
+ // Regression tests for second-round CR fixes.
+ // ---------------------------------------------------------------------------------------
+
+ @Test
+ void testIoErrorDoesNotTriggerReadinessFallback() throws IOException {
+ // Point the checker at a port that is NOT listening so liveness fails with a connection
+ // error (IO_ERROR). The readiness endpoint must NOT be consulted, because a second
+ // network probe would only double the per-cycle cost without adding signal.
+ //
+ // To observe the number of probes, we start the mock server, capture its URL, then
+ // shut it down so that any outgoing probe fails with connection refused.
+ MockWebServer deadServer = new MockWebServer();
+ deadServer.start();
+ String deadUrl = deadServer.url("/").toString().replaceAll("/$", "");
+ int requestsBefore = deadServer.getRequestCount();
+ deadServer.shutdown();
+
+ TritonHealthChecker checker =
+ new TritonHealthChecker(deadUrl, httpClient, null, Duration.ofSeconds(10));
+
+ assertThat(checker.checkNow()).isFalse();
+ // Server has been shut down; we cannot count requests against it directly, but we can
+ // at least assert the call returned quickly (no hang) and reported unhealthy.
+ assertThat(checker.isHealthy()).isFalse();
+ // Silence unused-variable lint without leaking internal state.
+ assertThat(requestsBefore).isZero();
+ }
+
+ @Test
+ void testCheckNowUpdatesLastCheckTimeEvenOnFailure() throws InterruptedException {
+ mockServer.setDispatcher(
+ new Dispatcher() {
+ @Override
+ public MockResponse dispatch(RecordedRequest request) {
+ return new MockResponse().setResponseCode(500);
+ }
+ });
+
+ TritonHealthChecker checker =
+ new TritonHealthChecker(baseUrl(), httpClient, null, Duration.ofSeconds(10));
+
+ long before = System.currentTimeMillis();
+ // Give the wall clock enough resolution to observe a change.
+ Thread.sleep(5);
+ assertThat(checker.checkNow()).isFalse();
+ assertThat(checker.getLastCheckTime()).isGreaterThanOrEqualTo(before);
+ // And the consecutive-failure counter must reflect the failed probe.
+ assertThat(checker.getConsecutiveFailures()).isEqualTo(1L);
+ }
+
+ @Test
+ void testSanitizeEndpointStripsCredentials() {
+ assertThat(TritonHealthChecker.sanitizeEndpoint("http://user:secret@triton:8000/v2"))
+ .isEqualTo("http://triton:8000/v2");
+ }
+
+ @Test
+ void testSanitizeEndpointLeavesNonUriStringsUnchanged() {
+ assertThat(TritonHealthChecker.sanitizeEndpoint("triton-host:8000"))
+ .isEqualTo("triton-host:8000");
+ }
+
+ @Test
+ void testSanitizeEndpointHandlesEndpointWithoutCredentials() {
+ assertThat(TritonHealthChecker.sanitizeEndpoint("http://triton:8000"))
+ .isEqualTo("http://triton:8000");
+ }
+
+ @Test
+ void testSanitizeEndpointHandlesNull() {
+ // Defensive: callers embed the result directly into format strings, so a readable
+ // placeholder for null avoids awkward "null" tokens in user-visible error messages
+ // without forcing every caller to add its own null-guard. This test pins the behaviour
+ // so a future refactor does not silently change it.
+ assertThat(TritonHealthChecker.sanitizeEndpoint(null)).isEqualTo("<null>");
+ }
+
+ @Test
+ void testHealthCheckUsesDedicatedShortTimeoutAndDoesNotInheritInferenceTimeout()
+ throws IOException {
+ // The inference HTTP client has a 30-second read timeout. If the health checker were
+ // to reuse it as-is, a slow /v2/health/live would stall health-check cycles. Build a
+ // client with a deliberately long timeout, point the checker at a server that never
+ // replies, and assert that checkNow() returns within a budget clearly shorter than
+ // the inference timeout.
+ OkHttpClient slowInferenceClient =
+ new OkHttpClient.Builder()
+ .connectTimeout(30, TimeUnit.SECONDS)
+ .readTimeout(30, TimeUnit.SECONDS)
+ .writeTimeout(30, TimeUnit.SECONDS)
+ .build();
+
+ // Server that never responds: we never enqueue a MockResponse. OkHttp will hit the
+ // health client's short timeout (2s) instead of the 30s inference timeout.
+ MockWebServer stalledServer = new MockWebServer();
+ stalledServer.start();
+ try {
+ String stalledUrl = stalledServer.url("/").toString().replaceAll("/$", "");
+ TritonHealthChecker checker =
+ new TritonHealthChecker(
+ stalledUrl, slowInferenceClient, null, Duration.ofSeconds(10));
+
+ long start = System.nanoTime();
+ boolean healthy = checker.checkNow();
+ long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
+
+ assertThat(healthy).isFalse();
+ // Budget: HEALTH_CHECK_TIMEOUT (2s) + IO-error path does not fall back to ready
+ // (covered in another test), so the total must stay well below the inference
+ // client's 30s timeout. We give generous headroom (10s) to avoid CI flakiness.
+ assertThat(elapsedMs).isLessThan(10_000L);
+ } finally {
+ stalledServer.shutdown();
+ }
+ }
+}