You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@juneau.apache.org by ja...@apache.org on 2026/08/04 03:35:18 UTC

(juneau) branch master updated: Fix intermittent MCP 2026-07-28 schema-validation timeout under load (TODO-328 follow-up)

This is an automated email from the ASF dual-hosted git repository.

jamesbognar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/juneau.git


The following commit(s) were added to refs/heads/master by this push:
     new e99d7f839b Fix intermittent MCP 2026-07-28 schema-validation timeout under load (TODO-328 follow-up)
e99d7f839b is described below

commit e99d7f839b56b56defda76d9275bd11719469fc9
Author: James Bognar <ja...@apache.org>
AuthorDate: Mon Aug 3 20:35:07 2026 -0700

    Fix intermittent MCP 2026-07-28 schema-validation timeout under load (TODO-328 follow-up)
    
    McpSchemaSafety was charging thread-pool scheduling/queue latency against the
    100ms compute-time DoS budget, causing spurious -32602 "exceeded 100 ms"
    failures under CI reactor load (surfaced as a Characterization_Test flake).
    The deadline is now anchored to the task's actual execution start rather than
    submission time, with a separate, much more generous MAX_SCHEDULING_MILLIS
    circuit breaker to still bound wedged/saturated-pool scheduling delay, plus
    pool prewarm to avoid paying thread-creation cost on the first validation.
    
    Co-authored-by: Cursor <cu...@cursor.com>
---
 .../rest/server/mcp/v20260728/McpSchemaSafety.java | 102 +++++++++++++++++----
 .../server/mcp/v20260728/McpSchemaSafety_Test.java |  32 +++++++
 2 files changed, 118 insertions(+), 16 deletions(-)

diff --git a/juneau-rest/juneau-rest-server-mcp-v20260728/src/main/java/org/apache/juneau/rest/server/mcp/v20260728/McpSchemaSafety.java b/juneau-rest/juneau-rest-server-mcp-v20260728/src/main/java/org/apache/juneau/rest/server/mcp/v20260728/McpSchemaSafety.java
index 29a1e949fa..d3c8dee7a1 100644
--- a/juneau-rest/juneau-rest-server-mcp-v20260728/src/main/java/org/apache/juneau/rest/server/mcp/v20260728/McpSchemaSafety.java
+++ b/juneau-rest/juneau-rest-server-mcp-v20260728/src/main/java/org/apache/juneau/rest/server/mcp/v20260728/McpSchemaSafety.java
@@ -20,6 +20,7 @@ import static java.util.concurrent.TimeUnit.*;
 
 import java.util.*;
 import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
 
 import org.apache.juneau.bean.jsonrpc.*;
 import org.apache.juneau.bean.jsonschema.*;
@@ -46,9 +47,14 @@ import org.apache.juneau.rest.server.mcp.McpSchema;
  * code path that opens a network connection or a file to dereference one.
  *
  * <p>
- * Validation itself runs on a fixed-size pool of daemon threads and is bounded by the same deadline: if a
- * pathological schema (for example a catastrophically-backtracking {@code pattern}) fails to complete in
- * time, the task is cancelled and a {@code -32602} error is raised instead of hanging the request thread.
+ * Validation itself runs on a fixed-size pool of daemon threads and is bounded by the same overall budget: if
+ * a pathological schema (for example a catastrophically-backtracking {@code pattern}) fails to complete in
+ * time, the task is cancelled and a {@code -32602} error is raised instead of hanging the request thread. That
+ * compute budget is measured from the moment the task actually starts running, not from the moment it is
+ * submitted: time spent waiting for a free thread in {@link #VALIDATION_POOL} (for example under heavy
+ * concurrent build/test load) is scheduling latency, not validation cost, so it is never charged against
+ * {@link #MAX_VALIDATION_MILLIS}. A separate, much more generous {@link #MAX_SCHEDULING_MILLIS} backstop still
+ * bounds the scheduling wait itself, purely so a wedged or saturated pool cannot block the caller forever.
  */
 final class McpSchemaSafety {
 
@@ -58,16 +64,38 @@ final class McpSchemaSafety {
 	/** Maximum number of nodes permitted in either the schema graph or the argument graph. */
 	static final int MAX_NODES = 10_000;
 
-	/** Maximum wall-clock time permitted for a single schema validation, in milliseconds. */
+	/**
+	 * Maximum compute time permitted for a single schema validation, in milliseconds, measured from the
+	 * moment the validation task actually starts running on {@link #VALIDATION_POOL} (not from submission).
+	 * This is the DoS bound: it caps how much CPU a pathological schema can burn, and is deliberately
+	 * unaffected by how long the task had to wait for a free thread.
+	 */
 	static final long MAX_VALIDATION_MILLIS = 100;
 
-	private static final ExecutorService VALIDATION_POOL = Executors.newFixedThreadPool(
-		Math.max(2, Runtime.getRuntime().availableProcessors()),
-		r -> {
-			var t = new Thread(r, "mcp-2026-07-28-schema-validation");
-			t.setDaemon(true);
-			return t;
-		});
+	/**
+	 * Maximum time permitted for a submitted validation task to begin executing on {@link #VALIDATION_POOL},
+	 * in milliseconds. This is a generous circuit breaker against a wedged or fully-saturated pool - not part
+	 * of the {@link #MAX_VALIDATION_MILLIS} DoS budget - so ordinary scheduling delay (queueing behind other
+	 * validations under heavy concurrent load) never gets mistaken for an expensive schema.
+	 */
+	static final long MAX_SCHEDULING_MILLIS = 2_000;
+
+	private static final ExecutorService VALIDATION_POOL = newValidationPool();
+
+	private static ExecutorService newValidationPool() {
+		var pool = new ThreadPoolExecutor(
+			Math.max(2, Runtime.getRuntime().availableProcessors()),
+			Math.max(2, Runtime.getRuntime().availableProcessors()),
+			0, MILLISECONDS,
+			new LinkedBlockingQueue<>(),
+			r -> {
+				var t = new Thread(r, "mcp-2026-07-28-schema-validation");
+				t.setDaemon(true);
+				return t;
+			});
+		pool.prestartAllCoreThreads(); // Warm the pool so the first validation after startup doesn't pay thread-creation cost.
+		return pool;
+	}
 
 	private McpSchemaSafety() {}
 
@@ -96,22 +124,60 @@ final class McpSchemaSafety {
 	}
 
 	/**
-	 * Runs {@link JsonSchemaValidator} on a daemon thread and enforces the remaining share of the
-	 * shared {@link JsonValueSafety} deadline.
+	 * Runs {@link JsonSchemaValidator} on a daemon thread and enforces the remaining share of the shared
+	 * {@link JsonValueSafety} deadline against the task's own compute time - never against however long it
+	 * had to wait in {@link #VALIDATION_POOL} for a free thread.
 	 */
 	private static void validateBounded(JsonSchema<?> schema, Object value, long deadlineNanos) {
 		var remaining = JsonValueSafety.remainingNanos(deadlineNanos);
 		if (remaining == 0)
-			throw new McpException(McpRevision.CODE_INVALID_PARAMS, "Tool input schema validation exceeded " + MAX_VALIDATION_MILLIS + " ms");
+			throw validationTimeoutException();
+		var started = new CountDownLatch(1);
+		var startedAtNanos = new AtomicLong();
 		var future = VALIDATION_POOL.submit(() -> {
+			startedAtNanos.set(System.nanoTime());
+			started.countDown();
 			JsonSchemaValidator.of(schema).validate(value);
 			return null;
 		});
+		awaitBounded(future, started, startedAtNanos, remaining);
+	}
+
+	/**
+	 * Waits for a submitted validation task to complete, charging only its own compute time - measured from
+	 * {@code startedAtNanos}, which the task sets as its first instruction - against {@code remaining}, the
+	 * caller's share of the shared {@link JsonValueSafety} deadline.
+	 *
+	 * <p>
+	 * This waits in two phases. First, {@code started} is awaited so the calling thread learns exactly when
+	 * the task begins running; this wait is bounded only by the generous {@link #MAX_SCHEDULING_MILLIS}
+	 * backstop, since scheduling delay under load (queueing behind other work in {@link #VALIDATION_POOL})
+	 * is not the thing being defended against. Second, once running, the task is given the full
+	 * {@code remaining} share of the deadline as its own fresh compute window - anchored to its actual start
+	 * time rather than to submission time - so a busy pool cannot eat into the budget that is meant to cap
+	 * the validator's own work.
+	 *
+	 * <p>
+	 * Package-private (rather than private) purely so this can be exercised directly against a test-local
+	 * {@link Future}/latch pair - deterministically simulating scheduling delay - without needing to saturate
+	 * the shared {@link #VALIDATION_POOL}.
+	 *
+	 * @param future The in-flight (or already-complete) validation task.
+	 * @param started Counted down by the task as its first instruction, once it actually begins running.
+	 * @param startedAtNanos Set by the task (before counting down {@code started}) to {@link System#nanoTime()}.
+	 * @param remaining The caller's remaining share of the shared deadline, in nanoseconds, captured before submission.
+	 */
+	static void awaitBounded(Future<?> future, CountDownLatch started, AtomicLong startedAtNanos, long remaining) {
 		try {
-			future.get(remaining, NANOSECONDS);
+			if (! started.await(MAX_SCHEDULING_MILLIS, MILLISECONDS)) {
+				future.cancel(true);
+				throw new McpException(McpRevision.CODE_INVALID_PARAMS, "Tool input schema validation could not be scheduled within " + MAX_SCHEDULING_MILLIS + " ms");
+			}
+			var computeRemainingNanos = remaining - (System.nanoTime() - startedAtNanos.get());
+			future.get(Math.max(0, computeRemainingNanos), NANOSECONDS);
 		} catch (TimeoutException e) {
 			future.cancel(true);
-			throw new McpException(McpRevision.CODE_INVALID_PARAMS, "Tool input schema validation exceeded " + MAX_VALIDATION_MILLIS + " ms");
+			throw validationTimeoutException();
 		} catch (ExecutionException e) {
 			var cause = e.getCause();
 			if (cause instanceof McpException me)
@@ -123,4 +189,8 @@ final class McpSchemaSafety {
 			throw new McpException(McpRevision.CODE_INVALID_PARAMS, "Tool input schema validation was interrupted");
 		}
 	}
+
+	private static McpException validationTimeoutException() {
+		return new McpException(McpRevision.CODE_INVALID_PARAMS, "Tool input schema validation exceeded " + MAX_VALIDATION_MILLIS + " ms");
+	}
 }
diff --git a/juneau-rest/juneau-rest-server-mcp-v20260728/src/test/java/org/apache/juneau/rest/server/mcp/v20260728/McpSchemaSafety_Test.java b/juneau-rest/juneau-rest-server-mcp-v20260728/src/test/java/org/apache/juneau/rest/server/mcp/v20260728/McpSchemaSafety_Test.java
index 7934f7e4d3..93eb846839 100644
--- a/juneau-rest/juneau-rest-server-mcp-v20260728/src/test/java/org/apache/juneau/rest/server/mcp/v20260728/McpSchemaSafety_Test.java
+++ b/juneau-rest/juneau-rest-server-mcp-v20260728/src/test/java/org/apache/juneau/rest/server/mcp/v20260728/McpSchemaSafety_Test.java
@@ -21,6 +21,8 @@ import static org.junit.jupiter.api.Assertions.*;
 
 import java.io.*;
 import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
 
 import org.apache.juneau.bean.jsonrpc.*;
 import org.apache.juneau.bean.jsonschema.*;
@@ -171,6 +173,36 @@ class McpSchemaSafety_Test {
 		assertTrue(elapsedMs < McpSchemaSafety.MAX_VALIDATION_MILLIS + 5000, () -> "validation did not terminate promptly: elapsed=" + elapsedMs + "ms");
 	}
 
+	@Test
+	void d02_schedulingLatency_notCountedAgainstComputeBudget() throws Exception {
+		// Exercises McpSchemaSafety.awaitBounded() directly (rather than saturating the shared
+		// VALIDATION_POOL, which other tests in this class can leave with a permanently-stuck thread since a
+		// catastrophically-backtracking regex match is not interruptible) with a task-local executor that
+		// deliberately delays counting down `started` well past MAX_VALIDATION_MILLIS before doing its
+		// (instantaneous) "work". If scheduling latency were - the regression this guards against - counted
+		// against the compute budget, awaitBounded() would throw a timeout error despite the task's own
+		// work costing nothing; with the root-cause fix, only compute time (measured from `started`) counts.
+		var schedulingDelayMillis = 3 * McpSchemaSafety.MAX_VALIDATION_MILLIS;
+		assertTrue(schedulingDelayMillis < McpSchemaSafety.MAX_SCHEDULING_MILLIS, "test fixture assumption");
+
+		var started = new CountDownLatch(1);
+		var startedAtNanos = new AtomicLong();
+		var executor = Executors.newSingleThreadExecutor();
+		try {
+			var future = executor.submit(() -> {
+				Thread.sleep(schedulingDelayMillis);  // simulates thread-pool queueing/scheduling delay
+				startedAtNanos.set(System.nanoTime());
+				started.countDown();
+				return null;  // the "validation" work itself is instantaneous
+			});
+
+			assertDoesNotThrow(() -> McpSchemaSafety.awaitBounded(
+				future, started, startedAtNanos, TimeUnit.MILLISECONDS.toNanos(McpSchemaSafety.MAX_VALIDATION_MILLIS)));
+		} finally {
+			executor.shutdownNow();
+		}
+	}
+
 	// -------- shared JsonValueSafety delegation now supports arrays ---------
 
 	@Test