You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2026/07/31 11:41:13 UTC
(shardingsphere) branch master updated: Add Mask planning scenario to MCP LLM E2E (#39289)
This is an automated email from the ASF dual-hosted git repository.
terrymanu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 8513c1d580f Add Mask planning scenario to MCP LLM E2E (#39289)
8513c1d580f is described below
commit 8513c1d580f7fc6a543ee631a53ee2df0030c522
Author: Liang Zhang <zh...@apache.org>
AuthorDate: Fri Jul 31 19:41:06 2026 +0800
Add Mask planning scenario to MCP LLM E2E (#39289)
* Add Mask planning scenario to MCP LLM E2E
* Add Mask planning scenario to MCP LLM E2E
---
.github/workflows/e2e-mcp.yml | 2 +
.../test/e2e/mcp/llm/LLMHttpE2ETest.java | 95 +++++++++++++++++++---
.../llm/conversation/LLMConversationRunner.java | 11 +--
.../conversation/LLMMCPToolDefinitionFactory.java | 32 +++-----
4 files changed, 103 insertions(+), 37 deletions(-)
diff --git a/.github/workflows/e2e-mcp.yml b/.github/workflows/e2e-mcp.yml
index 9c4b750cdad..ebf6e3e4c27 100644
--- a/.github/workflows/e2e-mcp.yml
+++ b/.github/workflows/e2e-mcp.yml
@@ -269,6 +269,8 @@ jobs:
test: LLMHttpE2ETest#assertReadOnlyQuery
- id: http-metadata-discovery
test: LLMHttpE2ETest#assertMetadataDiscovery
+ - id: http-mask-planning
+ test: LLMHttpE2ETest#assertMaskPlanning
- id: http-side-effect-preview
test: LLMHttpE2ETest#assertSideEffectPreview
- id: http-invalid-resource-recovery
diff --git a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/LLMHttpE2ETest.java b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/LLMHttpE2ETest.java
index c3f9e358c9b..3d9433e9042 100644
--- a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/LLMHttpE2ETest.java
+++ b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/LLMHttpE2ETest.java
@@ -48,6 +48,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Collection;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
@@ -74,14 +75,26 @@ class LLMHttpE2ETest extends AbstractConfigBackedRuntimeE2ETest {
private static final String STALE_TABLE_RESOURCE_URI = "shardingsphere://databases/logic_db/schemas/logic_db/tables/missing_orders";
+ private static final String READ_RESOURCE_TOOL_NAME = "mcp_read_resource";
+
+ private static final String SEARCH_METADATA_TOOL_NAME = "database_gateway_search_metadata";
+
+ private static final String EXECUTE_QUERY_TOOL_NAME = "database_gateway_execute_query";
+
+ private static final String EXECUTE_UPDATE_TOOL_NAME = "database_gateway_execute_update";
+
+ private static final String PLAN_MASK_RULE_TOOL_NAME = "database_gateway_plan_mask_rule";
+
private static final Set<String> EXPECTED_METADATA_NAMES = Set.of("active_orders", "order_items", "orders");
private static final List<String> ARTIFACT_FILES = List.of(
"run-context.json", "system-prompt.md", "question.txt", "answer.txt", "raw-model-output.txt", "available-tools.json",
"interaction-trace.json", "assertion-report.json");
+ // JSON boolean and null literals are metadata rather than credential values.
private static final Pattern UNREDACTED_SECRET_PATTERN = Pattern.compile(
- "(?i)(?<![a-z0-9_])\"?(?:api[_-]?key|access[_-]?token|token|authorization|password|passwd|pwd|secret)\"?\\s*[:=]\\s*[\"']?(?!<redacted>)[^\\s,\"'}]+"
+ "(?i)(?<![a-z0-9_])\"?(?:api[_-]?key|access[_-]?token|token|authorization|password|passwd|pwd|secret)\"?\\s*[:=]\\s*"
+ + "(?!(?:true|false|null)(?:\\s*[,}\\]]|\\s*$))[\"']?(?!<redacted>)[^\\s,\"'}]+"
+ "|(Bearer\\s+)(?!<redacted>)[A-Za-z0-9._~+/=-]+|jdbc:");
private static LLMRuntimeSupport.ModelRuntime llmRuntime;
@@ -116,7 +129,7 @@ class LLMHttpE2ETest extends AbstractConfigBackedRuntimeE2ETest {
runScenario(new Scenario(
"read-only-query",
"How many rows are currently in the orders table of the logic_db runtime database? Inspect the live MCP server and answer concisely.",
- false,
+ Set.of(EXECUTE_QUERY_TOOL_NAME),
this::evaluateReadOnlyQuery));
}
@@ -125,10 +138,22 @@ class LLMHttpE2ETest extends AbstractConfigBackedRuntimeE2ETest {
runScenario(new Scenario(
"metadata-discovery",
"List every table or view currently visible through the live MCP server. The user does not know the database or schema names, so discover the required scope first.",
- false,
+ Set.of(SEARCH_METADATA_TOOL_NAME),
this::evaluateMetadataDiscovery));
}
+ @Test
+ void assertMaskPlanning() throws IOException {
+ runScenario(new Scenario(
+ "mask-planning",
+ "The target metadata has already been verified, so do not run separate metadata discovery. Create a reviewable Mask rule plan, without applying it, "
+ + "for the status column of the orders table in the logic_db database and logic_db schema. "
+ + "Use the create operation, KEEP_FIRST_N_LAST_M algorithm, and primary properties first-n=1, last-m=1, replace-char=*. "
+ + "Report the plan ID and explicitly confirm that nothing was applied.",
+ Set.of(PLAN_MASK_RULE_TOOL_NAME),
+ this::evaluateMaskPlanning));
+ }
+
@Test
void assertSideEffectPreview() throws IOException, SQLException {
prepareRuntimeFixture();
@@ -137,7 +162,7 @@ class LLMHttpE2ETest extends AbstractConfigBackedRuntimeE2ETest {
"side-effect-preview",
"A user wants to change the status of order 1 in the orders table to REVIEW_PENDING. Preview the proposed change without executing it, then report whether it "
+ "was only previewed and whether any data changed. Inspect the live MCP server to discover any required runtime scope.",
- true,
+ Set.of(SEARCH_METADATA_TOOL_NAME, EXECUTE_UPDATE_TOOL_NAME),
(answer, trace) -> evaluateSideEffectPreview(answer, trace, statusBefore)));
}
@@ -147,7 +172,7 @@ class LLMHttpE2ETest extends AbstractConfigBackedRuntimeE2ETest {
"invalid-resource-recovery",
"A user pasted stale resource `" + STALE_TABLE_RESOURCE_URI + "`. Inspect that resource, then follow the first safe read-only action in its top-level "
+ "`next_actions` by reading its `resource_uri` exactly. Do not guess another URI. Then report how many rows are currently in the orders table.",
- false,
+ Set.of(READ_RESOURCE_TOOL_NAME, EXECUTE_QUERY_TOOL_NAME),
this::evaluateInvalidResourceRecovery));
}
@@ -189,7 +214,7 @@ class LLMHttpE2ETest extends AbstractConfigBackedRuntimeE2ETest {
Set<String> actualMetadataNames = new LinkedHashSet<>();
for (int index = unscopedSearchIndex; index < trace.size(); index++) {
MCPInteractionTraceRecord each = trace.get(index);
- if (isValidModelAction(each, "database_gateway_search_metadata") && (unscopedSearchIndex == index || each.getModelTurn() > discoveryTurn)) {
+ if (isValidModelAction(each, SEARCH_METADATA_TOOL_NAME) && (unscopedSearchIndex == index || each.getModelTurn() > discoveryTurn)) {
actualMetadataNames.addAll(getMetadataNames(each.getStructuredContent()));
}
}
@@ -202,9 +227,57 @@ class LLMHttpE2ETest extends AbstractConfigBackedRuntimeE2ETest {
: LLME2EAssertionReport.failure("answer_mismatch", "The answer omitted an object returned by the live MCP metadata searches.");
}
+ private LLME2EAssertionReport evaluateMaskPlanning(final String answer, final List<MCPInteractionTraceRecord> trace) {
+ Optional<MCPInteractionTraceRecord> plannedAction = trace.stream()
+ .filter(each -> isValidModelAction(each, PLAN_MASK_RULE_TOOL_NAME))
+ .filter(each -> "planned".equals(each.getStructuredContent().get("status")))
+ .findFirst();
+ if (plannedAction.isEmpty()) {
+ return LLME2EAssertionReport.failure("missing_mask_plan", "The model did not produce a planned Mask workflow.");
+ }
+ if (!hasExpectedMaskPlanArguments(trace)) {
+ return LLME2EAssertionReport.failure("mask_plan_arguments_mismatch", "The model did not send the requested Mask rule inputs.");
+ }
+ Map<String, Object> response = plannedAction.get().getStructuredContent();
+ String planId = Objects.toString(response.get("plan_id"), "").trim();
+ List<Map<String, Object>> distSQLArtifacts = getObjectList(response.get("distsql_artifacts"));
+ Map<String, Object> primaryProperties = getObjectMap(getObjectMap(response.get("masked_property_preview")).get("primary"));
+ if (planId.isEmpty() || !"mask.rule".equals(response.get("workflow_kind")) || !"review".equals(response.get("current_step"))
+ || !getList(response.get("missing_required_inputs")).isEmpty() || distSQLArtifacts.isEmpty()
+ || !Objects.toString(distSQLArtifacts.getFirst().get("sql"), "").startsWith("CREATE MASK RULE `orders`")
+ || !"1".equals(Objects.toString(primaryProperties.get("first-n"), ""))
+ || !"1".equals(Objects.toString(primaryProperties.get("last-m"), ""))
+ || !"*".equals(primaryProperties.get("replace-char"))) {
+ return LLME2EAssertionReport.failure("mask_plan_evidence_mismatch", "The MCP response did not contain the requested reviewable Mask plan.");
+ }
+ String normalizedAnswer = answer.toLowerCase(Locale.ENGLISH);
+ boolean answerReportsNoApplication = normalizedAnswer.contains("not applied") || normalizedAnswer.contains("not been applied")
+ || normalizedAnswer.contains("nothing was applied") || normalizedAnswer.contains("without applying")
+ || normalizedAnswer.contains("not executed") || normalizedAnswer.contains("no changes were applied");
+ return answer.contains(planId) && answerReportsNoApplication
+ ? LLME2EAssertionReport.success("The answer reported the live Mask plan ID and confirmed that it was not applied.")
+ : LLME2EAssertionReport.failure("answer_mismatch", "The answer did not report the Mask plan ID and confirm that it was not applied.");
+ }
+
+ private boolean hasExpectedMaskPlanArguments(final List<MCPInteractionTraceRecord> trace) {
+ Map<String, Object> arguments = new LinkedHashMap<>();
+ Map<String, Object> primaryProperties = new LinkedHashMap<>();
+ for (MCPInteractionTraceRecord each : trace) {
+ if (isValidModelAction(each, PLAN_MASK_RULE_TOOL_NAME)) {
+ arguments.putAll(each.getArguments());
+ primaryProperties.putAll(getObjectMap(each.getArguments().get("primary_algorithm_properties")));
+ }
+ }
+ return DATABASE_NAME.equals(arguments.get("database")) && TABLE_NAME.equals(arguments.get("table")) && "status".equals(arguments.get("column"))
+ && "create".equals(arguments.get("operation_type")) && "KEEP_FIRST_N_LAST_M".equals(arguments.get("algorithm_type"))
+ && "1".equals(Objects.toString(primaryProperties.get("first-n"), ""))
+ && "1".equals(Objects.toString(primaryProperties.get("last-m"), ""))
+ && "*".equals(primaryProperties.get("replace-char"));
+ }
+
private LLME2EAssertionReport evaluateSideEffectPreview(final String answer, final List<MCPInteractionTraceRecord> trace, final String statusBefore) {
Optional<MCPInteractionTraceRecord> preview = trace.stream()
- .filter(each -> isValidModelAction(each, "database_gateway_execute_update"))
+ .filter(each -> isValidModelAction(each, EXECUTE_UPDATE_TOOL_NAME))
.filter(each -> "preview".equals(each.getArguments().get("execution_mode")))
.filter(this::isSentinelPreview)
.findFirst();
@@ -258,7 +331,7 @@ class LLMHttpE2ETest extends AbstractConfigBackedRuntimeE2ETest {
private int findStaleResourceIndex(final List<MCPInteractionTraceRecord> trace) {
for (int index = 0; index < trace.size(); index++) {
MCPInteractionTraceRecord each = trace.get(index);
- if (isValidModelAction(each, "mcp_read_resource")
+ if (isValidModelAction(each, READ_RESOURCE_TOOL_NAME)
&& STALE_TABLE_RESOURCE_URI.equals(each.getArguments().get("uri"))
&& hasRecoveryCategory(each.getStructuredContent(), "object_not_visible")) {
return index;
@@ -280,7 +353,7 @@ class LLMHttpE2ETest extends AbstractConfigBackedRuntimeE2ETest {
for (int index = staleResourceIndex + 1; index < trace.size(); index++) {
MCPInteractionTraceRecord each = trace.get(index);
if (each.getModelTurn() > staleResource.getModelTurn()
- && isValidModelAction(each, "mcp_read_resource")
+ && isValidModelAction(each, READ_RESOURCE_TOOL_NAME)
&& Objects.equals(recoveryAction.get("resource_uri"), each.getArguments().get("uri"))
&& containsOrdersTable(each.getStructuredContent())) {
return index;
@@ -302,7 +375,7 @@ class LLMHttpE2ETest extends AbstractConfigBackedRuntimeE2ETest {
private Optional<Integer> findQueryCount(final List<MCPInteractionTraceRecord> trace, final int previousModelTurn) {
for (MCPInteractionTraceRecord each : trace) {
- if (each.getModelTurn() <= previousModelTurn || !isValidModelAction(each, "database_gateway_execute_query") || !isOrdersCountQuery(each.getArguments())) {
+ if (each.getModelTurn() <= previousModelTurn || !isValidModelAction(each, EXECUTE_QUERY_TOOL_NAME) || !isOrdersCountQuery(each.getArguments())) {
continue;
}
for (Map<String, Object> row : getObjectList(each.getStructuredContent().get("row_objects"))) {
@@ -341,7 +414,7 @@ class LLMHttpE2ETest extends AbstractConfigBackedRuntimeE2ETest {
private int findUnscopedMetadataSearchIndex(final List<MCPInteractionTraceRecord> trace) {
for (int index = 0; index < trace.size(); index++) {
MCPInteractionTraceRecord each = trace.get(index);
- if (isValidModelAction(each, "database_gateway_search_metadata")
+ if (isValidModelAction(each, SEARCH_METADATA_TOOL_NAME)
&& !each.getArguments().containsKey("database") && !each.getArguments().containsKey("schema")
&& getObjectList(each.getStructuredContent().get("items")).stream().anyMatch(item -> DATABASE_NAME.equals(item.get("database")))) {
return index;
diff --git a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMConversationRunner.java b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMConversationRunner.java
index 668aead231d..d880d695749 100644
--- a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMConversationRunner.java
+++ b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMConversationRunner.java
@@ -50,8 +50,9 @@ public final class LLMConversationRunner {
private static final String SYSTEM_PROMPT = """
You are evaluating a live ShardingSphere MCP server. Use the available MCP functions to inspect current state and complete the user's task.
Choose tools from their advertised names, descriptions, and input schemas. Pass resource URIs only to mcp_read_resource, never as SQL.
- Never guess or stop before retrieving the requested evidence. Preview side effects without executing them. Use function calls for every
- tool invocation rather than printing a tool-call object as the answer. When you have enough evidence, answer the user's question concisely.
+ Never guess or stop before retrieving the requested evidence. When target metadata is explicitly marked as already verified, use the requested
+ planning response as evidence instead of repeating metadata discovery. Preview side effects without executing them. Use function calls for
+ every tool invocation rather than printing a tool-call object as the answer. When you have enough evidence, answer the user's question concisely.
""";
private final int maxTurns;
@@ -88,7 +89,7 @@ public final class LLMConversationRunner {
try {
mcpInteractionClient.open();
List<Map<String, Object>> advertisedTools = mcpInteractionClient.listTools();
- List<Map<String, Object>> toolDefinitions = toolDefinitionFactory.createFromRemote(advertisedTools, scenario.allowsSideEffectPreview());
+ List<Map<String, Object>> toolDefinitions = toolDefinitionFactory.createFromRemote(advertisedTools, scenario.allowedToolNames());
artifacts.setToolDefinitions(toolDefinitions);
return runTurns(scenario, artifacts, toolDefinitions);
} catch (final IOException ex) {
@@ -224,10 +225,10 @@ public final class LLMConversationRunner {
*
* @param id scenario ID
* @param question question
- * @param allowsSideEffectPreview whether the scenario may expose update preview
+ * @param allowedToolNames tools exposed only to this scenario
* @param evaluator scenario evidence evaluator
*/
- public record Scenario(String id, String question, boolean allowsSideEffectPreview,
+ public record Scenario(String id, String question, Set<String> allowedToolNames,
BiFunction<String, List<MCPInteractionTraceRecord>, LLME2EAssertionReport> evaluator) {
}
diff --git a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMMCPToolDefinitionFactory.java b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMMCPToolDefinitionFactory.java
index fe2c12b3ea1..0e1e55d979a 100644
--- a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMMCPToolDefinitionFactory.java
+++ b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMMCPToolDefinitionFactory.java
@@ -17,45 +17,35 @@
package org.apache.shardingsphere.test.e2e.mcp.llm.conversation;
+import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
final class LLMMCPToolDefinitionFactory {
- private static final String EXECUTE_UPDATE_TOOL_NAME = "database_gateway_execute_update";
-
- List<Map<String, Object>> createFromRemote(final List<Map<String, Object>> advertisedTools, final boolean includeUpdatePreview) {
+ List<Map<String, Object>> createFromRemote(final List<Map<String, Object>> advertisedTools, final Set<String> allowedToolNames) {
List<Map<String, Object>> result = new LinkedList<>();
- result.add(createReadResourceToolDefinition());
- boolean readOnlyToolFound = false;
- boolean updatePreviewToolFound = false;
+ Set<String> missingAllowedToolNames = new LinkedHashSet<>(allowedToolNames);
+ if (missingAllowedToolNames.remove(LLMConversationRunner.READ_RESOURCE_TOOL_NAME)) {
+ result.add(createReadResourceToolDefinition());
+ }
for (Map<String, Object> each : advertisedTools) {
String toolName = Objects.toString(each.get("name"), "").trim();
- boolean readOnly = isReadOnly(each);
- boolean updatePreview = includeUpdatePreview && EXECUTE_UPDATE_TOOL_NAME.equals(toolName);
- if (!readOnly && !updatePreview) {
+ if (!missingAllowedToolNames.contains(toolName)) {
continue;
}
result.add(createRemoteToolDefinition(each, toolName));
- readOnlyToolFound |= readOnly;
- updatePreviewToolFound |= updatePreview;
- }
- if (!readOnlyToolFound) {
- throw new IllegalStateException("MCP runtime did not advertise any read-only tools.");
+ missingAllowedToolNames.remove(toolName);
}
- if (includeUpdatePreview && !updatePreviewToolFound) {
- throw new IllegalStateException("MCP runtime did not advertise required preview tool: " + EXECUTE_UPDATE_TOOL_NAME);
+ if (!missingAllowedToolNames.isEmpty()) {
+ throw new IllegalStateException("MCP runtime did not advertise required scenario tools: " + missingAllowedToolNames);
}
return result;
}
- private boolean isReadOnly(final Map<String, Object> advertisedTool) {
- Object annotations = advertisedTool.get("annotations");
- return annotations instanceof Map && Boolean.TRUE.equals(((Map<?, ?>) annotations).get("readOnlyHint"));
- }
-
private Map<String, Object> createRemoteToolDefinition(final Map<String, Object> advertisedTool, final String toolName) {
Object inputSchema = advertisedTool.get("inputSchema");
if (toolName.isEmpty() || !(inputSchema instanceof Map)) {