You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2026/07/08 02:00:33 UTC
(shardingsphere) branch master updated: Improve MCP builder evaluation coverage (#39044)
This is an automated email from the ASF dual-hosted git repository.
terrymanu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 325b0c56874 Improve MCP builder evaluation coverage (#39044)
325b0c56874 is described below
commit 325b0c56874280030e30ca9efc97ada1b87961d5
Author: Liang Zhang <zh...@apache.org>
AuthorDate: Wed Jul 8 10:00:27 2026 +0800
Improve MCP builder evaluation coverage (#39044)
* Improve MCP builder evaluation coverage
Strengthen MCP builder evaluation guidance for feature planning and plugin
resource discovery. Add encrypt completion and tool-result redaction coverage,
and simplify completion context handling without changing behavior.
* Improve MCP builder evaluation coverage
Strengthen MCP builder evaluation guidance for feature planning and plugin
resource discovery. Add encrypt completion and tool-result redaction coverage,
and simplify completion context handling without changing behavior.
---
.../MCPCompletionSpecificationFactory.java | 3 +--
.../tool/MCPCallToolResultFactoryTest.java | 3 +++
.../metadata/query/DatabaseTestDataFactory.java | 6 +++---
.../llm/suite/MCPBuilderEvaluationArtifactTest.java | 11 +++++++++++
.../HttpTransportCompletionE2ETest.java | 2 ++
.../llm/evaluation/mcp-builder-evaluation.xml | 21 ++++++++++++---------
6 files changed, 32 insertions(+), 14 deletions(-)
diff --git a/mcp/bootstrap/src/main/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/completion/MCPCompletionSpecificationFactory.java b/mcp/bootstrap/src/main/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/completion/MCPCompletionSpecificationFactory.java
index 58a5c9947eb..01f7abcc00e 100644
--- a/mcp/bootstrap/src/main/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/completion/MCPCompletionSpecificationFactory.java
+++ b/mcp/bootstrap/src/main/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/completion/MCPCompletionSpecificationFactory.java
@@ -29,7 +29,6 @@ import org.apache.shardingsphere.mcp.support.descriptor.MCPCompletionTargetDescr
import org.apache.shardingsphere.mcp.support.descriptor.MCPDescriptorCatalogIndex;
import java.util.Collection;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -65,7 +64,7 @@ public final class MCPCompletionSpecificationFactory {
final MCPCompletionTargetDescriptor descriptor) {
String argumentName = request.argument().name();
String prefix = Objects.toString(request.argument().value(), "");
- Map<String, String> contextArguments = new LinkedHashMap<>(null == request.context() || null == request.context().arguments() ? Map.of() : request.context().arguments());
+ Map<String, String> contextArguments = null == request.context() || null == request.context().arguments() ? Map.of() : request.context().arguments();
try {
MCPCompletionResult result = completionService.complete(exchange.sessionId(), descriptor, argumentName, prefix, contextArguments);
return new McpSchema.CompleteResult(new McpSchema.CompleteResult.CompleteCompletion(result.getValues(), result.getTotal(), result.isHasMore()), result.getMeta());
diff --git a/mcp/bootstrap/src/test/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/tool/MCPCallToolResultFactoryTest.java b/mcp/bootstrap/src/test/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/tool/MCPCallToolResultFactoryTest.java
index 9841268eec8..ae0b1b029eb 100644
--- a/mcp/bootstrap/src/test/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/tool/MCPCallToolResultFactoryTest.java
+++ b/mcp/bootstrap/src/test/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/tool/MCPCallToolResultFactoryTest.java
@@ -120,6 +120,9 @@ class MCPCallToolResultFactoryTest extends AbstractMCPToolSpecificationFactoryTe
CallToolResult actual = createRealDescriptorCallToolResult(EncryptFeatureDefinition.PLAN_TOOL_NAME, createEncryptPlanResponse());
assertFalse(actual.isError(), () -> String.valueOf(actual.structuredContent()));
assertThat(getStructuredContent(actual).get("status"), is("planned"));
+ String actualText = ((TextContent) actual.content().getFirst()).text();
+ assertFalse(actualText.contains("123456"));
+ assertTrue(actualText.contains("******"));
}
@Test
diff --git a/mcp/support/src/test/java/org/apache/shardingsphere/mcp/support/database/metadata/query/DatabaseTestDataFactory.java b/mcp/support/src/test/java/org/apache/shardingsphere/mcp/support/database/metadata/query/DatabaseTestDataFactory.java
index 52b2e58b01f..19c73f03d26 100644
--- a/mcp/support/src/test/java/org/apache/shardingsphere/mcp/support/database/metadata/query/DatabaseTestDataFactory.java
+++ b/mcp/support/src/test/java/org/apache/shardingsphere/mcp/support/database/metadata/query/DatabaseTestDataFactory.java
@@ -52,7 +52,7 @@ final class DatabaseTestDataFactory {
static List<MCPDatabaseMetadata> createDatabaseMetadata() {
return List.of(
- new MCPDatabaseMetadata("logic_db", "FixtureDB", "", List.of(
+ new MCPDatabaseMetadata("logic_db", "MySQL", "", List.of(
new MCPSchemaMetadata("logic_db", "public", List.of(
new MCPTableMetadata("logic_db", "public", "orders",
List.of(new MCPColumnMetadata("logic_db", "public", "orders", "", "order_id")),
@@ -62,9 +62,9 @@ final class DatabaseTestDataFactory {
List.of(new MCPViewMetadata("logic_db", "public", "orders_view",
List.of(new MCPColumnMetadata("logic_db", "public", "", "orders_view", "order_id")))),
List.of()))),
- new MCPDatabaseMetadata("runtime_db", "FixtureRuntimeDB", "", List.of(
+ new MCPDatabaseMetadata("runtime_db", "PostgreSQL", "", List.of(
new MCPSchemaMetadata("runtime_db", "public", List.of(), List.of(), List.of(new MCPSequenceMetadata("runtime_db", "public", "order_seq"))))),
- new MCPDatabaseMetadata("warehouse", "FixtureWarehouseDB", "", List.of(
+ new MCPDatabaseMetadata("warehouse", "Hive", "", List.of(
new MCPSchemaMetadata("warehouse", "warehouse", List.of(new MCPTableMetadata("warehouse", "warehouse", "facts", List.of(), List.of())), List.of(), List.of()))));
}
diff --git a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/MCPBuilderEvaluationArtifactTest.java b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/MCPBuilderEvaluationArtifactTest.java
index 1c77c02cf4a..5c6f024cac9 100644
--- a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/MCPBuilderEvaluationArtifactTest.java
+++ b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/MCPBuilderEvaluationArtifactTest.java
@@ -102,6 +102,17 @@ final class MCPBuilderEvaluationArtifactTest {
NodeList qaPairs = root.getElementsByTagName("qa_pair");
assertThat(qaPairs.getLength(), is(EXPECTED_QA_PAIR_COUNT));
assertEvaluationQuestions(qaPairs);
+ String content = root.getTextContent();
+ for (String each : Set.of(
+ "database_gateway_plan_encrypt_rule",
+ "database_gateway_plan_mask_rule",
+ "database_gateway_plan_broadcast_rule",
+ "database_gateway_plan_readwrite_splitting_rule",
+ "database_gateway_plan_shadow_rule",
+ "database_gateway_plan_sharding_table_rule",
+ "completion/complete")) {
+ assertTrue(content.contains(each), () -> "Evaluation artifact must cover " + each);
+ }
}
@Test
diff --git a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/runtime/programmatic/HttpTransportCompletionE2ETest.java b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/runtime/programmatic/HttpTransportCompletionE2ETest.java
index b182584f9af..0cb6618201e 100644
--- a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/runtime/programmatic/HttpTransportCompletionE2ETest.java
+++ b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/runtime/programmatic/HttpTransportCompletionE2ETest.java
@@ -77,6 +77,8 @@ class HttpTransportCompletionE2ETest extends AbstractSharedHttpProgrammaticRunti
launchHttpTransport();
HttpClient httpClient = HttpClient.newHttpClient();
String sessionId = initializeSession(httpClient);
+ assertCompletionValuesContain("encrypt algorithm completion", httpClient, sessionId,
+ Map.of("type", "ref/prompt", "name", "plan_encrypt_rule"), "algorithm_type", "AES", "AES");
assertCompletionValuesContain("readwrite-splitting load balancer completion", httpClient, sessionId,
Map.of("type", "ref/prompt", "name", "plan_readwrite_splitting_rule"), "load_balancer_type", "ROUND", "ROUND_ROBIN");
assertCompletionValuesContain("shadow algorithm completion", httpClient, sessionId,
diff --git a/test/e2e/mcp/src/test/resources/llm/evaluation/mcp-builder-evaluation.xml b/test/e2e/mcp/src/test/resources/llm/evaluation/mcp-builder-evaluation.xml
index ae21d68efd1..ddaa93219ed 100644
--- a/test/e2e/mcp/src/test/resources/llm/evaluation/mcp-builder-evaluation.xml
+++ b/test/e2e/mcp/src/test/resources/llm/evaluation/mcp-builder-evaluation.xml
@@ -154,10 +154,12 @@
<answer>feature_resources|plan_feature_workflows|no_apply_execute</answer>
<expected_answer>
The model should first use tools/list, resources/templates/list, and resources/read to discover the feature resources for encrypt,
- broadcast, readwrite-splitting, shadow, and sharding instead of guessing tool names. It should call the matching database_gateway_plan_*
- workflow tool only after required logical database, rule, table, storage-unit, or algorithm inputs are known, then report plan_id,
- reviewable DistSQL artifacts, resources_to_read, and next_actions. It must not call database_gateway_apply_workflow in execute mode,
- execute SQL updates, create physical metadata, or perform migration while the request is still a read-only review.
+ broadcast, readwrite-splitting, shadow, and sharding instead of guessing tool names. It should call the matching planning tool only
+ after required logical database, rule, table, storage-unit, or algorithm inputs are known: database_gateway_plan_encrypt_rule,
+ database_gateway_plan_broadcast_rule, database_gateway_plan_readwrite_splitting_rule, database_gateway_plan_shadow_rule, or
+ database_gateway_plan_sharding_table_rule. The model should then report plan_id, reviewable DistSQL artifacts, resources_to_read,
+ and next_actions. It must not call database_gateway_apply_workflow in execute mode, execute SQL updates, create physical metadata,
+ or perform migration while the request is still a read-only review.
</expected_answer>
<verification>
<step>Confirm the answer discovers feature resources and tool schemas before selecting a planning workflow.</step>
@@ -174,11 +176,12 @@
<answer>completion_or_plugin_resources|bounded_candidates|return_next_actions</answer>
<expected_answer>
The model should use completion/complete or resources/read against the nearest shardingsphere:// plugin resource before choosing
- uncertain algorithm fields: mask algorithms, readwrite-splitting load-balance algorithms, shadow algorithm plugins, sharding algorithm
- plugins, and sharding key-generate algorithm plugins. It should treat returned values as bounded candidates, keep context arguments such
- as database or resource reference explicit, and avoid exposing secret property values as completion suggestions. If completion has no safe
- candidate, it should use the recovery message, resources_to_read, and next_actions from the planning response instead of inventing an
- algorithm or applying a workflow.
+ uncertain algorithm fields: shardingsphere://features/mask/algorithms,
+ shardingsphere://features/readwrite-splitting/load-balance-algorithm-plugins, shardingsphere://features/shadow/algorithm-plugins,
+ shardingsphere://features/sharding/algorithm-plugins, and shardingsphere://features/sharding/key-generate-algorithm-plugins.
+ It should treat returned values as bounded candidates, keep context arguments such as database or resource reference explicit, and avoid
+ exposing secret property values as completion suggestions. If completion has no safe candidate, it should use the recovery message,
+ resources_to_read, and next_actions from the planning response instead of inventing an algorithm or applying a workflow.
</expected_answer>
<verification>
<step>Confirm the answer uses completion/complete or resources/read plugin resources before selecting uncertain algorithm names.</step>