You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by mi...@apache.org on 2026/10/12 22:30:54 UTC
(superset) branch master updated: fix(mcp): promote tools whose full names appear in a search query (#45149)
This is an automated email from the ASF dual-hosted git repository.
aminghadersohi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 945d8bbd632 fix(mcp): promote tools whose full names appear in a search query (#45149)
945d8bbd632 is described below
commit 945d8bbd6324fd47a10a3b7a4327f32e37884386
Author: Amin Ghadersohi <am...@gmail.com>
AuthorDate: Tue Oct 13 08:30:46 2026 +1000
fix(mcp): promote tools whose full names appear in a search query (#45149)
---
docs/admin_docs/configuration/mcp-server.mdx | 9 +-
superset/mcp_service/server.py | 85 +++++++-
.../mcp_service/test_tool_search_transform.py | 225 ++++++++++++++++++++-
3 files changed, 304 insertions(+), 15 deletions(-)
diff --git a/docs/admin_docs/configuration/mcp-server.mdx b/docs/admin_docs/configuration/mcp-server.mdx
index 7df717ae1f5..dba1a682ecf 100644
--- a/docs/admin_docs/configuration/mcp-server.mdx
+++ b/docs/admin_docs/configuration/mcp-server.mdx
@@ -1154,8 +1154,13 @@ promoted ahead of other matches before the `max_results` limit is applied. Match
ignores case and treats underscores and whitespace as equivalent separators
(including leading, trailing, or repeated whitespace). Only tools visible to
the caller are eligible. Pinned `always_visible` tools remain in `list_tools`
-rather than search results; non-exact queries retain BM25 ranking or regex
-catalog ordering.
+rather than search results. BM25 also promotes multi-word tool names spelled out
+as a contiguous, in-order run in a longer query. These matches are ordered by
+name length (longest first), then by where the name starts in the query, then
+by BM25 rank. A name contained in another covered name is not promoted separately:
+`create dataset metric for revenue` promotes `create_dataset_metric`, not
+`create_dataset`. Other candidates retain BM25 ranking; regex searches retain
+catalog ordering for non-exact matches.
Tool search keeps the initial listing to the pinned tools and the search
interface; see the recorded inventory above for the size of the full native
diff --git a/superset/mcp_service/server.py b/superset/mcp_service/server.py
index c7305a22990..eb6019ac508 100644
--- a/superset/mcp_service/server.py
+++ b/superset/mcp_service/server.py
@@ -707,6 +707,15 @@ def _apply_tool_search_transform(mcp_instance: Any, config: dict[str, Any]) -> N
)
+def _name_words(text: str) -> list[str]:
+ """Split a tool name or query into lowercase words for name matching.
+
+ Mirrors the BM25 tokenizer: runs of letters and digits, dropping
+ single-character fragments.
+ """
+ return [word for word in re.split(r"[^a-z0-9]+", text.lower()) if len(word) > 1]
+
+
def _create_search_transform( # noqa: C901
*,
strategy: str,
@@ -742,13 +751,32 @@ def _create_search_transform( # noqa: C901
tool = Tool.from_function(fn=search_tools, name=transform._search_tool_name)
return _fix_search_tool_query(tool)
- def _promote_exact_name(
+ def _promote_name_matches(
tools: Sequence[Tool],
query: str,
ranked: Sequence[Tool],
max_results: int,
+ *,
+ cover_query_words: bool = False,
) -> Sequence[Tool]:
- """Promote caller-visible exact names without duplicating ranked matches."""
+ """Promote tools whose names the query spells out, within ``max_results``.
+
+ BM25 indexes a tool's name as ordinary text next to its description, so
+ a tool with a long definition can rank below the result limit even when
+ the query contains every word of its name. Two kinds of name match are
+ moved ahead of the upstream ranking:
+
+ 1. Exact names, which always come first.
+ 2. With ``cover_query_words`` (natural-language search only), multi-word
+ names whose words appear in the query as a contiguous, in-order run,
+ for example ``get_dataset_info`` for "get dataset info execute sql",
+ ordered by name length, query position, and then upstream rank.
+ Names nested inside another covered name are not promoted. Words
+ scattered across the query do not count.
+
+ Only caller-visible candidates are inspected, never the full catalog,
+ and the result never exceeds ``max_results``.
+ """
normalized_query = " ".join(query.casefold().replace("_", " ").split())
exact = [
tool
@@ -756,15 +784,48 @@ def _create_search_transform( # noqa: C901
if " ".join(tool.name.casefold().replace("_", " ").split())
== normalized_query
]
- if not exact:
- return ranked
- # Only inspect the caller-filtered candidates, never the full catalog.
- # The upstream top-N contains enough non-exact results to fill the
- # remaining slots, without changing upstream ordering or shared limits.
exact_names = {tool.name for tool in exact}
+ covered: list[Tool] = []
+ if cover_query_words:
+ query_words = _name_words(query)
+ ranked_position = {tool.name: index for index, tool in enumerate(ranked)}
+ matches: list[tuple[Tool, list[str], int]] = []
+ for tool in tools:
+ if tool.name in exact_names:
+ continue
+ words = _name_words(tool.name)
+ if len(words) <= 1:
+ continue
+ for run_start in range(len(query_words) - len(words) + 1):
+ if query_words[run_start : run_start + len(words)] == words:
+ matches.append((tool, words, run_start))
+ break
+ covered = [
+ tool
+ for tool, words, run_start in sorted(
+ matches,
+ key=lambda match: (
+ -len(match[1]),
+ match[2],
+ ranked_position.get(match[0].name, len(ranked_position)),
+ ),
+ )
+ if not any(
+ len(other_words) > len(words)
+ and any(
+ other_words[start : start + len(words)] == words
+ for start in range(len(other_words) - len(words) + 1)
+ )
+ for _, other_words, _ in matches
+ )
+ ]
+ promoted_names = exact_names | {tool.name for tool in covered}
+ if not promoted_names:
+ return ranked
return [
*exact,
- *(tool for tool in ranked if tool.name not in exact_names),
+ *covered,
+ *(tool for tool in ranked if tool.name not in promoted_names),
][:max_results]
if strategy == "regex":
@@ -783,7 +844,7 @@ def _create_search_transform( # noqa: C901
) -> Sequence[Tool]:
"""Promote visible exact names before applying the result limit."""
ranked = await super()._search(tools, query)
- return _promote_exact_name(tools, query, ranked, self._max_results)
+ return _promote_name_matches(tools, query, ranked, self._max_results)
def _make_call_tool(self) -> Any:
"""Build the normalized ``call_tool`` proxy for regex search."""
@@ -808,9 +869,11 @@ def _create_search_transform( # noqa: C901
return await _filter_visible_tools_fail_open(tools)
async def _search(self, tools: Sequence[Tool], query: str) -> Sequence[Tool]:
- """Promote visible exact names before applying the final result limit."""
+ """Promote visible exact names and contiguous name runs before the limit."""
ranked = await super()._search(tools, query)
- return _promote_exact_name(tools, query, ranked, self._max_results)
+ return _promote_name_matches(
+ tools, query, ranked, self._max_results, cover_query_words=True
+ )
def _make_call_tool(self) -> Any:
"""Build the normalized ``call_tool`` proxy for BM25 search."""
diff --git a/tests/unit_tests/mcp_service/test_tool_search_transform.py b/tests/unit_tests/mcp_service/test_tool_search_transform.py
index 05d38536e9b..9131930a1ae 100644
--- a/tests/unit_tests/mcp_service/test_tool_search_transform.py
+++ b/tests/unit_tests/mcp_service/test_tool_search_transform.py
@@ -1376,8 +1376,9 @@ def _exact_name_search(
*,
can_access: bool | Callable[[str, str], bool],
can_view_metadata: bool,
+ query_for: Callable[[str], str] = lambda name: name,
) -> tuple[dict[str, list[str]], set[str]]:
- """Search each tool's exact name through search_tools as one caller.
+ """Search each tool's name (or ``query_for(name)``) through search_tools.
Returns the ranked names for every query and the caller's visible names.
"""
@@ -1405,7 +1406,8 @@ def _exact_name_search(
search = transform._make_search_tool().fn
results = {
tool.name: [
- result["name"] for result in asyncio.run(search(query=tool.name))
+ result["name"]
+ for result in asyncio.run(search(query=query_for(tool.name)))
]
for tool in catalog
}
@@ -1440,6 +1442,203 @@ def test_bm25_exact_name_finds_every_registered_tool(
assert name not in results[name]
+def _production_search(
+ transform: BM25SearchTransform, catalog: list[Tool], query: str
+) -> list[str]:
+ """Rank the unpinned registered catalog the way search_tools does."""
+ pinned: set[str] = set(MCP_TOOL_SEARCH_CONFIG["always_visible"])
+ searchable = [tool for tool in catalog if tool.name not in pinned]
+ return [tool.name for tool in asyncio.run(transform._search(searchable, query))]
+
+
+@pytest.mark.parametrize(
+ "query, expected",
+ [
+ ("get dataset info execute sql", ["get_dataset_info", "execute_sql"]),
+ ("get chart info and list dashboards", ["get_chart_info", "list_dashboards"]),
+ ("generate chart and list dashboards", ["generate_chart", "list_dashboards"]),
+ ("get dataset info", ["get_dataset_info"]),
+ ("execute sql", ["execute_sql"]),
+ ("list dashboards", ["list_dashboards"]),
+ ],
+)
+def test_bm25_names_spelled_out_by_query_rank_first(
+ production_bm25_transform: BM25SearchTransform,
+ registered_catalog: list[Tool],
+ query: str,
+ expected: list[str],
+) -> None:
+ """Tools whose full names appear in the query lead the results.
+
+ The long ``get_dataset_info`` definition sat at the edge of the result
+ limit for "get dataset info execute sql" even though its name is spelled
+ out, so small catalog changes pushed it out entirely.
+ """
+ results = _production_search(production_bm25_transform, registered_catalog, query)
+
+ assert set(results[: len(expected)]) == set(expected)
+ assert len(results) <= MCP_TOOL_SEARCH_CONFIG["max_results"]
+ assert len(set(results)) == len(results)
+
+
+def test_bm25_multi_intent_query_stays_within_result_limit(
+ production_bm25_transform: BM25SearchTransform,
+ registered_catalog: list[Tool],
+) -> None:
+ """Name matches never exceed the limit, even when more tools qualify."""
+ query = (
+ "generate chart get chart info get dataset info get dashboard info "
+ "list dashboards execute sql"
+ )
+
+ results = _production_search(production_bm25_transform, registered_catalog, query)
+
+ assert len(results) == MCP_TOOL_SEARCH_CONFIG["max_results"]
+ assert len(set(results)) == len(results)
+
+
+@pytest.mark.parametrize("query", ["charts", "create charts", "generate", "no_match"])
+def test_bm25_name_coverage_ignores_partial_name_matches(
+ bm25_transform: BM25SearchTransform,
+ crowded_catalog: list[Tool],
+ query: str,
+) -> None:
+ """A query containing only part of a name does not promote that tool."""
+ expected = asyncio.run(
+ BM25SearchTransform(max_results=5)._search(crowded_catalog, query)
+ )
+ assert asyncio.run(bm25_transform._search(crowded_catalog, query)) == expected
+
+
+def test_bm25_name_coverage_ranks_longer_names_first(
+ bm25_transform: BM25SearchTransform,
+) -> None:
+ """Among covered names, more specific (longer) names come first."""
+ catalog = [
+ Tool.from_function(
+ lambda: None, name=name, description="Report details. " * count
+ )
+ for count, name in enumerate(
+ ["list_things", "get_thing_info_detail", "get_thing_info"], start=1
+ )
+ ] + [
+ Tool.from_function(
+ lambda: None, name=f"noise_{index}", description="get thing info " * 20
+ )
+ for index in range(8)
+ ]
+
+ results = asyncio.run(
+ bm25_transform._search(catalog, "get thing info detail list things")
+ )
+
+ assert [tool.name for tool in results[:2]] == [
+ "get_thing_info_detail",
+ "list_things",
+ ]
+ assert len(results) == 5
+
+
+@pytest.mark.parametrize("reverse_catalog", [False, True])
+def test_bm25_name_coverage_cut_follows_query_order(
+ bm25_transform: BM25SearchTransform,
+ reverse_catalog: bool,
+) -> None:
+ """Unranked covered names survive the cut by query order, not catalog order."""
+ names = [f"get_report_{index}" for index in range(10, 16)]
+ catalog = [Tool.from_function(lambda: None, name=name) for name in names]
+ if reverse_catalog:
+ catalog.reverse()
+ query = " ".join(name.replace("_", " ") for name in names)
+
+ with patch.object(BM25SearchTransform, "_search", AsyncMock(return_value=[])):
+ results = asyncio.run(bm25_transform._search(catalog, query))
+
+ assert [tool.name for tool in results] == names[:5]
+
+
+@pytest.mark.parametrize("reverse_catalog", [False, True])
+def test_bm25_multi_intent_cut_is_independent_of_catalog_order(
+ production_bm25_transform: BM25SearchTransform,
+ registered_catalog: list[Tool],
+ reverse_catalog: bool,
+) -> None:
+ """The registered catalog gives the same multi-intent cut in either order."""
+ catalog = (
+ list(reversed(registered_catalog)) if reverse_catalog else registered_catalog
+ )
+ query = (
+ "generate chart get chart info get dataset info get dashboard info "
+ "list dashboards execute sql"
+ )
+
+ assert _production_search(production_bm25_transform, catalog, query) == [
+ "get_chart_info",
+ "get_dataset_info",
+ "get_dashboard_info",
+ "generate_chart",
+ "list_dashboards",
+ ]
+
+
+def test_bm25_nested_name_does_not_displace_metrics(
+ production_bm25_transform: BM25SearchTransform,
+ registered_catalog: list[Tool],
+) -> None:
+ """A metric creation intent does not also promote dataset creation."""
+ query = "create dataset metric for revenue"
+ results = _production_search(production_bm25_transform, registered_catalog, query)
+
+ assert results[0] == "create_dataset_metric"
+ assert "list_metrics" in results
+ assert "create_dataset" not in results
+
+
+def test_regex_does_not_use_name_coverage(
+ crowded_catalog: list[Tool],
+) -> None:
+ """Regex search keeps pattern semantics; only BM25 expands query words."""
+ server = MagicMock()
+ _apply_tool_search_transform(
+ server,
+ {"strategy": "regex", "max_results": 5, "always_visible": []},
+ )
+ transform = server.add_transform.call_args[0][0]
+ query = "generate.*chart"
+ expected = asyncio.run(
+ RegexSearchTransform(max_results=5)._search(crowded_catalog, query)
+ )
+ assert asyncio.run(transform._search(crowded_catalog, query)) == expected
+
+
+@pytest.mark.parametrize(
+ "query, unexpected",
+ [
+ ("create a dashboard from a dataset", "create_dataset"),
+ ("list all the charts on a dashboard", "list_charts"),
+ ],
+)
+def test_bm25_name_coverage_ignores_scattered_name_words(
+ bm25_transform: BM25SearchTransform,
+ query: str,
+ unexpected: str,
+) -> None:
+ """Name words scattered across the query do not promote that tool."""
+ catalog = [
+ Tool.from_function(lambda: None, name=name, description="Report details. ")
+ for name in [unexpected, "create_dashboard", "list_dashboards"]
+ ] + [
+ Tool.from_function(
+ lambda: None, name=f"noise_{index}", description=f"{query} " * 20
+ )
+ for index in range(8)
+ ]
+
+ results = asyncio.run(bm25_transform._search(catalog, query))
+
+ assert results[0].name != unexpected
+
+
def _read_only_can_access(permission: str, _view: str) -> bool:
"""Allow only read and get permissions for the read-only test caller."""
return permission in {"can_read", "can_get"}
@@ -1475,6 +1674,28 @@ def test_bm25_exact_name_never_surfaces_unauthorized_registered_tools(
assert not_first == {}
+def test_bm25_name_coverage_respects_visibility(
+ production_bm25_transform: BM25SearchTransform,
+ registered_catalog: list[Tool],
+) -> None:
+ """A hidden tool's full name inside a longer query is never returned."""
+ results, visible = _exact_name_search(
+ production_bm25_transform,
+ registered_catalog,
+ can_access=_read_only_can_access,
+ can_view_metadata=False,
+ query_for=lambda name: f"please {name.replace('_', ' ')} for me",
+ )
+ denied: set[str] = {tool.name for tool in registered_catalog} - visible
+ assert "generate_chart" in denied
+
+ assert "generate_chart" not in results["generate_chart"]
+ leaked: dict[str, list[str]] = {
+ name: sorted(set(ranked) & denied) for name, ranked in results.items()
+ }
+ assert {name: hidden for name, hidden in leaked.items() if hidden} == {}
+
+
@pytest.fixture
def regex_transform() -> RegexSearchTransform:
"""Build the production transform with the default search result budget."""