You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by mi...@apache.org on 2026/10/12 22:30:54 UTC

(superset) branch master updated: fix(mcp): promote tools whose full names appear in a search query (#45149)

This is an automated email from the ASF dual-hosted git repository.

aminghadersohi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git


The following commit(s) were added to refs/heads/master by this push:
     new 945d8bbd632 fix(mcp): promote tools whose full names appear in a search query (#45149)
945d8bbd632 is described below

commit 945d8bbd6324fd47a10a3b7a4327f32e37884386
Author: Amin Ghadersohi <am...@gmail.com>
AuthorDate: Tue Oct 13 08:30:46 2026 +1000

    fix(mcp): promote tools whose full names appear in a search query (#45149)
---
 docs/admin_docs/configuration/mcp-server.mdx       |   9 +-
 superset/mcp_service/server.py                     |  85 +++++++-
 .../mcp_service/test_tool_search_transform.py      | 225 ++++++++++++++++++++-
 3 files changed, 304 insertions(+), 15 deletions(-)

diff --git a/docs/admin_docs/configuration/mcp-server.mdx b/docs/admin_docs/configuration/mcp-server.mdx
index 7df717ae1f5..dba1a682ecf 100644
--- a/docs/admin_docs/configuration/mcp-server.mdx
+++ b/docs/admin_docs/configuration/mcp-server.mdx
@@ -1154,8 +1154,13 @@ promoted ahead of other matches before the `max_results` limit is applied. Match
 ignores case and treats underscores and whitespace as equivalent separators
 (including leading, trailing, or repeated whitespace). Only tools visible to
 the caller are eligible. Pinned `always_visible` tools remain in `list_tools`
-rather than search results; non-exact queries retain BM25 ranking or regex
-catalog ordering.
+rather than search results. BM25 also promotes multi-word tool names spelled out
+as a contiguous, in-order run in a longer query. These matches are ordered by
+name length (longest first), then by where the name starts in the query, then
+by BM25 rank. A name contained in another covered name is not promoted separately:
+`create dataset metric for revenue` promotes `create_dataset_metric`, not
+`create_dataset`. Other candidates retain BM25 ranking; regex searches retain
+catalog ordering for non-exact matches.
 
 Tool search keeps the initial listing to the pinned tools and the search
 interface; see the recorded inventory above for the size of the full native
diff --git a/superset/mcp_service/server.py b/superset/mcp_service/server.py
index c7305a22990..eb6019ac508 100644
--- a/superset/mcp_service/server.py
+++ b/superset/mcp_service/server.py
@@ -707,6 +707,15 @@ def _apply_tool_search_transform(mcp_instance: Any, config: dict[str, Any]) -> N
     )
 
 
+def _name_words(text: str) -> list[str]:
+    """Split a tool name or query into lowercase words for name matching.
+
+    Mirrors the BM25 tokenizer: runs of letters and digits, dropping
+    single-character fragments.
+    """
+    return [word for word in re.split(r"[^a-z0-9]+", text.lower()) if len(word) > 1]
+
+
 def _create_search_transform(  # noqa: C901
     *,
     strategy: str,
@@ -742,13 +751,32 @@ def _create_search_transform(  # noqa: C901
         tool = Tool.from_function(fn=search_tools, name=transform._search_tool_name)
         return _fix_search_tool_query(tool)
 
-    def _promote_exact_name(
+    def _promote_name_matches(
         tools: Sequence[Tool],
         query: str,
         ranked: Sequence[Tool],
         max_results: int,
+        *,
+        cover_query_words: bool = False,
     ) -> Sequence[Tool]:
-        """Promote caller-visible exact names without duplicating ranked matches."""
+        """Promote tools whose names the query spells out, within ``max_results``.
+
+        BM25 indexes a tool's name as ordinary text next to its description, so
+        a tool with a long definition can rank below the result limit even when
+        the query contains every word of its name. Two kinds of name match are
+        moved ahead of the upstream ranking:
+
+        1. Exact names, which always come first.
+        2. With ``cover_query_words`` (natural-language search only), multi-word
+           names whose words appear in the query as a contiguous, in-order run,
+           for example ``get_dataset_info`` for "get dataset info execute sql",
+           ordered by name length, query position, and then upstream rank.
+           Names nested inside another covered name are not promoted. Words
+           scattered across the query do not count.
+
+        Only caller-visible candidates are inspected, never the full catalog,
+        and the result never exceeds ``max_results``.
+        """
         normalized_query = " ".join(query.casefold().replace("_", " ").split())
         exact = [
             tool
@@ -756,15 +784,48 @@ def _create_search_transform(  # noqa: C901
             if " ".join(tool.name.casefold().replace("_", " ").split())
             == normalized_query
         ]
-        if not exact:
-            return ranked
-        # Only inspect the caller-filtered candidates, never the full catalog.
-        # The upstream top-N contains enough non-exact results to fill the
-        # remaining slots, without changing upstream ordering or shared limits.
         exact_names = {tool.name for tool in exact}
+        covered: list[Tool] = []
+        if cover_query_words:
+            query_words = _name_words(query)
+            ranked_position = {tool.name: index for index, tool in enumerate(ranked)}
+            matches: list[tuple[Tool, list[str], int]] = []
+            for tool in tools:
+                if tool.name in exact_names:
+                    continue
+                words = _name_words(tool.name)
+                if len(words) <= 1:
+                    continue
+                for run_start in range(len(query_words) - len(words) + 1):
+                    if query_words[run_start : run_start + len(words)] == words:
+                        matches.append((tool, words, run_start))
+                        break
+            covered = [
+                tool
+                for tool, words, run_start in sorted(
+                    matches,
+                    key=lambda match: (
+                        -len(match[1]),
+                        match[2],
+                        ranked_position.get(match[0].name, len(ranked_position)),
+                    ),
+                )
+                if not any(
+                    len(other_words) > len(words)
+                    and any(
+                        other_words[start : start + len(words)] == words
+                        for start in range(len(other_words) - len(words) + 1)
+                    )
+                    for _, other_words, _ in matches
+                )
+            ]
+        promoted_names = exact_names | {tool.name for tool in covered}
+        if not promoted_names:
+            return ranked
         return [
             *exact,
-            *(tool for tool in ranked if tool.name not in exact_names),
+            *covered,
+            *(tool for tool in ranked if tool.name not in promoted_names),
         ][:max_results]
 
     if strategy == "regex":
@@ -783,7 +844,7 @@ def _create_search_transform(  # noqa: C901
             ) -> Sequence[Tool]:
                 """Promote visible exact names before applying the result limit."""
                 ranked = await super()._search(tools, query)
-                return _promote_exact_name(tools, query, ranked, self._max_results)
+                return _promote_name_matches(tools, query, ranked, self._max_results)
 
             def _make_call_tool(self) -> Any:
                 """Build the normalized ``call_tool`` proxy for regex search."""
@@ -808,9 +869,11 @@ def _create_search_transform(  # noqa: C901
             return await _filter_visible_tools_fail_open(tools)
 
         async def _search(self, tools: Sequence[Tool], query: str) -> Sequence[Tool]:
-            """Promote visible exact names before applying the final result limit."""
+            """Promote visible exact names and contiguous name runs before the limit."""
             ranked = await super()._search(tools, query)
-            return _promote_exact_name(tools, query, ranked, self._max_results)
+            return _promote_name_matches(
+                tools, query, ranked, self._max_results, cover_query_words=True
+            )
 
         def _make_call_tool(self) -> Any:
             """Build the normalized ``call_tool`` proxy for BM25 search."""
diff --git a/tests/unit_tests/mcp_service/test_tool_search_transform.py b/tests/unit_tests/mcp_service/test_tool_search_transform.py
index 05d38536e9b..9131930a1ae 100644
--- a/tests/unit_tests/mcp_service/test_tool_search_transform.py
+++ b/tests/unit_tests/mcp_service/test_tool_search_transform.py
@@ -1376,8 +1376,9 @@ def _exact_name_search(
     *,
     can_access: bool | Callable[[str, str], bool],
     can_view_metadata: bool,
+    query_for: Callable[[str], str] = lambda name: name,
 ) -> tuple[dict[str, list[str]], set[str]]:
-    """Search each tool's exact name through search_tools as one caller.
+    """Search each tool's name (or ``query_for(name)``) through search_tools.
 
     Returns the ranked names for every query and the caller's visible names.
     """
@@ -1405,7 +1406,8 @@ def _exact_name_search(
         search = transform._make_search_tool().fn
         results = {
             tool.name: [
-                result["name"] for result in asyncio.run(search(query=tool.name))
+                result["name"]
+                for result in asyncio.run(search(query=query_for(tool.name)))
             ]
             for tool in catalog
         }
@@ -1440,6 +1442,203 @@ def test_bm25_exact_name_finds_every_registered_tool(
         assert name not in results[name]
 
 
+def _production_search(
+    transform: BM25SearchTransform, catalog: list[Tool], query: str
+) -> list[str]:
+    """Rank the unpinned registered catalog the way search_tools does."""
+    pinned: set[str] = set(MCP_TOOL_SEARCH_CONFIG["always_visible"])
+    searchable = [tool for tool in catalog if tool.name not in pinned]
+    return [tool.name for tool in asyncio.run(transform._search(searchable, query))]
+
+
+@pytest.mark.parametrize(
+    "query, expected",
+    [
+        ("get dataset info execute sql", ["get_dataset_info", "execute_sql"]),
+        ("get chart info and list dashboards", ["get_chart_info", "list_dashboards"]),
+        ("generate chart and list dashboards", ["generate_chart", "list_dashboards"]),
+        ("get dataset info", ["get_dataset_info"]),
+        ("execute sql", ["execute_sql"]),
+        ("list dashboards", ["list_dashboards"]),
+    ],
+)
+def test_bm25_names_spelled_out_by_query_rank_first(
+    production_bm25_transform: BM25SearchTransform,
+    registered_catalog: list[Tool],
+    query: str,
+    expected: list[str],
+) -> None:
+    """Tools whose full names appear in the query lead the results.
+
+    The long ``get_dataset_info`` definition sat at the edge of the result
+    limit for "get dataset info execute sql" even though its name is spelled
+    out, so small catalog changes pushed it out entirely.
+    """
+    results = _production_search(production_bm25_transform, registered_catalog, query)
+
+    assert set(results[: len(expected)]) == set(expected)
+    assert len(results) <= MCP_TOOL_SEARCH_CONFIG["max_results"]
+    assert len(set(results)) == len(results)
+
+
+def test_bm25_multi_intent_query_stays_within_result_limit(
+    production_bm25_transform: BM25SearchTransform,
+    registered_catalog: list[Tool],
+) -> None:
+    """Name matches never exceed the limit, even when more tools qualify."""
+    query = (
+        "generate chart get chart info get dataset info get dashboard info "
+        "list dashboards execute sql"
+    )
+
+    results = _production_search(production_bm25_transform, registered_catalog, query)
+
+    assert len(results) == MCP_TOOL_SEARCH_CONFIG["max_results"]
+    assert len(set(results)) == len(results)
+
+
+@pytest.mark.parametrize("query", ["charts", "create charts", "generate", "no_match"])
+def test_bm25_name_coverage_ignores_partial_name_matches(
+    bm25_transform: BM25SearchTransform,
+    crowded_catalog: list[Tool],
+    query: str,
+) -> None:
+    """A query containing only part of a name does not promote that tool."""
+    expected = asyncio.run(
+        BM25SearchTransform(max_results=5)._search(crowded_catalog, query)
+    )
+    assert asyncio.run(bm25_transform._search(crowded_catalog, query)) == expected
+
+
+def test_bm25_name_coverage_ranks_longer_names_first(
+    bm25_transform: BM25SearchTransform,
+) -> None:
+    """Among covered names, more specific (longer) names come first."""
+    catalog = [
+        Tool.from_function(
+            lambda: None, name=name, description="Report details. " * count
+        )
+        for count, name in enumerate(
+            ["list_things", "get_thing_info_detail", "get_thing_info"], start=1
+        )
+    ] + [
+        Tool.from_function(
+            lambda: None, name=f"noise_{index}", description="get thing info " * 20
+        )
+        for index in range(8)
+    ]
+
+    results = asyncio.run(
+        bm25_transform._search(catalog, "get thing info detail list things")
+    )
+
+    assert [tool.name for tool in results[:2]] == [
+        "get_thing_info_detail",
+        "list_things",
+    ]
+    assert len(results) == 5
+
+
+@pytest.mark.parametrize("reverse_catalog", [False, True])
+def test_bm25_name_coverage_cut_follows_query_order(
+    bm25_transform: BM25SearchTransform,
+    reverse_catalog: bool,
+) -> None:
+    """Unranked covered names survive the cut by query order, not catalog order."""
+    names = [f"get_report_{index}" for index in range(10, 16)]
+    catalog = [Tool.from_function(lambda: None, name=name) for name in names]
+    if reverse_catalog:
+        catalog.reverse()
+    query = " ".join(name.replace("_", " ") for name in names)
+
+    with patch.object(BM25SearchTransform, "_search", AsyncMock(return_value=[])):
+        results = asyncio.run(bm25_transform._search(catalog, query))
+
+    assert [tool.name for tool in results] == names[:5]
+
+
+@pytest.mark.parametrize("reverse_catalog", [False, True])
+def test_bm25_multi_intent_cut_is_independent_of_catalog_order(
+    production_bm25_transform: BM25SearchTransform,
+    registered_catalog: list[Tool],
+    reverse_catalog: bool,
+) -> None:
+    """The registered catalog gives the same multi-intent cut in either order."""
+    catalog = (
+        list(reversed(registered_catalog)) if reverse_catalog else registered_catalog
+    )
+    query = (
+        "generate chart get chart info get dataset info get dashboard info "
+        "list dashboards execute sql"
+    )
+
+    assert _production_search(production_bm25_transform, catalog, query) == [
+        "get_chart_info",
+        "get_dataset_info",
+        "get_dashboard_info",
+        "generate_chart",
+        "list_dashboards",
+    ]
+
+
+def test_bm25_nested_name_does_not_displace_metrics(
+    production_bm25_transform: BM25SearchTransform,
+    registered_catalog: list[Tool],
+) -> None:
+    """A metric creation intent does not also promote dataset creation."""
+    query = "create dataset metric for revenue"
+    results = _production_search(production_bm25_transform, registered_catalog, query)
+
+    assert results[0] == "create_dataset_metric"
+    assert "list_metrics" in results
+    assert "create_dataset" not in results
+
+
+def test_regex_does_not_use_name_coverage(
+    crowded_catalog: list[Tool],
+) -> None:
+    """Regex search keeps pattern semantics; only BM25 expands query words."""
+    server = MagicMock()
+    _apply_tool_search_transform(
+        server,
+        {"strategy": "regex", "max_results": 5, "always_visible": []},
+    )
+    transform = server.add_transform.call_args[0][0]
+    query = "generate.*chart"
+    expected = asyncio.run(
+        RegexSearchTransform(max_results=5)._search(crowded_catalog, query)
+    )
+    assert asyncio.run(transform._search(crowded_catalog, query)) == expected
+
+
+@pytest.mark.parametrize(
+    "query, unexpected",
+    [
+        ("create a dashboard from a dataset", "create_dataset"),
+        ("list all the charts on a dashboard", "list_charts"),
+    ],
+)
+def test_bm25_name_coverage_ignores_scattered_name_words(
+    bm25_transform: BM25SearchTransform,
+    query: str,
+    unexpected: str,
+) -> None:
+    """Name words scattered across the query do not promote that tool."""
+    catalog = [
+        Tool.from_function(lambda: None, name=name, description="Report details. ")
+        for name in [unexpected, "create_dashboard", "list_dashboards"]
+    ] + [
+        Tool.from_function(
+            lambda: None, name=f"noise_{index}", description=f"{query} " * 20
+        )
+        for index in range(8)
+    ]
+
+    results = asyncio.run(bm25_transform._search(catalog, query))
+
+    assert results[0].name != unexpected
+
+
 def _read_only_can_access(permission: str, _view: str) -> bool:
     """Allow only read and get permissions for the read-only test caller."""
     return permission in {"can_read", "can_get"}
@@ -1475,6 +1674,28 @@ def test_bm25_exact_name_never_surfaces_unauthorized_registered_tools(
     assert not_first == {}
 
 
+def test_bm25_name_coverage_respects_visibility(
+    production_bm25_transform: BM25SearchTransform,
+    registered_catalog: list[Tool],
+) -> None:
+    """A hidden tool's full name inside a longer query is never returned."""
+    results, visible = _exact_name_search(
+        production_bm25_transform,
+        registered_catalog,
+        can_access=_read_only_can_access,
+        can_view_metadata=False,
+        query_for=lambda name: f"please {name.replace('_', ' ')} for me",
+    )
+    denied: set[str] = {tool.name for tool in registered_catalog} - visible
+    assert "generate_chart" in denied
+
+    assert "generate_chart" not in results["generate_chart"]
+    leaked: dict[str, list[str]] = {
+        name: sorted(set(ranked) & denied) for name, ranked in results.items()
+    }
+    assert {name: hidden for name, hidden in leaked.items() if hidden} == {}
+
+
 @pytest.fixture
 def regex_transform() -> RegexSearchTransform:
     """Build the production transform with the default search result budget."""