You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by mi...@apache.org on 2026/07/22 03:49:58 UTC
(superset) 04/04: fix(mcp): sync per-statement row_count after execute_sql truncation
This is an automated email from the ASF dual-hosted git repository.
aminghadersohi pushed a commit to branch aminghadersohi/mcp-truncate-query-responses
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 00c5e7957b89daeba9a67f5448a70bbe76840a8b
Author: Amin Ghadersohi <am...@gmail.com>
AuthorDate: Wed Jul 22 03:49:17 2026 +0000
fix(mcp): sync per-statement row_count after execute_sql truncation
_bisect_row_limit already trims statements[*].data.rows in lockstep
with the top-level rows field to keep the response under the token
limit, but left each statement's row_count pointing at the
pre-truncation count, so a truncated execute_sql response could claim
more rows than statements[*].data actually contains. Track the
statement dicts (not just their row lists) and update row_count
alongside data.rows.
Also log a full traceback (logger.exception) instead of a plain
warning when data-query truncation hits an unexpected error, so a
real bug isn't indistinguishable from routine size-limit enforcement.
Adds regression tests for: total_rows surviving truncation distinct
from row_count, the no-recognised-row-field fallback path, and the
oversized-excel-export hard-failure path.
---
superset/mcp_service/middleware.py | 9 ++-
superset/mcp_service/utils/token_utils.py | 41 +++++++-----
tests/unit_tests/mcp_service/test_middleware.py | 72 ++++++++++++++++++++++
.../mcp_service/utils/test_token_utils.py | 59 ++++++++++++++++++
4 files changed, 164 insertions(+), 17 deletions(-)
diff --git a/superset/mcp_service/middleware.py b/superset/mcp_service/middleware.py
index 8b31335b8b5..ca308db9464 100644
--- a/superset/mcp_service/middleware.py
+++ b/superset/mcp_service/middleware.py
@@ -1382,7 +1382,14 @@ class ResponseSizeGuardMiddleware(Middleware):
size_fn=size_fn,
)
except Exception as trunc_error: # noqa: BLE001
- logger.warning(
+ # Deliberately broad: any failure here must fail safe into the
+ # hard size-limit error below rather than ship a broken
+ # truncation, so this stays a catch-all rather than narrowing to
+ # specific exception types. Logged with a full traceback (unlike
+ # the plain warning above) so a genuine bug (e.g. a KeyError from
+ # an unexpected response shape) is still visible instead of
+ # looking like routine size-limit enforcement.
+ logger.exception(
"Query result truncation failed for %s due to %s: %s",
tool_name,
type(trunc_error).__name__,
diff --git a/superset/mcp_service/utils/token_utils.py b/superset/mcp_service/utils/token_utils.py
index 91fdda3fe09..63f079dd1bf 100644
--- a/superset/mcp_service/utils/token_utils.py
+++ b/superset/mcp_service/utils/token_utils.py
@@ -698,8 +698,9 @@ def truncate_oversized_response(
return data, was_truncated, notes
-def _linked_statement_row_lists(data: Dict[str, Any]) -> list[list[Any]]:
- """Find per-statement row lists duplicated from an ``execute_sql`` response.
+def _linked_statement_entries(data: Dict[str, Any]) -> list[Dict[str, Any]]:
+ """Find per-statement dicts with row data duplicated from an
+ ``execute_sql`` response's top-level ``rows`` field.
``ExecuteSqlResponse.rows`` is a copy of the last data-bearing
statement's ``data.rows`` (see ``_convert_to_response`` in
@@ -707,8 +708,11 @@ def _linked_statement_row_lists(data: Dict[str, Any]) -> list[list[Any]]:
rows are additionally serialised under ``statements[*].data.rows``. If
only the top-level ``rows`` field is bisected, these nested copies keep
the full untruncated payload, which can leave the response oversized
- even after "truncation". Returns the mutable row lists (if any) so the
- caller can shrink them in lockstep with the top-level field.
+ even after "truncation". Returns the statement dicts themselves (not
+ just their row lists) so the caller can shrink ``data.rows`` in
+ lockstep with the top-level field *and* keep each statement's
+ ``row_count`` in sync, rather than leaving it pointing at the
+ pre-truncation count.
"""
statements = data.get("statements")
if not isinstance(statements, list):
@@ -722,7 +726,7 @@ def _linked_statement_row_lists(data: Dict[str, Any]) -> list[list[Any]]:
if isinstance(statement_data, dict) and isinstance(
statement_data.get("rows"), list
):
- linked.append(statement_data["rows"])
+ linked.append(statement)
return linked
@@ -741,23 +745,27 @@ def _bisect_row_limit(
When ``data`` is an ``execute_sql`` response, per-statement row lists
under ``statements[*].data.rows`` duplicate ``data[row_field]`` (see
- ``_linked_statement_row_lists``) and are shrunk to the same length at
- each step, so the size measured during the search — and the response
- ultimately returned — reflects the truncated nested data too, not just
- the top-level field.
+ ``_linked_statement_entries``) and are shrunk to the same length at
+ each step — along with each statement's ``row_count``, so it doesn't
+ keep reporting the pre-truncation count — so the size measured during
+ the search, and the response ultimately returned, reflects the
+ truncated nested data too, not just the top-level field.
``size_fn`` measures each candidate; pass a function that re-wraps the
payload (e.g. back into a ToolResult) before measuring so the search
converges on a prefix that still fits the limit after re-wrapping.
"""
- linked_rows = [list(rows) for rows in _linked_statement_row_lists(data)]
+ linked_statements = _linked_statement_entries(data)
+ linked_rows = [list(stmt["data"]["rows"]) for stmt in linked_statements]
def _apply(count: int) -> None:
data[row_field] = original_rows[:count]
- for original_linked, mutable_linked in zip(
- linked_rows, _linked_statement_row_lists(data), strict=False
+ for original_linked, statement in zip(
+ linked_rows, linked_statements, strict=False
):
- mutable_linked[:] = original_linked[: min(count, len(original_linked))]
+ kept_linked = original_linked[: min(count, len(original_linked))]
+ statement["data"]["rows"] = kept_linked
+ statement["row_count"] = len(kept_linked)
lo, hi = 0, len(original_rows)
while lo < hi:
@@ -973,9 +981,10 @@ def truncate_query_result(
rely on ``_truncation_notes`` and the true ``total_rows`` to catch this.
For ``execute_sql``, per-statement row lists under
- ``statements[*].data.rows`` are shrunk alongside the top-level field
- (see ``_linked_statement_row_lists``) so a duplicate, untruncated copy
- of the same rows can't leave the response oversized after "truncation".
+ ``statements[*].data.rows`` (and their ``row_count``) are shrunk
+ alongside the top-level field (see ``_linked_statement_entries``) so a
+ duplicate, untruncated copy of the same rows can't leave the response
+ oversized, or its metadata inconsistent, after "truncation".
Args:
response: The tool response containing tabular row data.
diff --git a/tests/unit_tests/mcp_service/test_middleware.py b/tests/unit_tests/mcp_service/test_middleware.py
index 705f4c5e864..61ffccbb116 100644
--- a/tests/unit_tests/mcp_service/test_middleware.py
+++ b/tests/unit_tests/mcp_service/test_middleware.py
@@ -582,6 +582,78 @@ class TestResponseSizeGuardMiddleware:
assert result["_response_truncated"] is True
assert len(result["csv_data"]) < len(large_response["csv_data"])
+ @pytest.mark.asyncio
+ async def test_data_query_truncation_preserves_total_rows(self) -> None:
+ """``total_rows`` (the true pre-truncation count) must survive.
+
+ ``row_count`` is overwritten to reflect the truncated row list, but
+ ``total_rows`` is the signal callers rely on to detect that a
+ response is partial — it must keep describing the original,
+ untruncated dataset.
+ """
+ middleware = ResponseSizeGuardMiddleware(token_limit=500)
+
+ context = MagicMock()
+ context.message.name = "query_dataset"
+ context.message.params = {}
+
+ row = {f"col_{i}": f"value_{i}" for i in range(10)}
+ large_response = {
+ "dataset_id": 1,
+ "dataset_name": "test",
+ "data": [row] * 200,
+ "row_count": 200,
+ "total_rows": 5000,
+ "summary": "",
+ }
+ call_next = AsyncMock(return_value=large_response)
+
+ with (
+ patch("superset.mcp_service.middleware.get_user_id", return_value=1),
+ patch("superset.mcp_service.middleware.event_logger"),
+ ):
+ result = await middleware.on_call_tool(context, call_next)
+
+ assert result["_response_truncated"] is True
+ assert result["row_count"] == len(result["data"])
+ assert result["row_count"] < 200
+ assert result["total_rows"] == 5000
+
+ @pytest.mark.asyncio
+ async def test_get_chart_data_excel_export_raises_tool_error(self) -> None:
+ """Oversized excel exports must hard-fail, not ship a corrupt file.
+
+ ``excel_data`` is base64-encoded binary and is deliberately left
+ untouched by truncation (cutting it would produce an unreadable
+ file), and there are no other truncatable fields (``data=[]``), so
+ ``truncate_query_result`` reports ``was_truncated=False`` and the
+ response must fall through to the hard size-limit error.
+ """
+ middleware = ResponseSizeGuardMiddleware(token_limit=500)
+
+ context = MagicMock()
+ context.message.name = "get_chart_data"
+ context.message.params = {}
+
+ large_response: dict[str, Any] = {
+ "chart_id": 1,
+ "chart_name": "test chart",
+ "chart_type": "table",
+ "data": [],
+ "row_count": 200,
+ "summary": "",
+ "excel_data": "QUJDREVGRw==" * 5000,
+ "format": "excel",
+ }
+ call_next = AsyncMock(return_value=large_response)
+
+ with (
+ patch("superset.mcp_service.middleware.get_user_id", return_value=1),
+ patch("superset.mcp_service.middleware.event_logger"),
+ pytest.raises(ToolError),
+ ):
+ await middleware.on_call_tool(context, call_next)
+
@pytest.mark.asyncio
async def test_data_query_blocks_when_single_row_still_exceeds_limit(self) -> None:
"""Should raise ToolError, not ship an over-budget response.
diff --git a/tests/unit_tests/mcp_service/utils/test_token_utils.py b/tests/unit_tests/mcp_service/utils/test_token_utils.py
index e8f7d864496..bce195a1ab0 100644
--- a/tests/unit_tests/mcp_service/utils/test_token_utils.py
+++ b/tests/unit_tests/mcp_service/utils/test_token_utils.py
@@ -934,6 +934,65 @@ class TestTruncateQueryResult:
assert len(result["statements"][0]["data"]["rows"]) == len(result["rows"])
assert notes
+ def test_execute_sql_statements_row_count_stays_in_sync(self) -> None:
+ """Each statement's ``row_count`` must track its trimmed ``data.rows``.
+
+ Regression test: bisection already shrinks the nested
+ ``statements[*].data.rows`` list in lockstep with the top-level
+ field (see the sibling ``..._rows_truncated_alongside_top_level``
+ test), but the per-statement ``row_count`` field is a separate
+ value copied at response-build time and must be updated too, or a
+ client would see a statement whose declared row count doesn't match
+ the row list actually returned.
+ """
+ row = {f"col_{i}": f"value_{i}" for i in range(10)}
+ rows = [row] * 200
+ response: dict[str, Any] = {
+ "success": True,
+ "rows": rows,
+ "row_count": 200,
+ "statements": [
+ {
+ "original_sql": "SELECT * FROM t",
+ "executed_sql": "SELECT * FROM t",
+ "row_count": 200,
+ "data": {
+ "rows": rows,
+ "columns": [
+ {"name": f"col_{i}", "type": "TEXT"} for i in range(10)
+ ],
+ },
+ }
+ ],
+ }
+ result, was_truncated, _ = truncate_query_result(
+ response, 500, tool_name="execute_sql"
+ )
+ assert was_truncated is True
+ statement = result["statements"][0]
+ assert statement["row_count"] == len(statement["data"]["rows"])
+ assert statement["row_count"] < 200
+
+ def test_falls_back_to_generic_truncation_when_no_row_field(self) -> None:
+ """Responses without a recognised row/data field truncate generically.
+
+ A DML-final ``execute_sql`` response (or any data-query response
+ with no ``rows``/``data`` list) has no row field to bisect on, so
+ ``truncate_query_result`` delegates to the generic
+ ``truncate_oversized_response`` field-truncation phases instead of
+ raising or returning the payload untouched.
+ """
+ response: dict[str, Any] = {
+ "success": True,
+ "affected_rows": 5,
+ "multi_statement_warning": "x" * 5000,
+ }
+ result, was_truncated, notes = truncate_query_result(response, 200)
+ assert was_truncated is True
+ assert notes
+ assert len(result["multi_statement_warning"]) < 5000
+ assert result["affected_rows"] == 5
+
def test_execute_sql_multiple_statements_each_capped(self) -> None:
"""Every statement's rows are capped, not just the last (top-level) one."""
row = {f"col_{i}": f"value_{i}" for i in range(10)}