You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by mi...@apache.org on 2026/07/22 03:49:58 UTC

(superset) 04/04: fix(mcp): sync per-statement row_count after execute_sql truncation

This is an automated email from the ASF dual-hosted git repository.

aminghadersohi pushed a commit to branch aminghadersohi/mcp-truncate-query-responses
in repository https://gitbox.apache.org/repos/asf/superset.git

commit 00c5e7957b89daeba9a67f5448a70bbe76840a8b
Author: Amin Ghadersohi <am...@gmail.com>
AuthorDate: Wed Jul 22 03:49:17 2026 +0000

    fix(mcp): sync per-statement row_count after execute_sql truncation
    
    _bisect_row_limit already trims statements[*].data.rows in lockstep
    with the top-level rows field to keep the response under the token
    limit, but left each statement's row_count pointing at the
    pre-truncation count, so a truncated execute_sql response could claim
    more rows than statements[*].data actually contains. Track the
    statement dicts (not just their row lists) and update row_count
    alongside data.rows.
    
    Also log a full traceback (logger.exception) instead of a plain
    warning when data-query truncation hits an unexpected error, so a
    real bug isn't indistinguishable from routine size-limit enforcement.
    
    Adds regression tests for: total_rows surviving truncation distinct
    from row_count, the no-recognised-row-field fallback path, and the
    oversized-excel-export hard-failure path.
---
 superset/mcp_service/middleware.py                 |  9 ++-
 superset/mcp_service/utils/token_utils.py          | 41 +++++++-----
 tests/unit_tests/mcp_service/test_middleware.py    | 72 ++++++++++++++++++++++
 .../mcp_service/utils/test_token_utils.py          | 59 ++++++++++++++++++
 4 files changed, 164 insertions(+), 17 deletions(-)

diff --git a/superset/mcp_service/middleware.py b/superset/mcp_service/middleware.py
index 8b31335b8b5..ca308db9464 100644
--- a/superset/mcp_service/middleware.py
+++ b/superset/mcp_service/middleware.py
@@ -1382,7 +1382,14 @@ class ResponseSizeGuardMiddleware(Middleware):
                 size_fn=size_fn,
             )
         except Exception as trunc_error:  # noqa: BLE001
-            logger.warning(
+            # Deliberately broad: any failure here must fail safe into the
+            # hard size-limit error below rather than ship a broken
+            # truncation, so this stays a catch-all rather than narrowing to
+            # specific exception types. Logged with a full traceback (unlike
+            # the plain warning above) so a genuine bug (e.g. a KeyError from
+            # an unexpected response shape) is still visible instead of
+            # looking like routine size-limit enforcement.
+            logger.exception(
                 "Query result truncation failed for %s due to %s: %s",
                 tool_name,
                 type(trunc_error).__name__,
diff --git a/superset/mcp_service/utils/token_utils.py b/superset/mcp_service/utils/token_utils.py
index 91fdda3fe09..63f079dd1bf 100644
--- a/superset/mcp_service/utils/token_utils.py
+++ b/superset/mcp_service/utils/token_utils.py
@@ -698,8 +698,9 @@ def truncate_oversized_response(
     return data, was_truncated, notes
 
 
-def _linked_statement_row_lists(data: Dict[str, Any]) -> list[list[Any]]:
-    """Find per-statement row lists duplicated from an ``execute_sql`` response.
+def _linked_statement_entries(data: Dict[str, Any]) -> list[Dict[str, Any]]:
+    """Find per-statement dicts with row data duplicated from an
+    ``execute_sql`` response's top-level ``rows`` field.
 
     ``ExecuteSqlResponse.rows`` is a copy of the last data-bearing
     statement's ``data.rows`` (see ``_convert_to_response`` in
@@ -707,8 +708,11 @@ def _linked_statement_row_lists(data: Dict[str, Any]) -> list[list[Any]]:
     rows are additionally serialised under ``statements[*].data.rows``. If
     only the top-level ``rows`` field is bisected, these nested copies keep
     the full untruncated payload, which can leave the response oversized
-    even after "truncation". Returns the mutable row lists (if any) so the
-    caller can shrink them in lockstep with the top-level field.
+    even after "truncation". Returns the statement dicts themselves (not
+    just their row lists) so the caller can shrink ``data.rows`` in
+    lockstep with the top-level field *and* keep each statement's
+    ``row_count`` in sync, rather than leaving it pointing at the
+    pre-truncation count.
     """
     statements = data.get("statements")
     if not isinstance(statements, list):
@@ -722,7 +726,7 @@ def _linked_statement_row_lists(data: Dict[str, Any]) -> list[list[Any]]:
         if isinstance(statement_data, dict) and isinstance(
             statement_data.get("rows"), list
         ):
-            linked.append(statement_data["rows"])
+            linked.append(statement)
     return linked
 
 
@@ -741,23 +745,27 @@ def _bisect_row_limit(
 
     When ``data`` is an ``execute_sql`` response, per-statement row lists
     under ``statements[*].data.rows`` duplicate ``data[row_field]`` (see
-    ``_linked_statement_row_lists``) and are shrunk to the same length at
-    each step, so the size measured during the search — and the response
-    ultimately returned — reflects the truncated nested data too, not just
-    the top-level field.
+    ``_linked_statement_entries``) and are shrunk to the same length at
+    each step — along with each statement's ``row_count``, so it doesn't
+    keep reporting the pre-truncation count — so the size measured during
+    the search, and the response ultimately returned, reflects the
+    truncated nested data too, not just the top-level field.
 
     ``size_fn`` measures each candidate; pass a function that re-wraps the
     payload (e.g. back into a ToolResult) before measuring so the search
     converges on a prefix that still fits the limit after re-wrapping.
     """
-    linked_rows = [list(rows) for rows in _linked_statement_row_lists(data)]
+    linked_statements = _linked_statement_entries(data)
+    linked_rows = [list(stmt["data"]["rows"]) for stmt in linked_statements]
 
     def _apply(count: int) -> None:
         data[row_field] = original_rows[:count]
-        for original_linked, mutable_linked in zip(
-            linked_rows, _linked_statement_row_lists(data), strict=False
+        for original_linked, statement in zip(
+            linked_rows, linked_statements, strict=False
         ):
-            mutable_linked[:] = original_linked[: min(count, len(original_linked))]
+            kept_linked = original_linked[: min(count, len(original_linked))]
+            statement["data"]["rows"] = kept_linked
+            statement["row_count"] = len(kept_linked)
 
     lo, hi = 0, len(original_rows)
     while lo < hi:
@@ -973,9 +981,10 @@ def truncate_query_result(
     rely on ``_truncation_notes`` and the true ``total_rows`` to catch this.
 
     For ``execute_sql``, per-statement row lists under
-    ``statements[*].data.rows`` are shrunk alongside the top-level field
-    (see ``_linked_statement_row_lists``) so a duplicate, untruncated copy
-    of the same rows can't leave the response oversized after "truncation".
+    ``statements[*].data.rows`` (and their ``row_count``) are shrunk
+    alongside the top-level field (see ``_linked_statement_entries``) so a
+    duplicate, untruncated copy of the same rows can't leave the response
+    oversized, or its metadata inconsistent, after "truncation".
 
     Args:
         response: The tool response containing tabular row data.
diff --git a/tests/unit_tests/mcp_service/test_middleware.py b/tests/unit_tests/mcp_service/test_middleware.py
index 705f4c5e864..61ffccbb116 100644
--- a/tests/unit_tests/mcp_service/test_middleware.py
+++ b/tests/unit_tests/mcp_service/test_middleware.py
@@ -582,6 +582,78 @@ class TestResponseSizeGuardMiddleware:
         assert result["_response_truncated"] is True
         assert len(result["csv_data"]) < len(large_response["csv_data"])
 
+    @pytest.mark.asyncio
+    async def test_data_query_truncation_preserves_total_rows(self) -> None:
+        """``total_rows`` (the true pre-truncation count) must survive.
+
+        ``row_count`` is overwritten to reflect the truncated row list, but
+        ``total_rows`` is the signal callers rely on to detect that a
+        response is partial — it must keep describing the original,
+        untruncated dataset.
+        """
+        middleware = ResponseSizeGuardMiddleware(token_limit=500)
+
+        context = MagicMock()
+        context.message.name = "query_dataset"
+        context.message.params = {}
+
+        row = {f"col_{i}": f"value_{i}" for i in range(10)}
+        large_response = {
+            "dataset_id": 1,
+            "dataset_name": "test",
+            "data": [row] * 200,
+            "row_count": 200,
+            "total_rows": 5000,
+            "summary": "",
+        }
+        call_next = AsyncMock(return_value=large_response)
+
+        with (
+            patch("superset.mcp_service.middleware.get_user_id", return_value=1),
+            patch("superset.mcp_service.middleware.event_logger"),
+        ):
+            result = await middleware.on_call_tool(context, call_next)
+
+        assert result["_response_truncated"] is True
+        assert result["row_count"] == len(result["data"])
+        assert result["row_count"] < 200
+        assert result["total_rows"] == 5000
+
+    @pytest.mark.asyncio
+    async def test_get_chart_data_excel_export_raises_tool_error(self) -> None:
+        """Oversized excel exports must hard-fail, not ship a corrupt file.
+
+        ``excel_data`` is base64-encoded binary and is deliberately left
+        untouched by truncation (cutting it would produce an unreadable
+        file), and there are no other truncatable fields (``data=[]``), so
+        ``truncate_query_result`` reports ``was_truncated=False`` and the
+        response must fall through to the hard size-limit error.
+        """
+        middleware = ResponseSizeGuardMiddleware(token_limit=500)
+
+        context = MagicMock()
+        context.message.name = "get_chart_data"
+        context.message.params = {}
+
+        large_response: dict[str, Any] = {
+            "chart_id": 1,
+            "chart_name": "test chart",
+            "chart_type": "table",
+            "data": [],
+            "row_count": 200,
+            "summary": "",
+            "excel_data": "QUJDREVGRw==" * 5000,
+            "format": "excel",
+        }
+        call_next = AsyncMock(return_value=large_response)
+
+        with (
+            patch("superset.mcp_service.middleware.get_user_id", return_value=1),
+            patch("superset.mcp_service.middleware.event_logger"),
+            pytest.raises(ToolError),
+        ):
+            await middleware.on_call_tool(context, call_next)
+
     @pytest.mark.asyncio
     async def test_data_query_blocks_when_single_row_still_exceeds_limit(self) -> None:
         """Should raise ToolError, not ship an over-budget response.
diff --git a/tests/unit_tests/mcp_service/utils/test_token_utils.py b/tests/unit_tests/mcp_service/utils/test_token_utils.py
index e8f7d864496..bce195a1ab0 100644
--- a/tests/unit_tests/mcp_service/utils/test_token_utils.py
+++ b/tests/unit_tests/mcp_service/utils/test_token_utils.py
@@ -934,6 +934,65 @@ class TestTruncateQueryResult:
         assert len(result["statements"][0]["data"]["rows"]) == len(result["rows"])
         assert notes
 
+    def test_execute_sql_statements_row_count_stays_in_sync(self) -> None:
+        """Each statement's ``row_count`` must track its trimmed ``data.rows``.
+
+        Regression test: bisection already shrinks the nested
+        ``statements[*].data.rows`` list in lockstep with the top-level
+        field (see the sibling ``..._rows_truncated_alongside_top_level``
+        test), but the per-statement ``row_count`` field is a separate
+        value copied at response-build time and must be updated too, or a
+        client would see a statement whose declared row count doesn't match
+        the row list actually returned.
+        """
+        row = {f"col_{i}": f"value_{i}" for i in range(10)}
+        rows = [row] * 200
+        response: dict[str, Any] = {
+            "success": True,
+            "rows": rows,
+            "row_count": 200,
+            "statements": [
+                {
+                    "original_sql": "SELECT * FROM t",
+                    "executed_sql": "SELECT * FROM t",
+                    "row_count": 200,
+                    "data": {
+                        "rows": rows,
+                        "columns": [
+                            {"name": f"col_{i}", "type": "TEXT"} for i in range(10)
+                        ],
+                    },
+                }
+            ],
+        }
+        result, was_truncated, _ = truncate_query_result(
+            response, 500, tool_name="execute_sql"
+        )
+        assert was_truncated is True
+        statement = result["statements"][0]
+        assert statement["row_count"] == len(statement["data"]["rows"])
+        assert statement["row_count"] < 200
+
+    def test_falls_back_to_generic_truncation_when_no_row_field(self) -> None:
+        """Responses without a recognised row/data field truncate generically.
+
+        A DML-final ``execute_sql`` response (or any data-query response
+        with no ``rows``/``data`` list) has no row field to bisect on, so
+        ``truncate_query_result`` delegates to the generic
+        ``truncate_oversized_response`` field-truncation phases instead of
+        raising or returning the payload untouched.
+        """
+        response: dict[str, Any] = {
+            "success": True,
+            "affected_rows": 5,
+            "multi_statement_warning": "x" * 5000,
+        }
+        result, was_truncated, notes = truncate_query_result(response, 200)
+        assert was_truncated is True
+        assert notes
+        assert len(result["multi_statement_warning"]) < 5000
+        assert result["affected_rows"] == 5
+
     def test_execute_sql_multiple_statements_each_capped(self) -> None:
         """Every statement's rows are capped, not just the last (top-level) one."""
         row = {f"col_{i}": f"value_{i}" for i in range(10)}