You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2026/10/20 18:29:45 UTC
(superset) branch master updated: fix(export): honor CSV_EXPORT encoding in streaming CSV exports (#45351)
This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 973ac9b8efb fix(export): honor CSV_EXPORT encoding in streaming CSV exports (#45351)
973ac9b8efb is described below
commit 973ac9b8efba331e4b76d0d8b506f61ea35bd3a4
Author: Divyansh Yadav <an...@gmail.com>
AuthorDate: Tue Oct 20 23:59:38 2026 +0530
fix(export): honor CSV_EXPORT encoding in streaming CSV exports (#45351)
---
superset/charts/data/api.py | 4 +-
superset/commands/streaming_export/base.py | 34 +++++++++++++--
superset/sqllab/api.py | 4 +-
.../charts/data/streaming_filename_test.py | 18 +++++++-
.../commands/streaming_export/__init__.py | 16 +++++++
.../commands/streaming_export/base_test.py | 51 ++++++++++++++++++++++
tests/unit_tests/sqllab/api_test.py | 24 +++++++++-
7 files changed, 143 insertions(+), 8 deletions(-)
diff --git a/superset/charts/data/api.py b/superset/charts/data/api.py
index fa9c2f73184..be0316d40b6 100644
--- a/superset/charts/data/api.py
+++ b/superset/charts/data/api.py
@@ -47,6 +47,7 @@ from superset.commands.chart.exceptions import (
ChartDataCacheLoadError,
ChartDataQueryFailedError,
)
+from superset.commands.streaming_export.base import encode_csv_stream
from superset.common.chart_data import ChartDataResultFormat, ChartDataResultType
from superset.common.chart_data_timing import ChartDataExecutionResult
from superset.common.query_context_factory import QueryContextFactory
@@ -887,7 +888,8 @@ class ChartDataRestApi(ChartRestApi):
# Create response with streaming headers
response = Response(
- csv_generator_callable(), # Call the callable to get generator
+ # Call the callable to get generator
+ encode_csv_stream(csv_generator_callable(), encoding),
# Use content_type (not mimetype) so the charset is set verbatim;
# passing a charset via mimetype makes Werkzeug append a second
# charset, producing a malformed doubled Content-Type header.
diff --git a/superset/commands/streaming_export/base.py b/superset/commands/streaming_export/base.py
index d5814b1908f..e87065bbedb 100644
--- a/superset/commands/streaming_export/base.py
+++ b/superset/commands/streaming_export/base.py
@@ -18,6 +18,7 @@
from __future__ import annotations
+import codecs
import csv
import io
import logging
@@ -26,7 +27,7 @@ from abc import abstractmethod
from contextlib import contextmanager
from decimal import Decimal
from numbers import Real
-from typing import Any, Callable, Generator
+from typing import Any, Callable, Generator, Iterable
from flask import current_app as app, g, has_app_context
from sqlalchemy import text
@@ -91,6 +92,32 @@ def preserve_g_context(
yield
+def encode_csv_stream(
+ chunks: Iterable[str], encoding: str
+) -> Generator[bytes, None, None]:
+ """
+ Encode streamed CSV chunks with the configured CSV_EXPORT encoding.
+
+ Werkzeug encodes ``str`` chunks of a response body as plain UTF-8, which
+ ignores the configured encoding. An incremental encoder is used so that
+ codecs with a preamble, such as the default "utf-8-sig", write the byte
+ order mark once at the start of the file rather than once per chunk.
+
+ Args:
+ chunks: CSV data chunks as strings
+ encoding: Name of the codec to encode the chunks with
+
+ Yields:
+ Encoded CSV data chunks
+ """
+ encoder = codecs.getincrementalencoder(encoding)()
+ for chunk in chunks:
+ if data := encoder.encode(chunk):
+ yield data
+ if data := encoder.encode("", final=True):
+ yield data
+
+
class BaseStreamingCSVExportCommand(BaseCommand):
"""
Base class for streaming CSV export commands.
@@ -283,8 +310,9 @@ class BaseStreamingCSVExportCommand(BaseCommand):
# superset.charts.client_processing (which builds the whole file with a
# single DataFrame.to_csv(**CSV_EXPORT) call), this path writes rows
# incrementally via csv.writer, so the remaining pandas to_csv kwargs
- # (e.g. quotechar, lineterminator, encoding) do not map onto it and are
- # intentionally not applied here.
+ # (e.g. quotechar, lineterminator) do not map onto it and are
+ # intentionally not applied here. The `encoding` key is applied by the
+ # API layer when it encodes the streamed chunks (see encode_csv_stream).
csv_export_config = app.config["CSV_EXPORT"]
delimiter = csv_export_config.get("sep", ",")
decimal_separator = csv_export_config.get("decimal", ".")
diff --git a/superset/sqllab/api.py b/superset/sqllab/api.py
index 21a49feb149..9d98485bbe5 100644
--- a/superset/sqllab/api.py
+++ b/superset/sqllab/api.py
@@ -35,6 +35,7 @@ from superset.commands.sql_lab.results import SqlExecutionResultsCommand
from superset.commands.sql_lab.streaming_export_command import (
StreamingSqlResultExportCommand,
)
+from superset.commands.streaming_export.base import encode_csv_stream
from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP
from superset.daos.database import DatabaseDAO
from superset.daos.query import QueryDAO
@@ -466,7 +467,8 @@ class SqlLabRestApi(BaseSupersetApi):
# Create response with streaming headers
response = Response(
- csv_generator_callable(), # Call the callable to get generator
+ # Call the callable to get generator
+ encode_csv_stream(csv_generator_callable(), encoding),
mimetype=f"text/csv; charset={encoding}",
headers={
"Content-Disposition": f'attachment; filename="{filename}"',
diff --git a/tests/unit_tests/charts/data/streaming_filename_test.py b/tests/unit_tests/charts/data/streaming_filename_test.py
index 444e4e32b5e..1f8aa72af7e 100644
--- a/tests/unit_tests/charts/data/streaming_filename_test.py
+++ b/tests/unit_tests/charts/data/streaming_filename_test.py
@@ -14,12 +14,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+import codecs
from unittest.mock import MagicMock, patch
import pytest
+from flask import Flask
-def _build_response(filename: str | None):
+def _build_response(filename: str | None, chunks: list[str] | None = None):
from superset.charts.data.api import ChartDataRestApi
api = ChartDataRestApi.__new__(ChartDataRestApi)
@@ -29,7 +31,7 @@ def _build_response(filename: str | None):
"superset.charts.data.api.StreamingCSVExportCommand"
) as mock_command_cls:
mock_command = mock_command_cls.return_value
- mock_command.run.return_value = lambda: iter([b"a,b\n"])
+ mock_command.run.return_value = lambda: iter(chunks or ["a,b\n"])
return api._create_streaming_csv_response(result, filename=filename)
@@ -57,3 +59,15 @@ def test_blank_filename_falls_back_to_default() -> None:
response = _build_response("...")
disposition = response.headers["Content-Disposition"]
assert 'filename="export.csv"' in disposition
+
+
+def test_streamed_body_uses_csv_export_encoding(app: Flask) -> None:
+ """The streamed body honors CSV_EXPORT encoding, writing the BOM once."""
+ chunks = ["name;city\r\n", "Müller;Köln\r\n"]
+
+ with patch.dict(app.config, {"CSV_EXPORT": {"encoding": "utf-8-sig"}}):
+ response = _build_response("export.csv", chunks)
+ body = b"".join(response.iter_encoded())
+
+ assert response.headers["Content-Type"] == "text/csv; charset=utf-8-sig"
+ assert body == codecs.BOM_UTF8 + "".join(chunks).encode("utf-8")
diff --git a/tests/unit_tests/commands/streaming_export/__init__.py b/tests/unit_tests/commands/streaming_export/__init__.py
new file mode 100644
index 00000000000..13a83393a91
--- /dev/null
+++ b/tests/unit_tests/commands/streaming_export/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/tests/unit_tests/commands/streaming_export/base_test.py b/tests/unit_tests/commands/streaming_export/base_test.py
new file mode 100644
index 00000000000..3af42c91383
--- /dev/null
+++ b/tests/unit_tests/commands/streaming_export/base_test.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Unit tests for the shared streaming CSV export helpers."""
+
+import codecs
+
+import pytest
+
+from superset.commands.streaming_export.base import encode_csv_stream
+
+
+def test_encode_csv_stream_writes_bom_once() -> None:
+ """utf-8-sig writes a single BOM at the start, not one per chunk."""
+ chunks = ["name;city\r\n", "Müller;Köln\r\n", "Weiß;Zürich\r\n"]
+
+ data = b"".join(encode_csv_stream(chunks, "utf-8-sig"))
+
+ assert data.startswith(codecs.BOM_UTF8)
+ assert data.count(codecs.BOM_UTF8) == 1
+ assert data == codecs.BOM_UTF8 + "".join(chunks).encode("utf-8")
+
+
+def test_encode_csv_stream_skips_empty_chunks() -> None:
+ """Empty chunks don't produce empty byte chunks in the response body."""
+ encoded = list(encode_csv_stream(["", "a,b\r\n", ""], "utf-8"))
+
+ assert encoded == [b"a,b\r\n"]
+
+
+@pytest.mark.parametrize("encoding", ["utf-8", "latin-1", "cp1252", "utf-16"])
+def test_encode_csv_stream_uses_configured_encoding(encoding: str) -> None:
+ """The streamed bytes match encoding the whole file in one go."""
+ chunks = ["name,city\r\n", "Müller,Köln\r\n"]
+
+ data = b"".join(encode_csv_stream(chunks, encoding))
+
+ assert data == "".join(chunks).encode(encoding)
diff --git a/tests/unit_tests/sqllab/api_test.py b/tests/unit_tests/sqllab/api_test.py
index 00517326329..ad0b6cadceb 100644
--- a/tests/unit_tests/sqllab/api_test.py
+++ b/tests/unit_tests/sqllab/api_test.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import annotations
+import codecs
import re
from typing import Any
from unittest.mock import MagicMock, patch
@@ -39,7 +40,7 @@ def _disposition_filename(form_filename: str | None) -> str:
patch("superset.sqllab.api.StreamingSqlResultExportCommand") as command_cls,
):
command = command_cls.return_value
- command.run.return_value = lambda: iter([b""])
+ command.run.return_value = lambda: iter([""])
response = SqlLabRestApi._create_streaming_csv_response(
MagicMock(), client_id="abc123", filename=form_filename
)
@@ -70,6 +71,27 @@ def test_streaming_csv_falls_back_when_filename_empty() -> None:
assert filename.endswith(".csv")
+def test_streaming_csv_body_uses_csv_export_encoding() -> None:
+ """The streamed body honors CSV_EXPORT encoding, writing the BOM once."""
+ from superset.sqllab.api import SqlLabRestApi
+
+ chunks = ["name;city\r\n", "Müller;Köln\r\n"]
+ app = Flask(__name__)
+ app.config["CSV_EXPORT"] = {"encoding": "utf-8-sig", "sep": ";"}
+ with (
+ app.app_context(),
+ patch("superset.sqllab.api.StreamingSqlResultExportCommand") as command_cls,
+ ):
+ command = command_cls.return_value
+ command.run.return_value = lambda: iter(chunks)
+ response = SqlLabRestApi._create_streaming_csv_response(
+ MagicMock(), client_id="abc123", filename="results.csv"
+ )
+ body = b"".join(response.iter_encoded())
+
+ assert body == codecs.BOM_UTF8 + "".join(chunks).encode("utf-8")
+
+
def test_format_sql_checks_access_before_rendering(
mocker: MockerFixture,
client: Any,