You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2026/08/21 16:10:54 UTC
(superset) 05/05: fix(dataset-import): cap bytes read from a data URI, including after gzip
This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch fix/asset-importer-hardening
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 0fb11e44cc7c65d897c96ecfa07c630f529f4539
Author: Superset Dev <de...@superset.apache.org>
AuthorDate: Fri Aug 21 00:53:20 2026 -0700
fix(dataset-import): cap bytes read from a data URI, including after gzip
load_data() streamed a dataset import's remote data URI straight into
pd.read_csv with no size bound; for .gz URIs the gzip-decompressed
stream had no bound either, unlike the ZIP-bundle and columnar-upload
paths, which enforce ZIPPED_FILE_MAX_SIZE via check_is_safe_zip. Add
_read_bounded(), which drains the (optionally decompressed) stream in
1 MB chunks and raises ImportFailedError once the running total
exceeds ZIPPED_FILE_MAX_SIZE, reusing the same per-file knob.
Co-Authored-By: Claude Sonnet 5 <no...@anthropic.com>
---
superset/commands/dataset/importers/v1/utils.py | 29 +++++++++++++++++++++-
.../unit_tests/commands/importers/v1/utils_test.py | 24 ++++++++++++++++++
2 files changed, 52 insertions(+), 1 deletion(-)
diff --git a/superset/commands/dataset/importers/v1/utils.py b/superset/commands/dataset/importers/v1/utils.py
index b23c1183761..7f7859a4000 100644
--- a/superset/commands/dataset/importers/v1/utils.py
+++ b/superset/commands/dataset/importers/v1/utils.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
import gzip
+import io
import ipaddress
import logging
import os
@@ -611,6 +612,29 @@ def _convert_temporal_columns(df: pd.DataFrame, dtype: dict[str, Any]) -> None:
df[column_name] = converted
+def _read_bounded(stream: Any, max_bytes: int) -> io.BytesIO:
+ """
+ Read ``stream`` into memory, failing once more than ``max_bytes`` bytes
+ have been produced.
+
+ Bounds both the raw download and gzip decompression amplification for
+ dataset data URIs: the ``.gz`` path had no analogue of
+ ``check_is_safe_zip`` and allowed unbounded expansion from a small
+ payload.
+ """
+ buffer = io.BytesIO()
+ while chunk := stream.read(1024 * 1024):
+ # Both http.client responses and gzip.open() yield bytes; a handful
+ # of tests substitute a text stream, so normalize either shape.
+ if isinstance(chunk, str):
+ chunk = chunk.encode("utf-8")
+ buffer.write(chunk)
+ if buffer.tell() > max_bytes:
+ raise ImportFailedError("Data URI payload exceeds the maximum allowed size")
+ buffer.seek(0)
+ return buffer
+
+
def load_data(data_uri: str, dataset: SqlaTable, database: Database) -> None:
"""
Load data from a data URI into a dataset.
@@ -637,9 +661,12 @@ def load_data(data_uri: str, dataset: SqlaTable, database: Database) -> None:
handlers.extend([_PeerValidatingHTTPHandler, _PeerValidatingHTTPSHandler])
opener = request.build_opener(*handlers)
data = opener.open(data_uri) # pylint: disable=consider-using-with # noqa: S310
+ # Cap the bytes materialized from the download, before and after gzip
+ # decompression (same per-file knob as ZIP bundle uploads).
+ max_bytes = app.config["ZIPPED_FILE_MAX_SIZE"]
if data_uri.endswith(".gz"):
data = gzip.open(data)
- df = pd.read_csv(data, encoding="utf-8")
+ df = pd.read_csv(_read_bounded(data, max_bytes), encoding="utf-8")
dtype = get_dtype(df, dataset)
_convert_temporal_columns(df, dtype)
diff --git a/tests/unit_tests/commands/importers/v1/utils_test.py b/tests/unit_tests/commands/importers/v1/utils_test.py
index 4fdc58bc83d..8d7cf95e01e 100644
--- a/tests/unit_tests/commands/importers/v1/utils_test.py
+++ b/tests/unit_tests/commands/importers/v1/utils_test.py
@@ -16,6 +16,8 @@
# under the License.
"""Tests for superset/commands/dataset/importers/v1/utils.py temporal helpers."""
+import gzip
+import io
from unittest.mock import patch
import pandas as pd
@@ -119,6 +121,28 @@ class TestConvertTemporalColumns:
assert call_args[1] == 2 # 2 out-of-bounds, 1 pre-existing null
+class TestReadBounded:
+ """``_read_bounded`` caps the bytes materialized from a dataset import
+ data URI, including after gzip decompression, so a small compressed
+ payload can't expand to an unbounded in-memory allocation."""
+
+ def test_rejects_oversized_gzip_stream(self) -> None:
+ """A small compressed payload that decompresses past the cap must fail."""
+ from superset.commands.dataset.importers.v1.utils import _read_bounded
+ from superset.commands.exceptions import ImportFailedError
+
+ payload = gzip.compress(b"a" * 100_000)
+ stream = gzip.GzipFile(fileobj=io.BytesIO(payload))
+ with pytest.raises(ImportFailedError):
+ _read_bounded(stream, max_bytes=10_000)
+
+ def test_passes_small_payload_through(self) -> None:
+ from superset.commands.dataset.importers.v1.utils import _read_bounded
+
+ buffer = _read_bounded(io.BytesIO(b"a,b\n1,2\n"), max_bytes=10_000)
+ assert buffer.read() == b"a,b\n1,2\n"
+
+
class TestLoadYaml:
def test_parser_error_raises_validation_error(self) -> None:
"""A malformed flow sequence raises yaml.parser.ParserError."""