You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2026/08/21 16:10:54 UTC

(superset) 05/05: fix(dataset-import): cap bytes read from a data URI, including after gzip

This is an automated email from the ASF dual-hosted git repository.

rusackas pushed a commit to branch fix/asset-importer-hardening
in repository https://gitbox.apache.org/repos/asf/superset.git

commit 0fb11e44cc7c65d897c96ecfa07c630f529f4539
Author: Superset Dev <de...@superset.apache.org>
AuthorDate: Fri Aug 21 00:53:20 2026 -0700

    fix(dataset-import): cap bytes read from a data URI, including after gzip
    
    load_data() streamed a dataset import's remote data URI straight into
    pd.read_csv with no size bound; for .gz URIs the gzip-decompressed
    stream had no bound either, unlike the ZIP-bundle and columnar-upload
    paths, which enforce ZIPPED_FILE_MAX_SIZE via check_is_safe_zip. Add
    _read_bounded(), which drains the (optionally decompressed) stream in
    1 MB chunks and raises ImportFailedError once the running total
    exceeds ZIPPED_FILE_MAX_SIZE, reusing the same per-file knob.
    
    Co-Authored-By: Claude Sonnet 5 <no...@anthropic.com>
---
 superset/commands/dataset/importers/v1/utils.py    | 29 +++++++++++++++++++++-
 .../unit_tests/commands/importers/v1/utils_test.py | 24 ++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/superset/commands/dataset/importers/v1/utils.py b/superset/commands/dataset/importers/v1/utils.py
index b23c1183761..7f7859a4000 100644
--- a/superset/commands/dataset/importers/v1/utils.py
+++ b/superset/commands/dataset/importers/v1/utils.py
@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 import gzip
+import io
 import ipaddress
 import logging
 import os
@@ -611,6 +612,29 @@ def _convert_temporal_columns(df: pd.DataFrame, dtype: dict[str, Any]) -> None:
                 df[column_name] = converted
 
 
+def _read_bounded(stream: Any, max_bytes: int) -> io.BytesIO:
+    """
+    Read ``stream`` into memory, failing once more than ``max_bytes`` bytes
+    have been produced.
+
+    Bounds both the raw download and gzip decompression amplification for
+    dataset data URIs: the ``.gz`` path had no analogue of
+    ``check_is_safe_zip`` and allowed unbounded expansion from a small
+    payload.
+    """
+    buffer = io.BytesIO()
+    while chunk := stream.read(1024 * 1024):
+        # Both http.client responses and gzip.open() yield bytes; a handful
+        # of tests substitute a text stream, so normalize either shape.
+        if isinstance(chunk, str):
+            chunk = chunk.encode("utf-8")
+        buffer.write(chunk)
+        if buffer.tell() > max_bytes:
+            raise ImportFailedError("Data URI payload exceeds the maximum allowed size")
+    buffer.seek(0)
+    return buffer
+
+
 def load_data(data_uri: str, dataset: SqlaTable, database: Database) -> None:
     """
     Load data from a data URI into a dataset.
@@ -637,9 +661,12 @@ def load_data(data_uri: str, dataset: SqlaTable, database: Database) -> None:
         handlers.extend([_PeerValidatingHTTPHandler, _PeerValidatingHTTPSHandler])
     opener = request.build_opener(*handlers)
     data = opener.open(data_uri)  # pylint: disable=consider-using-with  # noqa: S310
+    # Cap the bytes materialized from the download, before and after gzip
+    # decompression (same per-file knob as ZIP bundle uploads).
+    max_bytes = app.config["ZIPPED_FILE_MAX_SIZE"]
     if data_uri.endswith(".gz"):
         data = gzip.open(data)
-    df = pd.read_csv(data, encoding="utf-8")
+    df = pd.read_csv(_read_bounded(data, max_bytes), encoding="utf-8")
     dtype = get_dtype(df, dataset)
 
     _convert_temporal_columns(df, dtype)
diff --git a/tests/unit_tests/commands/importers/v1/utils_test.py b/tests/unit_tests/commands/importers/v1/utils_test.py
index 4fdc58bc83d..8d7cf95e01e 100644
--- a/tests/unit_tests/commands/importers/v1/utils_test.py
+++ b/tests/unit_tests/commands/importers/v1/utils_test.py
@@ -16,6 +16,8 @@
 # under the License.
 """Tests for superset/commands/dataset/importers/v1/utils.py temporal helpers."""
 
+import gzip
+import io
 from unittest.mock import patch
 
 import pandas as pd
@@ -119,6 +121,28 @@ class TestConvertTemporalColumns:
         assert call_args[1] == 2  # 2 out-of-bounds, 1 pre-existing null
 
 
+class TestReadBounded:
+    """``_read_bounded`` caps the bytes materialized from a dataset import
+    data URI, including after gzip decompression, so a small compressed
+    payload can't expand to an unbounded in-memory allocation."""
+
+    def test_rejects_oversized_gzip_stream(self) -> None:
+        """A small compressed payload that decompresses past the cap must fail."""
+        from superset.commands.dataset.importers.v1.utils import _read_bounded
+        from superset.commands.exceptions import ImportFailedError
+
+        payload = gzip.compress(b"a" * 100_000)
+        stream = gzip.GzipFile(fileobj=io.BytesIO(payload))
+        with pytest.raises(ImportFailedError):
+            _read_bounded(stream, max_bytes=10_000)
+
+    def test_passes_small_payload_through(self) -> None:
+        from superset.commands.dataset.importers.v1.utils import _read_bounded
+
+        buffer = _read_bounded(io.BytesIO(b"a,b\n1,2\n"), max_bytes=10_000)
+        assert buffer.read() == b"a,b\n1,2\n"
+
+
 class TestLoadYaml:
     def test_parser_error_raises_validation_error(self) -> None:
         """A malformed flow sequence raises yaml.parser.ParserError."""