You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hamilton.apache.org by je...@apache.org on 2026/08/13 09:03:29 UTC
(hamilton) branch review/fix-1653-lazyframe-sinks created (now 8f6cde2a)
This is an automated email from the ASF dual-hosted git repository.
jernejfrank pushed a change to branch review/fix-1653-lazyframe-sinks
in repository https://gitbox.apache.org/repos/asf/hamilton.git
at 8f6cde2a fix(polars): close LazyFrame sink gaps
This branch includes the following new commits:
new 8f6cde2a fix(polars): close LazyFrame sink gaps
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(hamilton) 01/01: fix(polars): close LazyFrame sink gaps
Posted by je...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jernejfrank pushed a commit to branch review/fix-1653-lazyframe-sinks
in repository https://gitbox.apache.org/repos/asf/hamilton.git
commit 8f6cde2a8cb182ec14a1b37c83a39213ee52c075
Author: jernejfrank <je...@safeintelligence.ai>
AuthorDate: Thu Aug 13 10:03:16 2026 +0100
fix(polars): close LazyFrame sink gaps
Make streaming sinks the sole adapters for supported LazyFrame formats while preserving eager writers for DataFrames. Align sink options with Polars and cover registry and materializer resolution.
Refs apache/hamilton#1653
---
hamilton/plugins/polars_lazyframe_extensions.py | 178 +++++++++++++++++++---
hamilton/plugins/polars_post_1_0_0_extensions.py | 26 +---
hamilton/plugins/polars_pre_1_0_0_extension.py | 19 +--
tests/plugins/test_polars_extensions.py | 8 +-
tests/plugins/test_polars_lazyframe_extensions.py | 129 +++++++++++++---
5 files changed, 277 insertions(+), 83 deletions(-)
diff --git a/hamilton/plugins/polars_lazyframe_extensions.py b/hamilton/plugins/polars_lazyframe_extensions.py
index a0b476aa..1c57188b 100644
--- a/hamilton/plugins/polars_lazyframe_extensions.py
+++ b/hamilton/plugins/polars_lazyframe_extensions.py
@@ -194,8 +194,12 @@ class PolarsSinkCSVWriter(DataSaver):
the file to exist immediately after the call returns.
"""
- path: str | Path
+ file: str | Path
# kwargs:
+ include_bom: bool | None = None
+ compression: str | None = None
+ compression_level: int | None = None
+ check_extension: bool | None = None
include_header: bool = True
separator: str = ","
line_terminator: str = "\n"
@@ -206,12 +210,29 @@ class PolarsSinkCSVWriter(DataSaver):
time_format: str | None = None
float_scientific: bool | None = None
float_precision: int | None = None
- null_value: str = ""
- quote_style: str = "necessary"
+ decimal_comma: bool | None = None
+ null_value: str | None = None
+ quote_style: Any = None
maintain_order: bool = True
-
- def _get_saving_kwargs(self):
+ storage_options: dict[str, Any] | None = None
+ credential_provider: Any = None
+ retries: int | None = None
+ sync_on_close: Any = None
+ mkdir: bool | None = None
+ engine: Any = None
+ optimizations: Any = None
+ extra_kwargs: dict[str, Any] | None = None
+
+ def _get_saving_kwargs(self) -> dict[str, Any]:
kwargs = {}
+ if self.include_bom is not None:
+ kwargs["include_bom"] = self.include_bom
+ if self.compression is not None:
+ kwargs["compression"] = self.compression
+ if self.compression_level is not None:
+ kwargs["compression_level"] = self.compression_level
+ if self.check_extension is not None:
+ kwargs["check_extension"] = self.check_extension
if self.include_header is not None:
kwargs["include_header"] = self.include_header
if self.separator is not None:
@@ -232,12 +253,30 @@ class PolarsSinkCSVWriter(DataSaver):
kwargs["float_scientific"] = self.float_scientific
if self.float_precision is not None:
kwargs["float_precision"] = self.float_precision
+ if self.decimal_comma is not None:
+ kwargs["decimal_comma"] = self.decimal_comma
if self.null_value is not None:
kwargs["null_value"] = self.null_value
if self.quote_style is not None:
kwargs["quote_style"] = self.quote_style
if self.maintain_order is not None:
kwargs["maintain_order"] = self.maintain_order
+ if self.storage_options is not None:
+ kwargs["storage_options"] = self.storage_options
+ if self.credential_provider is not None:
+ kwargs["credential_provider"] = self.credential_provider
+ if self.retries is not None:
+ kwargs["retries"] = self.retries
+ if self.sync_on_close is not None:
+ kwargs["sync_on_close"] = self.sync_on_close
+ if self.mkdir is not None:
+ kwargs["mkdir"] = self.mkdir
+ if self.engine is not None:
+ kwargs["engine"] = self.engine
+ if self.optimizations is not None:
+ kwargs["optimizations"] = self.optimizations
+ if self.extra_kwargs is not None:
+ kwargs.update(self.extra_kwargs)
return kwargs
@classmethod
@@ -245,8 +284,8 @@ class PolarsSinkCSVWriter(DataSaver):
return [DATAFRAME_TYPE]
def save_data(self, data: pl.LazyFrame) -> dict[str, Any]:
- data.sink_csv(self.path, **self._get_saving_kwargs())
- return utils.get_file_metadata(self.path)
+ data.sink_csv(self.file, **self._get_saving_kwargs())
+ return utils.get_file_metadata(self.file)
@classmethod
def name(cls) -> str:
@@ -320,16 +359,26 @@ class PolarsSinkParquetWriter(DataSaver):
the file to exist immediately after the call returns.
"""
- path: str | Path
+ file: str | Path
# kwargs:
compression: str = "zstd"
compression_level: int | None = None
- statistics: bool = True
+ statistics: bool | str | dict[str, bool] = True
row_group_size: int | None = None
data_page_size: int | None = None
maintain_order: bool = True
-
- def _get_saving_kwargs(self):
+ storage_options: dict[str, Any] | None = None
+ credential_provider: Any = None
+ retries: int | None = None
+ sync_on_close: Any = None
+ metadata: Any = None
+ arrow_schema: Any = None
+ mkdir: bool | None = None
+ engine: Any = None
+ optimizations: Any = None
+ extra_kwargs: dict[str, Any] | None = None
+
+ def _get_saving_kwargs(self) -> dict[str, Any]:
kwargs = {}
if self.compression is not None:
kwargs["compression"] = self.compression
@@ -343,6 +392,26 @@ class PolarsSinkParquetWriter(DataSaver):
kwargs["data_page_size"] = self.data_page_size
if self.maintain_order is not None:
kwargs["maintain_order"] = self.maintain_order
+ if self.storage_options is not None:
+ kwargs["storage_options"] = self.storage_options
+ if self.credential_provider is not None:
+ kwargs["credential_provider"] = self.credential_provider
+ if self.retries is not None:
+ kwargs["retries"] = self.retries
+ if self.sync_on_close is not None:
+ kwargs["sync_on_close"] = self.sync_on_close
+ if self.metadata is not None:
+ kwargs["metadata"] = self.metadata
+ if self.arrow_schema is not None:
+ kwargs["arrow_schema"] = self.arrow_schema
+ if self.mkdir is not None:
+ kwargs["mkdir"] = self.mkdir
+ if self.engine is not None:
+ kwargs["engine"] = self.engine
+ if self.optimizations is not None:
+ kwargs["optimizations"] = self.optimizations
+ if self.extra_kwargs is not None:
+ kwargs.update(self.extra_kwargs)
return kwargs
@classmethod
@@ -350,8 +419,8 @@ class PolarsSinkParquetWriter(DataSaver):
return [DATAFRAME_TYPE]
def save_data(self, data: pl.LazyFrame) -> dict[str, Any]:
- data.sink_parquet(self.path, **self._get_saving_kwargs())
- return utils.get_file_metadata(self.path)
+ data.sink_parquet(self.file, **self._get_saving_kwargs())
+ return utils.get_file_metadata(self.file)
@classmethod
def name(cls) -> str:
@@ -419,17 +488,47 @@ class PolarsSinkFeatherWriter(DataSaver):
the file to exist immediately after the call returns.
"""
- path: str | Path
+ file: str | Path
# kwargs:
compression: str = "uncompressed"
+ compat_level: Any = None
+ record_batch_size: int | None = None
maintain_order: bool = True
-
- def _get_saving_kwargs(self):
+ storage_options: dict[str, Any] | None = None
+ credential_provider: Any = None
+ retries: int | None = None
+ sync_on_close: Any = None
+ mkdir: bool | None = None
+ engine: Any = None
+ optimizations: Any = None
+ extra_kwargs: dict[str, Any] | None = None
+
+ def _get_saving_kwargs(self) -> dict[str, Any]:
kwargs = {}
if self.compression is not None:
kwargs["compression"] = self.compression
+ if self.compat_level is not None:
+ kwargs["compat_level"] = self.compat_level
+ if self.record_batch_size is not None:
+ kwargs["record_batch_size"] = self.record_batch_size
if self.maintain_order is not None:
kwargs["maintain_order"] = self.maintain_order
+ if self.storage_options is not None:
+ kwargs["storage_options"] = self.storage_options
+ if self.credential_provider is not None:
+ kwargs["credential_provider"] = self.credential_provider
+ if self.retries is not None:
+ kwargs["retries"] = self.retries
+ if self.sync_on_close is not None:
+ kwargs["sync_on_close"] = self.sync_on_close
+ if self.mkdir is not None:
+ kwargs["mkdir"] = self.mkdir
+ if self.engine is not None:
+ kwargs["engine"] = self.engine
+ if self.optimizations is not None:
+ kwargs["optimizations"] = self.optimizations
+ if self.extra_kwargs is not None:
+ kwargs.update(self.extra_kwargs)
return kwargs
@classmethod
@@ -437,8 +536,8 @@ class PolarsSinkFeatherWriter(DataSaver):
return [DATAFRAME_TYPE]
def save_data(self, data: pl.LazyFrame) -> dict[str, Any]:
- data.sink_ipc(self.path, **self._get_saving_kwargs())
- return utils.get_file_metadata(self.path)
+ data.sink_ipc(self.file, **self._get_saving_kwargs())
+ return utils.get_file_metadata(self.file)
@classmethod
def name(cls) -> str:
@@ -456,14 +555,47 @@ class PolarsSinkNDJSONWriter(DataSaver):
the file to exist immediately after the call returns.
"""
- path: str | Path
+ file: str | Path
# kwargs:
+ compression: str | None = None
+ compression_level: int | None = None
+ check_extension: bool | None = None
maintain_order: bool = True
-
- def _get_saving_kwargs(self):
+ storage_options: dict[str, Any] | None = None
+ credential_provider: Any = None
+ retries: int | None = None
+ sync_on_close: Any = None
+ mkdir: bool | None = None
+ engine: Any = None
+ optimizations: Any = None
+ extra_kwargs: dict[str, Any] | None = None
+
+ def _get_saving_kwargs(self) -> dict[str, Any]:
kwargs = {}
+ if self.compression is not None:
+ kwargs["compression"] = self.compression
+ if self.compression_level is not None:
+ kwargs["compression_level"] = self.compression_level
+ if self.check_extension is not None:
+ kwargs["check_extension"] = self.check_extension
if self.maintain_order is not None:
kwargs["maintain_order"] = self.maintain_order
+ if self.storage_options is not None:
+ kwargs["storage_options"] = self.storage_options
+ if self.credential_provider is not None:
+ kwargs["credential_provider"] = self.credential_provider
+ if self.retries is not None:
+ kwargs["retries"] = self.retries
+ if self.sync_on_close is not None:
+ kwargs["sync_on_close"] = self.sync_on_close
+ if self.mkdir is not None:
+ kwargs["mkdir"] = self.mkdir
+ if self.engine is not None:
+ kwargs["engine"] = self.engine
+ if self.optimizations is not None:
+ kwargs["optimizations"] = self.optimizations
+ if self.extra_kwargs is not None:
+ kwargs.update(self.extra_kwargs)
return kwargs
@classmethod
@@ -471,8 +603,8 @@ class PolarsSinkNDJSONWriter(DataSaver):
return [DATAFRAME_TYPE]
def save_data(self, data: pl.LazyFrame) -> dict[str, Any]:
- data.sink_ndjson(self.path, **self._get_saving_kwargs())
- return utils.get_file_metadata(self.path)
+ data.sink_ndjson(self.file, **self._get_saving_kwargs())
+ return utils.get_file_metadata(self.file)
@classmethod
def name(cls) -> str:
diff --git a/hamilton/plugins/polars_post_1_0_0_extensions.py b/hamilton/plugins/polars_post_1_0_0_extensions.py
index dab1a397..44b37964 100644
--- a/hamilton/plugins/polars_post_1_0_0_extensions.py
+++ b/hamilton/plugins/polars_post_1_0_0_extensions.py
@@ -197,7 +197,7 @@ class PolarsCSVWriter(DataSaver):
@classmethod
def applicable_types(cls) -> Collection[type]:
- return [DATAFRAME_TYPE, pl.LazyFrame]
+ return [DATAFRAME_TYPE]
def _get_saving_kwargs(self):
kwargs = {}
@@ -227,9 +227,7 @@ class PolarsCSVWriter(DataSaver):
kwargs["quote_style"] = self.quote_style
return kwargs
- def save_data(self, data: DATAFRAME_TYPE | pl.LazyFrame) -> dict[str, Any]:
- if isinstance(data, pl.LazyFrame):
- data = data.collect()
+ def save_data(self, data: DATAFRAME_TYPE) -> dict[str, Any]:
data.write_csv(self.file, **self._get_saving_kwargs())
return utils.get_file_and_dataframe_metadata(self.file, data)
@@ -318,7 +316,7 @@ class PolarsParquetWriter(DataSaver):
@classmethod
def applicable_types(cls) -> Collection[type]:
- return [DATAFRAME_TYPE, pl.LazyFrame]
+ return [DATAFRAME_TYPE]
def _get_saving_kwargs(self):
kwargs = {}
@@ -336,10 +334,7 @@ class PolarsParquetWriter(DataSaver):
kwargs["pyarrow_options"] = self.pyarrow_options
return kwargs
- def save_data(self, data: DATAFRAME_TYPE | pl.LazyFrame) -> dict[str, Any]:
- if isinstance(data, pl.LazyFrame):
- data = data.collect()
-
+ def save_data(self, data: DATAFRAME_TYPE) -> dict[str, Any]:
data.write_parquet(self.file, **self._get_saving_kwargs())
return utils.get_file_and_dataframe_metadata(self.file, data)
@@ -414,7 +409,7 @@ class PolarsFeatherWriter(DataSaver):
@classmethod
def applicable_types(cls) -> Collection[type]:
- return [DATAFRAME_TYPE, pl.LazyFrame]
+ return [DATAFRAME_TYPE]
def _get_saving_kwargs(self):
kwargs = {}
@@ -422,9 +417,7 @@ class PolarsFeatherWriter(DataSaver):
kwargs["compression"] = self.compression
return kwargs
- def save_data(self, data: DATAFRAME_TYPE | pl.LazyFrame) -> dict[str, Any]:
- if isinstance(data, pl.LazyFrame):
- data = data.collect()
+ def save_data(self, data: DATAFRAME_TYPE) -> dict[str, Any]:
data.write_ipc(self.file, **self._get_saving_kwargs())
return utils.get_file_and_dataframe_metadata(self.file, data)
@@ -600,12 +593,9 @@ class PolarsNDJSONWriter(DataSaver):
@classmethod
def applicable_types(cls) -> Collection[type]:
- return [DATAFRAME_TYPE, pl.LazyFrame]
-
- def save_data(self, data: DATAFRAME_TYPE | pl.LazyFrame) -> dict[str, Any]:
- if isinstance(data, pl.LazyFrame):
- data = data.collect()
+ return [DATAFRAME_TYPE]
+ def save_data(self, data: DATAFRAME_TYPE) -> dict[str, Any]:
data.write_ndjson(self.file)
return utils.get_file_and_dataframe_metadata(self.file, data)
diff --git a/hamilton/plugins/polars_pre_1_0_0_extension.py b/hamilton/plugins/polars_pre_1_0_0_extension.py
index 4dd92dc4..bd3b54d2 100644
--- a/hamilton/plugins/polars_pre_1_0_0_extension.py
+++ b/hamilton/plugins/polars_pre_1_0_0_extension.py
@@ -206,7 +206,7 @@ class PolarsCSVWriter(DataSaver):
@classmethod
def applicable_types(cls) -> Collection[type]:
- return [DATAFRAME_TYPE, pl.LazyFrame]
+ return [DATAFRAME_TYPE]
def _get_saving_kwargs(self):
kwargs = {}
@@ -236,9 +236,7 @@ class PolarsCSVWriter(DataSaver):
kwargs["quote_style"] = self.quote_style
return kwargs
- def save_data(self, data: DATAFRAME_TYPE | pl.LazyFrame) -> dict[str, Any]:
- if isinstance(data, pl.LazyFrame):
- data = data.collect()
+ def save_data(self, data: DATAFRAME_TYPE) -> dict[str, Any]:
data.write_csv(self.file, **self._get_saving_kwargs())
return utils.get_file_and_dataframe_metadata(self.file, data)
@@ -327,7 +325,7 @@ class PolarsParquetWriter(DataSaver):
@classmethod
def applicable_types(cls) -> Collection[type]:
- return [DATAFRAME_TYPE, pl.LazyFrame]
+ return [DATAFRAME_TYPE]
def _get_saving_kwargs(self):
kwargs = {}
@@ -345,10 +343,7 @@ class PolarsParquetWriter(DataSaver):
kwargs["pyarrow_options"] = self.pyarrow_options
return kwargs
- def save_data(self, data: DATAFRAME_TYPE | pl.LazyFrame) -> dict[str, Any]:
- if isinstance(data, pl.LazyFrame):
- data = data.collect()
-
+ def save_data(self, data: DATAFRAME_TYPE) -> dict[str, Any]:
data.write_parquet(self.file, **self._get_saving_kwargs())
return utils.get_file_and_dataframe_metadata(self.file, data)
@@ -423,7 +418,7 @@ class PolarsFeatherWriter(DataSaver):
@classmethod
def applicable_types(cls) -> Collection[type]:
- return [DATAFRAME_TYPE, pl.LazyFrame]
+ return [DATAFRAME_TYPE]
def _get_saving_kwargs(self):
kwargs = {}
@@ -431,9 +426,7 @@ class PolarsFeatherWriter(DataSaver):
kwargs["compression"] = self.compression
return kwargs
- def save_data(self, data: DATAFRAME_TYPE | pl.LazyFrame) -> dict[str, Any]:
- if isinstance(data, pl.LazyFrame):
- data = data.collect()
+ def save_data(self, data: DATAFRAME_TYPE) -> dict[str, Any]:
data.write_ipc(self.file, **self._get_saving_kwargs())
return utils.get_file_and_dataframe_metadata(self.file, data)
diff --git a/tests/plugins/test_polars_extensions.py b/tests/plugins/test_polars_extensions.py
index 2709dd85..1d4b9882 100644
--- a/tests/plugins/test_polars_extensions.py
+++ b/tests/plugins/test_polars_extensions.py
@@ -67,7 +67,7 @@ def test_polars_csv(df: pl.DataFrame, tmp_path: pathlib.Path) -> None:
kwargs2 = reader._get_loading_kwargs()
df2, metadata = reader.load_data(pl.DataFrame)
- assert PolarsCSVWriter.applicable_types() == [pl.DataFrame, pl.LazyFrame]
+ assert PolarsCSVWriter.applicable_types() == [pl.DataFrame]
assert PolarsCSVReader.applicable_types() == [pl.DataFrame]
assert kwargs1["separator"] == ","
assert kwargs2["has_header"] is True
@@ -85,7 +85,7 @@ def test_polars_parquet(df: pl.DataFrame, tmp_path: pathlib.Path) -> None:
kwargs2 = reader._get_loading_kwargs()
df2, metadata = reader.load_data(pl.DataFrame)
- assert PolarsParquetWriter.applicable_types() == [pl.DataFrame, pl.LazyFrame]
+ assert PolarsParquetWriter.applicable_types() == [pl.DataFrame]
assert PolarsParquetReader.applicable_types() == [pl.DataFrame]
assert kwargs1["compression"] == "zstd"
assert kwargs2["n_rows"] == 2
@@ -107,7 +107,7 @@ def test_polars_feather(tmp_path: pathlib.Path) -> None:
assert "n_rows" not in read_kwargs
assert df.shape == (4, 3)
- assert PolarsFeatherWriter.applicable_types() == [pl.DataFrame, pl.LazyFrame]
+ assert PolarsFeatherWriter.applicable_types() == [pl.DataFrame]
assert "compression" in write_kwargs
assert file_path.exists()
assert metadata["file_metadata"]["path"] == str(file_path)
@@ -140,7 +140,7 @@ def test_polars_ndjson(df: pl.DataFrame, tmp_path: pathlib.Path) -> None:
kwargs2 = reader._get_loading_kwargs()
df2, metadata = reader.load_data(pl.DataFrame)
- assert PolarsNDJSONWriter.applicable_types() == [pl.DataFrame, pl.LazyFrame]
+ assert PolarsNDJSONWriter.applicable_types() == [pl.DataFrame]
assert PolarsNDJSONReader.applicable_types() == [pl.DataFrame]
assert df2.shape == (2, 2)
assert "schema" not in kwargs2
diff --git a/tests/plugins/test_polars_lazyframe_extensions.py b/tests/plugins/test_polars_lazyframe_extensions.py
index b909584e..c4b6a741 100644
--- a/tests/plugins/test_polars_lazyframe_extensions.py
+++ b/tests/plugins/test_polars_lazyframe_extensions.py
@@ -23,6 +23,9 @@ import pytest
from polars.testing import assert_frame_equal
from sqlalchemy import create_engine
+from hamilton import ad_hoc_utils, driver, registry
+from hamilton.function_modifiers.adapters import resolve_adapter_class
+from hamilton.io.materialization import to
from hamilton.plugins.polars_lazyframe_extensions import (
PolarsScanCSVReader,
PolarsScanFeatherReader,
@@ -35,18 +38,24 @@ from hamilton.plugins.polars_lazyframe_extensions import (
from hamilton.plugins.polars_post_1_0_0_extensions import (
PolarsAvroReader,
PolarsAvroWriter,
- PolarsCSVWriter,
PolarsDatabaseReader,
PolarsDatabaseWriter,
PolarsFeatherWriter,
PolarsJSONReader,
PolarsJSONWriter,
PolarsNDJSONReader,
- PolarsNDJSONWriter,
- PolarsParquetWriter,
PolarsSpreadsheetReader,
PolarsSpreadsheetWriter,
)
+from hamilton.plugins.polars_pre_1_0_0_extension import (
+ PolarsCSVWriter as Pre1PolarsCSVWriter,
+)
+from hamilton.plugins.polars_pre_1_0_0_extension import (
+ PolarsFeatherWriter as Pre1PolarsFeatherWriter,
+)
+from hamilton.plugins.polars_pre_1_0_0_extension import (
+ PolarsParquetWriter as Pre1PolarsParquetWriter,
+)
@pytest.fixture
@@ -57,7 +66,7 @@ def df():
def test_lazy_polars_lazyframe_csv(df: pl.LazyFrame, tmp_path: pathlib.Path) -> None:
file = tmp_path / "test.csv"
- writer = PolarsCSVWriter(file=file)
+ writer = PolarsSinkCSVWriter(file=file)
kwargs1 = writer._get_saving_kwargs()
writer.save_data(df)
@@ -65,7 +74,7 @@ def test_lazy_polars_lazyframe_csv(df: pl.LazyFrame, tmp_path: pathlib.Path) ->
kwargs2 = reader._get_loading_kwargs()
df2, _metadata = reader.load_data(pl.LazyFrame)
- assert PolarsCSVWriter.applicable_types() == [pl.DataFrame, pl.LazyFrame]
+ assert PolarsSinkCSVWriter.applicable_types() == [pl.LazyFrame]
assert PolarsScanCSVReader.applicable_types() == [pl.LazyFrame]
assert kwargs1["separator"] == ","
assert kwargs2["has_header"] is True
@@ -75,7 +84,7 @@ def test_lazy_polars_lazyframe_csv(df: pl.LazyFrame, tmp_path: pathlib.Path) ->
def test_lazy_polars_parquet(df: pl.LazyFrame, tmp_path: pathlib.Path) -> None:
file = tmp_path / "test.parquet"
- writer = PolarsParquetWriter(file=file)
+ writer = PolarsSinkParquetWriter(file=file)
kwargs1 = writer._get_saving_kwargs()
writer.save_data(df)
@@ -83,7 +92,7 @@ def test_lazy_polars_parquet(df: pl.LazyFrame, tmp_path: pathlib.Path) -> None:
kwargs2 = reader._get_loading_kwargs()
df2, _metadata = reader.load_data(pl.LazyFrame)
- assert PolarsParquetWriter.applicable_types() == [pl.DataFrame, pl.LazyFrame]
+ assert PolarsSinkParquetWriter.applicable_types() == [pl.LazyFrame]
assert PolarsScanParquetReader.applicable_types() == [pl.LazyFrame]
assert kwargs1["compression"] == "zstd"
assert kwargs2["n_rows"] == 2
@@ -105,7 +114,7 @@ def test_lazy_polars_feather(tmp_path: pathlib.Path) -> None:
assert "n_rows" not in read_kwargs
assert df.collect().shape == (4, 3)
- assert PolarsFeatherWriter.applicable_types() == [pl.DataFrame, pl.LazyFrame]
+ assert PolarsFeatherWriter.applicable_types() == [pl.DataFrame]
assert "compression" in write_kwargs
assert file_path.exists()
assert metadata["file_metadata"]["path"] == str(file_path)
@@ -153,14 +162,14 @@ def test_polars_json(df: pl.LazyFrame, tmp_path: pathlib.Path) -> None:
def test_polars_ndjson(df: pl.LazyFrame, tmp_path: pathlib.Path) -> None:
file = tmp_path / "test.ndjson"
- writer = PolarsNDJSONWriter(file=file)
+ writer = PolarsSinkNDJSONWriter(file=file)
writer.save_data(df)
reader = PolarsNDJSONReader(source=file)
kwargs2 = reader._get_loading_kwargs()
df2, _metadata = reader.load_data(pl.DataFrame)
- assert PolarsNDJSONWriter.applicable_types() == [pl.DataFrame, pl.LazyFrame]
+ assert PolarsSinkNDJSONWriter.applicable_types() == [pl.LazyFrame]
assert PolarsNDJSONReader.applicable_types() == [pl.DataFrame]
assert df2.shape == (2, 2)
assert "schema" not in kwargs2
@@ -219,14 +228,15 @@ def test_polars_spreadsheet(df: pl.LazyFrame, tmp_path: pathlib.Path) -> None:
def test_polars_lazyframe_sink_parquet(df: pl.LazyFrame, tmp_path: pathlib.Path) -> None:
file = tmp_path / "test.parquet"
- sink = PolarsSinkParquetWriter(path=file)
+ sink = PolarsSinkParquetWriter(file=file)
kwargs = sink._get_saving_kwargs()
- sink.save_data(df)
+ metadata = sink.save_data(df)
df2 = pl.read_parquet(file)
assert PolarsSinkParquetWriter.applicable_types() == [pl.LazyFrame]
assert file.exists()
assert kwargs["compression"] == "zstd"
+ assert metadata["file_metadata"]["path"] == str(file)
assert_frame_equal(df.collect(), df2)
@@ -235,7 +245,7 @@ def test_polars_lazyframe_sink_parquet_custom_kwargs(
) -> None:
"""Test that non-default kwargs are passed through correctly."""
file = tmp_path / "test.parquet"
- sink = PolarsSinkParquetWriter(path=file, compression="snappy", maintain_order=False)
+ sink = PolarsSinkParquetWriter(file=file, compression="snappy", maintain_order=False)
kwargs = sink._get_saving_kwargs()
sink.save_data(df)
df2 = pl.read_parquet(file)
@@ -243,27 +253,28 @@ def test_polars_lazyframe_sink_parquet_custom_kwargs(
assert kwargs["compression"] == "snappy"
assert kwargs["maintain_order"] is False
assert file.exists()
- assert_frame_equal(df.collect(), df2)
+ assert_frame_equal(df.collect().sort(["a", "b"]), df2.sort(["a", "b"]))
def test_polars_lazyframe_sink_csv(df: pl.LazyFrame, tmp_path: pathlib.Path) -> None:
file = tmp_path / "test.csv"
- sink = PolarsSinkCSVWriter(path=file)
+ sink = PolarsSinkCSVWriter(file=file)
kwargs = sink._get_saving_kwargs()
- sink.save_data(df)
+ metadata = sink.save_data(df)
df2 = pl.read_csv(file)
assert PolarsSinkCSVWriter.applicable_types() == [pl.LazyFrame]
assert file.exists()
assert kwargs["separator"] == ","
assert kwargs["include_header"] is True
+ assert metadata["file_metadata"]["path"] == str(file)
assert_frame_equal(df.collect(), df2)
def test_polars_lazyframe_sink_csv_custom_kwargs(df: pl.LazyFrame, tmp_path: pathlib.Path) -> None:
"""Test that non-default kwargs are passed through correctly."""
file = tmp_path / "test.csv"
- sink = PolarsSinkCSVWriter(path=file, separator=";", include_header=False)
+ sink = PolarsSinkCSVWriter(file=file, separator=";", include_header=False)
kwargs = sink._get_saving_kwargs()
sink.save_data(df)
df2 = pl.read_csv(file, separator=";", has_header=False, new_columns=["a", "b"])
@@ -276,21 +287,22 @@ def test_polars_lazyframe_sink_csv_custom_kwargs(df: pl.LazyFrame, tmp_path: pat
def test_polars_lazyframe_sink_ipc(df: pl.LazyFrame, tmp_path: pathlib.Path) -> None:
file = tmp_path / "test.ipc"
- sink = PolarsSinkFeatherWriter(path=file)
+ sink = PolarsSinkFeatherWriter(file=file)
kwargs = sink._get_saving_kwargs()
- sink.save_data(df)
+ metadata = sink.save_data(df)
df2 = pl.read_ipc(file)
assert PolarsSinkFeatherWriter.applicable_types() == [pl.LazyFrame]
assert file.exists()
assert kwargs["compression"] == "uncompressed"
+ assert metadata["file_metadata"]["path"] == str(file)
assert_frame_equal(df.collect(), df2)
def test_polars_lazyframe_sink_ipc_custom_kwargs(df: pl.LazyFrame, tmp_path: pathlib.Path) -> None:
"""Test that non-default kwargs are passed through correctly."""
file = tmp_path / "test.ipc"
- sink = PolarsSinkFeatherWriter(path=file, compression="lz4", maintain_order=False)
+ sink = PolarsSinkFeatherWriter(file=file, compression="lz4", maintain_order=False)
kwargs = sink._get_saving_kwargs()
sink.save_data(df)
df2 = pl.read_ipc(file)
@@ -298,19 +310,20 @@ def test_polars_lazyframe_sink_ipc_custom_kwargs(df: pl.LazyFrame, tmp_path: pat
assert kwargs["compression"] == "lz4"
assert kwargs["maintain_order"] is False
assert file.exists()
- assert_frame_equal(df.collect(), df2)
+ assert_frame_equal(df.collect().sort(["a", "b"]), df2.sort(["a", "b"]))
def test_polars_lazyframe_sink_ndjson(df: pl.LazyFrame, tmp_path: pathlib.Path) -> None:
file = tmp_path / "test.ndjson"
- sink = PolarsSinkNDJSONWriter(path=file)
+ sink = PolarsSinkNDJSONWriter(file=file)
kwargs = sink._get_saving_kwargs()
- sink.save_data(df)
+ metadata = sink.save_data(df)
df2 = pl.read_ndjson(file)
assert PolarsSinkNDJSONWriter.applicable_types() == [pl.LazyFrame]
assert file.exists()
assert kwargs["maintain_order"] is True
+ assert metadata["file_metadata"]["path"] == str(file)
assert_frame_equal(df.collect(), df2)
@@ -319,14 +332,14 @@ def test_polars_lazyframe_sink_ndjson_custom_kwargs(
) -> None:
"""Test that non-default kwargs are passed through correctly."""
file = tmp_path / "test.ndjson"
- sink = PolarsSinkNDJSONWriter(path=file, maintain_order=False)
+ sink = PolarsSinkNDJSONWriter(file=file, maintain_order=False)
kwargs = sink._get_saving_kwargs()
sink.save_data(df)
df2 = pl.read_ndjson(file)
assert kwargs["maintain_order"] is False
assert file.exists()
- assert_frame_equal(df.collect(), df2)
+ assert_frame_equal(df.collect().sort(["a", "b"]), df2.sort(["a", "b"]))
def test_polars_lazyframe_sink_feather_adapter_name() -> None:
@@ -347,3 +360,69 @@ def test_polars_lazyframe_sink_parquet_adapter_name() -> None:
def test_polars_lazyframe_sink_ndjson_adapter_name() -> None:
"""Test that PolarsSinkNDJSONWriter is registered under 'ndjson'."""
assert PolarsSinkNDJSONWriter.name() == "ndjson"
+
+
+@pytest.mark.parametrize(
+ ("adapter_name", "expected_saver"),
+ [
+ ("csv", PolarsSinkCSVWriter),
+ ("parquet", PolarsSinkParquetWriter),
+ ("feather", PolarsSinkFeatherWriter),
+ ("ndjson", PolarsSinkNDJSONWriter),
+ ],
+)
+def test_lazyframe_sink_registry_resolution(adapter_name, expected_saver) -> None:
+ """Each format resolves one LazyFrame saver, independent of registration order."""
+ registered_savers = registry.SAVER_REGISTRY[adapter_name]
+ applicable_savers = [saver for saver in registered_savers if saver.applies_to(pl.LazyFrame)]
+
+ assert applicable_savers == [expected_saver]
+ assert resolve_adapter_class(pl.LazyFrame, registered_savers) is expected_saver
+ assert resolve_adapter_class(pl.LazyFrame, list(reversed(registered_savers))) is expected_saver
+
+
+@pytest.mark.parametrize(
+ ("eager_saver", "streaming_saver"),
+ [
+ (Pre1PolarsCSVWriter, PolarsSinkCSVWriter),
+ (Pre1PolarsParquetWriter, PolarsSinkParquetWriter),
+ (Pre1PolarsFeatherWriter, PolarsSinkFeatherWriter),
+ ],
+)
+def test_pre_1_0_lazyframe_sink_resolution_is_unambiguous(eager_saver, streaming_saver) -> None:
+ assert eager_saver.applicable_types() == [pl.DataFrame]
+ assert resolve_adapter_class(pl.LazyFrame, [eager_saver, streaming_saver]) is streaming_saver
+ assert resolve_adapter_class(pl.LazyFrame, [streaming_saver, eager_saver]) is streaming_saver
+
+
+def test_lazyframe_sink_csv_complete_kwargs(tmp_path: pathlib.Path) -> None:
+ sink = PolarsSinkCSVWriter(
+ file=tmp_path / "output.csv",
+ include_bom=True,
+ decimal_comma=True,
+ storage_options={"key": "value"},
+ extra_kwargs={"retries": 7},
+ )
+
+ assert sink._get_saving_kwargs()["include_bom"] is True
+ assert sink._get_saving_kwargs()["decimal_comma"] is True
+ assert sink._get_saving_kwargs()["storage_options"] == {"key": "value"}
+ assert sink._get_saving_kwargs()["retries"] == 7
+
+
+def test_lazyframe_sink_materializer_preserves_file_argument(tmp_path: pathlib.Path) -> None:
+ """The streaming saver must remain compatible with the existing Polars ``file`` API."""
+
+ def lazy_data() -> pl.LazyFrame:
+ return pl.LazyFrame({"a": [1, 2], "b": [3, 4]})
+
+ output_file = tmp_path / "output.csv"
+ module = ad_hoc_utils.create_temporary_module(lazy_data)
+ dr = driver.Driver({}, module)
+
+ materialization_result, _ = dr.materialize(
+ to.csv(id="save_lazy_data", dependencies=["lazy_data"], file=output_file)
+ )
+
+ assert materialization_result["save_lazy_data"]["file_metadata"]["path"] == str(output_file)
+ assert_frame_equal(pl.read_csv(output_file), lazy_data().collect())