You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2026/07/19 06:10:47 UTC
(iceberg-python) branch main updated: Implement `ParquetFormatModel` and update `write_file` to use the format API (#3381)
This is an automated email from the ASF dual-hosted git repository.
Fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 921e1118f Implement `ParquetFormatModel` and update `write_file` to use the format API (#3381)
921e1118f is described below
commit 921e1118f7bd45b22e59144d5295b7cea30d3bee
Author: Neelesh Salian <ns...@users.noreply.github.com>
AuthorDate: Sat Jul 18 23:10:42 2026 -0700
Implement `ParquetFormatModel` and update `write_file` to use the format API (#3381)
Continued work on https://github.com/apache/iceberg-python/issues/3100
## PR Description
Follow-up to #3119. Implements `ParquetFormatWriter` and
`ParquetFormatModel`, registers Parquet in the `FileFormatFactory`, and
rewrites `write_file` to dispatch through the factory using the
`write.format.default` table property. Future formats can be added in a
similar way.
## Rationale for this change
The `write.format.default` table property was never read - the write
path was hardcoded to Parquet. This PR makes the property functional.
Also threads `file_format` through `_to_requested_schema` /
`ArrowProjectionVisitor` / `_construct_field` so field ID metadata keys
are correct per format (`PARQUET:field_id` for Parquet, `iceberg.id`
plus `iceberg.required` for ORC), preparing the write path for ORC
support without changing default behavior.
## Are these changes tested?
- `tests/io/test_format_writers.py` adds parametrized tests modeled
after Java's `BaseFormatModelTests` covering round-trip, statistics,
null handling, context manager caching, close idempotency,
close-without-write, and ORC vs Parquet field ID dispatch.
- `tests/io/test_pyarrow.py` adds `test_write_file_parquet_round_trip`
and `test_write_file_dispatches_on_write_format_default` exercising the
full `write_file` path.
## Are there any user-facing changes?
No. Default behavior is unchanged. Setting `write.format.default` to an
unregistered format now raises a `ValueError`.
---
pyiceberg/io/fileformat.py | 5 ++
pyiceberg/io/pyarrow.py | 133 +++++++++++++++++++++++++++++---------
tests/conftest.py | 14 ++++
tests/io/test_fileformat.py | 3 +
tests/io/test_format_writers.py | 140 ++++++++++++++++++++++++++++++++++++++++
tests/io/test_pyarrow.py | 58 ++++++++++++++++-
6 files changed, 322 insertions(+), 31 deletions(-)
diff --git a/pyiceberg/io/fileformat.py b/pyiceberg/io/fileformat.py
index 8ab280527..337e69860 100644
--- a/pyiceberg/io/fileformat.py
+++ b/pyiceberg/io/fileformat.py
@@ -28,6 +28,7 @@ from pyiceberg.manifest import FileFormat
from pyiceberg.partitioning import PartitionField, PartitionSpec, partition_record_value
from pyiceberg.schema import Schema
from pyiceberg.typedef import Properties, Record
+from pyiceberg.types import NestedField
if TYPE_CHECKING:
import pyarrow as pa
@@ -161,6 +162,10 @@ class FileFormatModel(ABC):
properties: Properties,
) -> FileFormatWriter: ...
+ @abstractmethod
+ def add_field_metadata(self, field: NestedField, metadata: dict[bytes, bytes], include_field_ids: bool) -> None:
+ """Add format-specific Arrow field metadata."""
+
class FileFormatFactory:
"""Registry of FileFormatModel implementations."""
diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py
index f05781d9b..29ede3ce9 100644
--- a/pyiceberg/io/pyarrow.py
+++ b/pyiceberg/io/pyarrow.py
@@ -43,6 +43,7 @@ from dataclasses import dataclass
from enum import Enum
from functools import lru_cache, singledispatch
from typing import (
+ IO,
TYPE_CHECKING,
Any,
Generic,
@@ -122,6 +123,7 @@ from pyiceberg.io import (
OutputStream,
)
from pyiceberg.io.fileformat import DataFileStatistics as DataFileStatistics
+from pyiceberg.io.fileformat import FileFormatFactory, FileFormatModel, FileFormatWriter
from pyiceberg.manifest import (
DataFile,
DataFileContent,
@@ -1899,6 +1901,7 @@ def _to_requested_schema(
include_field_ids: bool = False,
projected_missing_fields: dict[int, Any] = EMPTY_DICT,
allow_timestamp_tz_mismatch: bool = False,
+ format_model: FileFormatModel | None = None,
) -> pa.RecordBatch:
# We could reuse some of these visitors
struct_array = visit_with_partner(
@@ -1910,6 +1913,7 @@ def _to_requested_schema(
include_field_ids,
projected_missing_fields=projected_missing_fields,
allow_timestamp_tz_mismatch=allow_timestamp_tz_mismatch,
+ format_model=format_model,
),
ArrowAccessor(file_schema),
)
@@ -1922,6 +1926,7 @@ class ArrowProjectionVisitor(SchemaWithPartnerVisitor[pa.Array, pa.Array | None]
_downcast_ns_timestamp_to_us: bool
_projected_missing_fields: dict[int, Any]
_allow_timestamp_tz_mismatch: bool
+ _format_model: FileFormatModel | None
def __init__(
self,
@@ -1930,7 +1935,10 @@ class ArrowProjectionVisitor(SchemaWithPartnerVisitor[pa.Array, pa.Array | None]
include_field_ids: bool = False,
projected_missing_fields: dict[int, Any] = EMPTY_DICT,
allow_timestamp_tz_mismatch: bool = False,
+ format_model: FileFormatModel | None = None,
) -> None:
+ if include_field_ids and format_model is None:
+ raise ValueError("format_model is required when include_field_ids=True")
self._file_schema = file_schema
self._include_field_ids = include_field_ids
self._downcast_ns_timestamp_to_us = downcast_ns_timestamp_to_us
@@ -1938,6 +1946,7 @@ class ArrowProjectionVisitor(SchemaWithPartnerVisitor[pa.Array, pa.Array | None]
# When True, allows projecting timestamptz (UTC) to timestamp (no tz).
# Allowed for reading (aligns with Spark); disallowed for writing to enforce Iceberg spec's strict typing.
self._allow_timestamp_tz_mismatch = allow_timestamp_tz_mismatch
+ self._format_model = format_model
def _cast_if_needed(self, field: NestedField, values: pa.Array) -> pa.Array:
file_field = self._file_schema.find_field(field.field_id)
@@ -2001,13 +2010,11 @@ class ArrowProjectionVisitor(SchemaWithPartnerVisitor[pa.Array, pa.Array | None]
return values
def _construct_field(self, field: NestedField, arrow_type: pa.DataType) -> pa.Field:
- metadata = {}
+ metadata: dict[bytes, bytes] = {}
if field.doc:
- metadata[PYARROW_FIELD_DOC_KEY] = field.doc
- if self._include_field_ids:
- # For projection visitor, we don't know the file format, so default to Parquet
- # This is used for schema conversion during reads, not writes
- metadata[PYARROW_PARQUET_FIELD_ID_KEY] = str(field.field_id)
+ metadata[PYARROW_FIELD_DOC_KEY] = field.doc.encode()
+ if self._format_model is not None:
+ self._format_model.add_field_metadata(field, metadata, self._include_field_ids)
return pa.field(
name=field.name,
@@ -2627,21 +2634,93 @@ def data_file_statistics_from_parquet_metadata(
)
+class ParquetFormatWriter(FileFormatWriter):
+ """Writes Arrow tables to a Parquet file."""
+
+ def __init__(self, output_file: OutputFile, file_schema: Schema, properties: Properties) -> None:
+ self._output_file = output_file
+ self._file_schema = file_schema
+ self._properties = properties
+ self._writer: pq.ParquetWriter | None = None
+ self._fos: OutputStream | None = None
+ self._parquet_writer_kwargs = _get_parquet_writer_kwargs(properties)
+ self._row_group_size = property_as_int(
+ properties=properties,
+ property_name=TableProperties.PARQUET_ROW_GROUP_LIMIT,
+ default=TableProperties.PARQUET_ROW_GROUP_LIMIT_DEFAULT,
+ )
+
+ def write(self, table: pa.Table) -> None:
+ if self._writer is None:
+ fos = self._output_file.create(overwrite=True)
+ try:
+ self._writer = pq.ParquetWriter(
+ cast(IO[Any], fos),
+ schema=table.schema,
+ store_decimal_as_integer=True,
+ **self._parquet_writer_kwargs,
+ )
+ except Exception:
+ fos.close()
+ raise
+ self._fos = fos
+ self._writer.write(table, row_group_size=self._row_group_size)
+
+ def close(self) -> DataFileStatistics:
+ if self._result is not None:
+ return self._result
+ if self._writer is None or self._fos is None:
+ raise ValueError("Cannot close a writer that was never written to")
+ with self._fos:
+ self._writer.close()
+ self._result = data_file_statistics_from_parquet_metadata(
+ parquet_metadata=self._writer.writer.metadata,
+ stats_columns=compute_statistics_plan(self._file_schema, self._properties),
+ parquet_column_mapping=parquet_path_to_id_mapping(self._file_schema),
+ )
+ return self._result
+
+
+class ParquetFormatModel(FileFormatModel):
+ """Format model for Apache Parquet."""
+
+ @property
+ def format(self) -> FileFormat:
+ return FileFormat.PARQUET
+
+ def file_extension(self) -> str:
+ return "parquet"
+
+ def create_writer(
+ self,
+ output_file: OutputFile,
+ file_schema: Schema,
+ properties: Properties,
+ ) -> ParquetFormatWriter:
+ return ParquetFormatWriter(output_file, file_schema, properties)
+
+ def add_field_metadata(self, field: NestedField, metadata: dict[bytes, bytes], include_field_ids: bool) -> None:
+ if include_field_ids:
+ metadata[PYARROW_PARQUET_FIELD_ID_KEY] = str(field.field_id).encode()
+
+
+FileFormatFactory.register(ParquetFormatModel())
+
+
def write_file(io: FileIO, table_metadata: TableMetadata, tasks: Iterator[WriteTask]) -> Iterator[DataFile]:
from pyiceberg.table import DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE, TableProperties
- parquet_writer_kwargs = _get_parquet_writer_kwargs(table_metadata.properties)
- row_group_size = property_as_int(
- properties=table_metadata.properties,
- property_name=TableProperties.PARQUET_ROW_GROUP_LIMIT,
- default=TableProperties.PARQUET_ROW_GROUP_LIMIT_DEFAULT,
+ file_format = FileFormat(
+ table_metadata.properties.get(
+ TableProperties.WRITE_FILE_FORMAT,
+ TableProperties.WRITE_FILE_FORMAT_DEFAULT,
+ )
)
+ format_model = FileFormatFactory.get(file_format)
location_provider = load_location_provider(table_location=table_metadata.location, table_properties=table_metadata.properties)
- def write_parquet(task: WriteTask) -> DataFile:
+ def write_data_file(task: WriteTask) -> DataFile:
table_schema = table_metadata.schema()
- # if schema needs to be transformed, use the transformed schema and adjust the arrow table accordingly
- # otherwise use the original schema
if (sanitized_schema := sanitize_column_names(table_schema)) != table_schema:
file_schema = sanitized_schema
else:
@@ -2655,29 +2734,25 @@ def write_file(io: FileIO, table_metadata: TableMetadata, tasks: Iterator[WriteT
batch=batch,
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
include_field_ids=True,
+ format_model=format_model,
)
for batch in task.record_batches
]
arrow_table = pa.Table.from_batches(batches)
file_path = location_provider.new_data_location(
- data_file_name=task.generate_data_file_filename("parquet"),
+ data_file_name=task.generate_data_file_filename(format_model.file_extension()),
partition_key=task.partition_key,
)
fo = io.new_output(file_path)
- with fo.create(overwrite=True) as fos:
- with pq.ParquetWriter(
- fos, schema=arrow_table.schema, store_decimal_as_integer=True, **parquet_writer_kwargs
- ) as writer:
- writer.write(arrow_table, row_group_size=row_group_size)
- statistics = data_file_statistics_from_parquet_metadata(
- parquet_metadata=writer.writer.metadata,
- stats_columns=compute_statistics_plan(file_schema, table_metadata.properties),
- parquet_column_mapping=parquet_path_to_id_mapping(file_schema),
- )
- data_file = DataFile.from_args(
+ writer = format_model.create_writer(fo, file_schema, table_metadata.properties)
+ with writer:
+ writer.write(arrow_table)
+ statistics = writer.result()
+
+ return DataFile.from_args(
content=DataFileContent.DATA,
file_path=file_path,
- file_format=FileFormat.PARQUET,
+ file_format=file_format,
partition=task.partition_key.partition if task.partition_key else Record(),
file_size_in_bytes=len(fo),
# After this has been fixed:
@@ -2691,10 +2766,8 @@ def write_file(io: FileIO, table_metadata: TableMetadata, tasks: Iterator[WriteT
**statistics.to_serialized_dict(),
)
- return data_file
-
executor = ExecutorFactory.get_or_create()
- data_files = executor.map(write_parquet, tasks)
+ data_files = executor.map(write_data_file, tasks)
return iter(data_files)
diff --git a/tests/conftest.py b/tests/conftest.py
index eb0b712d1..2b1f2366e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -172,6 +172,20 @@ def table_schema_simple() -> Schema:
)
+@pytest.fixture(scope="session")
+def arrow_table_simple() -> "pa.Table":
+ """Pyarrow table that pairs with `table_schema_simple` (3 rows, no nulls)."""
+ import pyarrow as pa
+
+ return pa.table(
+ {
+ "foo": ["a", "b", "c"],
+ "bar": pa.array([1, 2, 3], type=pa.int32()),
+ "baz": [True, False, True],
+ }
+ )
+
+
@pytest.fixture(scope="session")
def table_schema_with_full_nested_fields() -> Schema:
return Schema(
diff --git a/tests/io/test_fileformat.py b/tests/io/test_fileformat.py
index 8248e6b68..328fee927 100644
--- a/tests/io/test_fileformat.py
+++ b/tests/io/test_fileformat.py
@@ -51,6 +51,9 @@ def test_duplicate_registration_raises() -> None:
def create_writer(self, output_file: Any, file_schema: Any, properties: Any) -> Any:
raise NotImplementedError
+ def add_field_metadata(self, field: Any, metadata: Any, include_field_ids: bool) -> None:
+ pass
+
original = dict(FileFormatFactory._registry)
try:
model = _DummyModel()
diff --git a/tests/io/test_format_writers.py b/tests/io/test_format_writers.py
new file mode 100644
index 000000000..86ae65156
--- /dev/null
+++ b/tests/io/test_format_writers.py
@@ -0,0 +1,140 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Parametrized format writer tests, modeled after Java's BaseFormatModelTests."""
+
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.dataset as ds
+import pytest
+
+from pyiceberg.io.fileformat import FileFormatFactory, FileFormatModel
+from pyiceberg.io.pyarrow import PyArrowFileIO
+from pyiceberg.manifest import FileFormat
+from pyiceberg.schema import Schema
+from pyiceberg.types import LongType, NestedField
+
+
+@pytest.fixture(params=FileFormatFactory.available_formats(), ids=lambda f: f.name.lower())
+def format_model(request: pytest.FixtureRequest) -> FileFormatModel:
+ return FileFormatFactory.get(request.param)
+
+
+def test_parquet_registered() -> None:
+ """ParquetFormatModel is registered in the factory."""
+ model = FileFormatFactory.get(FileFormat.PARQUET)
+ assert model.format == FileFormat.PARQUET
+ assert model.file_extension() == "parquet"
+
+
+def test_round_trip(
+ format_model: FileFormatModel, table_schema_simple: Schema, arrow_table_simple: pa.Table, tmp_path: Path
+) -> None:
+ """Write a table and read it back, to verify equality and record count."""
+ file_path = str(tmp_path / f"test.{format_model.file_extension()}")
+ writer = format_model.create_writer(PyArrowFileIO().new_output(file_path), table_schema_simple, {})
+ writer.write(arrow_table_simple)
+ statistics = writer.close()
+
+ result = ds.dataset(file_path).to_table()
+ assert result.equals(arrow_table_simple)
+ assert statistics.record_count == 3
+
+
+def test_null_handling(format_model: FileFormatModel, table_schema_simple: Schema, tmp_path: Path) -> None:
+ """Nullable columns produce correct null_value_counts in statistics."""
+ table = pa.table(
+ {
+ "foo": ["a", None, "c"], # field_id=1, optional
+ "bar": pa.array([1, 2, 3], type=pa.int32()), # field_id=2, required
+ "baz": [True, False, True], # field_id=3, optional
+ }
+ )
+ file_path = str(tmp_path / f"test.{format_model.file_extension()}")
+ writer = format_model.create_writer(PyArrowFileIO().new_output(file_path), table_schema_simple, {})
+ writer.write(table)
+ stats = writer.close()
+ assert stats.record_count == 3
+ assert stats.null_value_counts.get(1) == 1
+
+
+def test_context_manager_caches_result(
+ format_model: FileFormatModel, table_schema_simple: Schema, arrow_table_simple: pa.Table, tmp_path: Path
+) -> None:
+ """writer.result() returns cached statistics after context manager exit."""
+ file_path = str(tmp_path / f"test.{format_model.file_extension()}")
+ writer = format_model.create_writer(PyArrowFileIO().new_output(file_path), table_schema_simple, {})
+ with writer:
+ writer.write(arrow_table_simple)
+ assert writer.result().record_count == 3
+
+
+def test_close_is_idempotent(
+ format_model: FileFormatModel, table_schema_simple: Schema, arrow_table_simple: pa.Table, tmp_path: Path
+) -> None:
+ """Calling close() twice returns the same cached statistics object."""
+ file_path = str(tmp_path / f"test.{format_model.file_extension()}")
+ writer = format_model.create_writer(PyArrowFileIO().new_output(file_path), table_schema_simple, {})
+ writer.write(arrow_table_simple)
+ stats1 = writer.close()
+ stats2 = writer.close()
+ assert stats1 is stats2
+
+
+def test_close_without_write_raises(format_model: FileFormatModel, table_schema_simple: Schema, tmp_path: Path) -> None:
+ """Closing a writer that was never written to raises ValueError."""
+ file_path = str(tmp_path / f"test.{format_model.file_extension()}")
+ writer = format_model.create_writer(PyArrowFileIO().new_output(file_path), table_schema_simple, {})
+ with pytest.raises(ValueError, match="Cannot close a writer that was never written to"):
+ writer.close()
+
+
+def test_parquet_writer_closes_output_stream_on_construction_failure(
+ table_schema_simple: Schema,
+ arrow_table_simple: pa.Table,
+ tmp_path: Path,
+) -> None:
+ """ParquetWriter construction failure inside write() closes the opened output stream."""
+ from unittest.mock import patch
+
+ from pyiceberg.io.pyarrow import ParquetFormatModel
+
+ file_path = str(tmp_path / "test.parquet")
+ writer = ParquetFormatModel().create_writer(PyArrowFileIO().new_output(file_path), table_schema_simple, {})
+
+ with patch("pyiceberg.io.pyarrow.pq.ParquetWriter", side_effect=RuntimeError("simulated failure")):
+ with pytest.raises(RuntimeError, match="simulated failure"):
+ writer.write(arrow_table_simple)
+
+ assert writer._writer is None
+ assert writer._fos is None
+
+
+def test_parquet_format_model_adds_field_id_metadata() -> None:
+ """ParquetFormatModel.add_field_metadata writes the Parquet field-id key when requested."""
+ from pyiceberg.io.pyarrow import PYARROW_PARQUET_FIELD_ID_KEY, ParquetFormatModel
+
+ field = NestedField(field_id=1, name="x", field_type=LongType(), required=True)
+
+ metadata: dict[bytes, bytes] = {}
+ ParquetFormatModel().add_field_metadata(field, metadata, include_field_ids=True)
+ assert metadata == {PYARROW_PARQUET_FIELD_ID_KEY: b"1"}
+
+ metadata_no_ids: dict[bytes, bytes] = {}
+ ParquetFormatModel().add_field_metadata(field, metadata_no_ids, include_field_ids=False)
+ assert metadata_no_ids == {}
diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py
index dde354735..8dba165c7 100644
--- a/tests/io/test_pyarrow.py
+++ b/tests/io/test_pyarrow.py
@@ -29,6 +29,7 @@ from uuid import uuid4
import pyarrow
import pyarrow as pa
+import pyarrow.dataset as ds
import pyarrow.orc as orc
import pyarrow.parquet as pq
import pytest
@@ -88,7 +89,7 @@ from pyiceberg.io.pyarrow import (
from pyiceberg.manifest import DataFile, DataFileContent, FileFormat
from pyiceberg.partitioning import PartitionField, PartitionSpec
from pyiceberg.schema import Schema, make_compatible_name, visit
-from pyiceberg.table import FileScanTask, TableProperties
+from pyiceberg.table import FileScanTask, TableProperties, WriteTask
from pyiceberg.table.metadata import TableMetadataV2
from pyiceberg.table.name_mapping import create_mapping_from_schema
from pyiceberg.transforms import HourTransform, IdentityTransform
@@ -3045,6 +3046,61 @@ def test_write_file_rejects_timestamptz_to_timestamp(tmp_path: Path) -> None:
list(write_file(io=PyArrowFileIO(), table_metadata=table_metadata, tasks=iter([task])))
+def _simple_write_task_and_metadata(
+ tmp_path: Path, table_schema_simple: Schema, properties: dict[str, str]
+) -> tuple[TableMetadataV2, WriteTask]:
+ """Build a TableMetadataV2 and a 3-row WriteTask matching table_schema_simple."""
+ arrow_data = pa.table(
+ {
+ "foo": ["a", "b", "c"],
+ "bar": pa.array([1, 2, 3], type=pa.int32()),
+ "baz": [True, False, True],
+ }
+ )
+ table_metadata = TableMetadataV2(
+ location=f"file://{tmp_path}",
+ last_column_id=3,
+ current_schema_id=1,
+ format_version=2,
+ schemas=[table_schema_simple],
+ partition_specs=[PartitionSpec()],
+ properties=properties,
+ )
+ task = WriteTask(
+ write_uuid=uuid.uuid4(),
+ task_id=0,
+ record_batches=arrow_data.to_batches(),
+ schema=table_schema_simple,
+ )
+ return table_metadata, task
+
+
+def test_write_file_dispatches_on_write_format_default(tmp_path: Path, table_schema_simple: Schema) -> None:
+ """write_file() reads write.format.default and raises if the format is not registered."""
+ table_metadata, task = _simple_write_task_and_metadata(tmp_path, table_schema_simple, {"write.format.default": "orc"})
+
+ with pytest.raises(ValueError, match="No writer registered for"):
+ list(write_file(io=PyArrowFileIO(), table_metadata=table_metadata, tasks=iter([task])))
+
+
+def test_write_file_parquet_round_trip(tmp_path: Path, table_schema_simple: Schema) -> None:
+ """write_file() with write.format.default=parquet writes a readable Parquet file with correct DataFile metadata."""
+ table_metadata, task = _simple_write_task_and_metadata(tmp_path, table_schema_simple, {"write.format.default": "parquet"})
+
+ data_files = list(write_file(io=PyArrowFileIO(), table_metadata=table_metadata, tasks=iter([task])))
+
+ assert len(data_files) == 1
+ data_file = data_files[0]
+ assert data_file.file_format == FileFormat.PARQUET
+ assert data_file.record_count == 3
+ assert data_file.file_path.endswith(".parquet")
+ assert data_file.file_size_in_bytes > 0
+
+ result = ds.dataset(data_file.file_path.replace("file://", "")).to_table()
+ assert result.num_rows == 3
+ assert result.column_names == ["foo", "bar", "baz"]
+
+
def test__to_requested_schema_timestamps(
arrow_table_schema_with_all_timestamp_precisions: pa.Schema,
arrow_table_with_all_timestamp_precisions: pa.Table,