You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2026/10/21 06:51:37 UTC
(sedona) branch master updated: [GH-3467] GeoPandas: implement read_feather (#3483)
This is an automated email from the ASF dual-hosted git repository.
jiayuasu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new be3109609ec [GH-3467] GeoPandas: implement read_feather (#3483)
be3109609ec is described below
commit be3109609ecea35e0b344b6c616bc1799140e8da
Author: Mayukh Das <69...@users.noreply.github.com>
AuthorDate: Wed Oct 21 17:51:31 2026 +1100
[GH-3467] GeoPandas: implement read_feather (#3483)
---
docs/api/geopandas-coverage.md | 10 +-
docs/tutorial/geopandas-api.md | 1 +
python/sedona/spark/geopandas/__init__.py | 2 +-
python/sedona/spark/geopandas/io.py | 87 +++++++++++++
python/tests/geopandas/test_io.py | 198 +++++++++++++++++++++++++++++-
5 files changed, 291 insertions(+), 7 deletions(-)
diff --git a/docs/api/geopandas-coverage.md b/docs/api/geopandas-coverage.md
index 63bf53a9829..2d35b3aa855 100644
--- a/docs/api/geopandas-coverage.md
+++ b/docs/api/geopandas-coverage.md
@@ -28,7 +28,7 @@ API names so they can be compared directly with the GeoPandas catalog.
## Coverage summary
-**91.4% API availability (160 of 175 catalogued APIs)**, including partial implementations.
+**92.6% API availability (162 of 175 catalogued APIs)**, including partial implementations.
Availability is the number of APIs marked **Available / partially available**
divided by the total. It measures whether a named API has an implementation, not
@@ -38,11 +38,11 @@ The notes below identify known restrictions.
| API group | Available / partially available | Unsupported |
| --- | ---: | ---: |
| GeoSeries | 119 | 4 |
-| GeoDataFrame | 26 | 7 |
-| Input/output | 3 | 2 |
+| GeoDataFrame | 27 | 6 |
+| Input/output | 4 | 1 |
| Tools | 6 | 2 |
| Spatial index | 6 | 0 |
-| Total | 160 | 15 |
+| Total | 162 | 13 |
### Status definitions
@@ -250,7 +250,7 @@ GeoPandas reference; they describe the target interface, not Sedona's support le
| [`geopandas.list_layers`](https://geopandas.org/en/v1.1.4/docs/reference/api/geopandas.list_layers.html) | Available / partially available | GeoPackage only; string/path-like inputs only. Reads layer metadata through Spark and collects the small layer listing to pandas. |
| [`geopandas.read_file`](https://geopandas.org/en/v1.1.4/docs/reference/api/geopandas.read_file.html) | Available / partially available | Reads Shapefile, GeoJSON, GeoPackage and GeoParquet only. GeoPandas bbox/mask/columns/rows/engine options are warned about and ignored; GeoPackage requires table_name rather than layer. |
| [`geopandas.read_postgis`](https://geopandas.org/en/v1.1.4/docs/reference/api/geopandas.read_postgis.html) | Unsupported | No implementation or public export in sedona.spark.geopandas. |
-| [`geopandas.read_feather`](https://geopandas.org/en/v1.1.4/docs/reference/api/geopandas.read_feather.html) | Unsupported | No implementation or public export in sedona.spark.geopandas. |
+| [`geopandas.read_feather`](https://geopandas.org/en/v1.1.4/docs/reference/api/geopandas.read_feather.html) | Available / partially available | Reads the file on the driver with GeoPandas and distributes the result, so the path must be readable from the driver; columns, to_pandas_kwargs and PyArrow read options are passed through. Counterpart of GeoDataFrame.to_feather. |
| [`geopandas.read_parquet`](https://geopandas.org/en/v1.1.4/docs/reference/api/geopandas.read_parquet.html) | Available / partially available | Reads GeoParquet through Spark. The columns, storage_options, bbox and to_pandas_kwargs parameters are accepted but silently ignored (no warning); column selection and bbox filtering are not applied. Extra kwargs trigger a warning. |
## Tools
diff --git a/docs/tutorial/geopandas-api.md b/docs/tutorial/geopandas-api.md
index acdf3d463e1..ffc86688546 100644
--- a/docs/tutorial/geopandas-api.md
+++ b/docs/tutorial/geopandas-api.md
@@ -343,6 +343,7 @@ The GeoPandas API for Apache Sedona implements the most commonly used GeoSeries
- `read_file()` - Read various geospatial formats
- `list_layers()` - List vector layers and nonspatial tables in a GeoPackage
- `to_parquet()` - Write to Parquet format
+- `read_feather()` and `to_feather()` - Read and write Feather files on the driver, for small files
### Spatial Operations
diff --git a/python/sedona/spark/geopandas/__init__.py b/python/sedona/spark/geopandas/__init__.py
index 2b60008d6bf..ac2a6e0ead5 100644
--- a/python/sedona/spark/geopandas/__init__.py
+++ b/python/sedona/spark/geopandas/__init__.py
@@ -26,7 +26,7 @@ from sedona.spark.geopandas.array import points_from_xy
from sedona.spark.geopandas.tools import clip, overlay, sjoin, sjoin_nearest
-from sedona.spark.geopandas.io import list_layers, read_file, read_parquet
+from sedona.spark.geopandas.io import list_layers, read_feather, read_file, read_parquet
# This used to default to False, but Spark 4.0.0 changed it to True
# We also want also it to default to True for Sedona, so we set it here
diff --git a/python/sedona/spark/geopandas/io.py b/python/sedona/spark/geopandas/io.py
index 7bf2744fea6..93c2ea0575a 100644
--- a/python/sedona/spark/geopandas/io.py
+++ b/python/sedona/spark/geopandas/io.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import inspect
import os
from typing import Union
import warnings
@@ -394,3 +395,89 @@ def read_parquet(
warnings.warn(f"The given arguments are not supported in Sedona: {kwargs}")
return read_file(path, format="geoparquet")
+
+
+def read_feather(path, columns=None, to_pandas_kwargs=None, **kwargs):
+ """
+ Load a Feather object from the file path, returning a GeoDataFrame.
+
+ .. versionadded:: 2.0.0
+
+ You can read a subset of columns in the file using the ``columns`` parameter.
+ However, the structure of the returned GeoDataFrame will depend on which
+ columns you read:
+
+ * if no geometry columns are read, this will raise a ``ValueError`` - you
+ should use the pandas `read_feather` method instead.
+ * if the primary geometry column saved to this file is not included in
+ columns, the first available geometry column will be set as the geometry
+ column of the returned GeoDataFrame.
+
+ If 'crs' key is not present in the Feather metadata associated with the
+ Feather object, it will default to "OGC:CRS84" according to the specification.
+
+ Note: Feather is a single-file format with no distributed reader, so this
+ reads the whole file into the driver's memory with GeoPandas and then
+ distributes it. It should only be used if the file is expected to be small.
+ This is the counterpart of :meth:`GeoDataFrame.to_feather`.
+
+ Requires 'pyarrow' >= 0.17.
+
+ Parameters
+ ----------
+ path : str, path object or file-like object
+ String, path object (implementing os.PathLike[str]) or file-like object
+ implementing a binary read() function. The path is read on the driver,
+ so it must be accessible from the driver.
+ columns : list-like of strings, default=None
+ If not None, only these columns will be read from the file. If
+ the primary geometry column is not included, the first secondary
+ geometry read from the file will be set as the geometry column
+ of the returned GeoDataFrame. If no geometry columns are present,
+ a ``ValueError`` will be raised.
+ to_pandas_kwargs : dict, optional
+ Arguments passed to the `pa.Table.to_pandas` method for non-geometry
+ columns. Requires a GeoPandas version whose ``read_feather`` accepts it.
+ **kwargs
+ Any additional kwargs passed to :func:`pyarrow.feather.read_table`.
+
+ Returns
+ -------
+ GeoDataFrame
+
+ Examples
+ --------
+ >>> from sedona.spark.geopandas import read_feather
+ >>> df = read_feather("data.feather") # doctest: +SKIP
+
+ Specifying columns to read:
+
+ >>> df = read_feather(
+ ... "data.feather",
+ ... columns=["geometry", "pop_est"]
+ ... ) # doctest: +SKIP
+
+ See Also
+ --------
+ GeoDataFrame.to_feather : write GeoDataFrame to Feather
+ read_parquet : read GeoParquet with Spark's distributed reader
+ """
+ import geopandas as gpd
+
+ if to_pandas_kwargs is not None:
+ # GeoPandas added to_pandas_kwargs to read_feather after its first releases.
+ if "to_pandas_kwargs" not in inspect.signature(gpd.read_feather).parameters:
+ raise NotImplementedError(
+ "to_pandas_kwargs is not supported by the installed GeoPandas version"
+ )
+ kwargs["to_pandas_kwargs"] = to_pandas_kwargs
+
+ # Feather has no distributed reader, so we rely on GeoPandas's implementation,
+ # the same way GeoDataFrame.to_feather() does. This also keeps GeoPandas's
+ # handling of the geo metadata: the active geometry column, CRS and index.
+ log_advice(
+ "`read_feather` loads all data into the driver's memory. "
+ "It should only be used if the file is expected to be small."
+ )
+ gpd_df = gpd.read_feather(path, columns=columns, **kwargs)
+ return GeoDataFrame._from_local_geopandas(gpd_df)
diff --git a/python/tests/geopandas/test_io.py b/python/tests/geopandas/test_io.py
index 387722afe68..b2727b0c19b 100644
--- a/python/tests/geopandas/test_io.py
+++ b/python/tests/geopandas/test_io.py
@@ -16,6 +16,7 @@
# under the License.
import glob
+import inspect
import json
import os
import sqlite3
@@ -25,11 +26,18 @@ import pytest
import shapely
import pandas as pd
import geopandas as gpd
+import pyarrow as pa
import pyarrow.parquet as pq
import pyspark.pandas as ps
import sedona.spark.geopandas as sgpd
from functools import partial
-from sedona.spark.geopandas import GeoDataFrame, GeoSeries, read_file, read_parquet
+from sedona.spark.geopandas import (
+ GeoDataFrame,
+ GeoSeries,
+ read_feather,
+ read_file,
+ read_parquet,
+)
from sedona.spark.geopandas._crs import read_crs_metadata
from tests import tests_resource
from tests.geopandas.test_geopandas_base import TestGeopandasBase
@@ -484,6 +492,194 @@ class TestIO(TestGeopandasBase):
else:
self.check_index_equal(gpd_df, pd.RangeIndex(start=0, stop=2, step=1))
+ #########################################################
+ # read_feather tests
+ #########################################################
+
+ def _write_gpd_feather(self, gpd_df, **kwargs):
+ temp_file_path = self._get_next_temp_file_path("feather")
+ gpd_df.to_feather(temp_file_path, **kwargs)
+ return temp_file_path
+
+ def test_read_feather(self):
+ gpd_df = gpd.GeoDataFrame(
+ {
+ "geometry": [
+ Point(0, 0),
+ LineString([(0, 0), (1, 1)]),
+ Polygon([(0, 0), (1, 0), (1, 1), (0, 0)]),
+ None,
+ ],
+ "int": [1, 2, 3, 4],
+ "str": ["a", "b", "c", "d"],
+ }
+ )
+ temp_file_path = self._write_gpd_feather(gpd_df)
+
+ sgpd_df = read_feather(temp_file_path)
+
+ assert isinstance(sgpd_df, GeoDataFrame)
+ self.check_sgpd_df_equals_gpd_df(sgpd_df, gpd.read_feather(temp_file_path))
+
+ def test_read_feather_leading_null_geometry(self):
+ # Spark 3 pandas-on-Spark infers the geometry type from the first value only
+ gpd_df = gpd.GeoDataFrame(
+ {"geometry": [None, Point(1, 2)], "int": [1, 2]}, crs="EPSG:4326"
+ )
+ temp_file_path = self._write_gpd_feather(gpd_df)
+
+ result = read_feather(temp_file_path)
+
+ assert result.crs == "EPSG:4326"
+ self.check_sgpd_df_equals_gpd_df(result, gpd.read_feather(temp_file_path))
+
+ def test_read_feather_is_exported(self):
+ assert sgpd.read_feather is read_feather
+
+ def test_read_feather_round_trips_to_feather(self):
+ sgpd_df = GeoDataFrame(
+ {"geometry": [Point(0, 0), LineString([(0, 0), (1, 1)])], "int": [1, 2]},
+ index=["a", "b"],
+ )
+ temp_file_path = self._get_next_temp_file_path("feather")
+ sgpd_df.to_feather(temp_file_path)
+
+ result = read_feather(temp_file_path)
+
+ self.check_sgpd_df_equals_gpd_df(result, sgpd_df.to_geopandas())
+
+ def test_read_feather_columns(self):
+ gpd_df = gpd.GeoDataFrame(
+ {"geometry": [Point(0, 0), Point(1, 1)], "int": [1, 2], "str": ["a", "b"]}
+ )
+ temp_file_path = self._write_gpd_feather(gpd_df)
+
+ result = read_feather(temp_file_path, columns=["geometry", "int"])
+
+ assert list(result.columns) == ["geometry", "int"]
+ self.check_sgpd_df_equals_gpd_df(result, gpd_df[["geometry", "int"]])
+
+ def test_read_feather_columns_without_primary_geometry(self):
+ # As in GeoPandas, the first geometry column read becomes the active one
+ gpd_df = gpd.GeoDataFrame(
+ {
+ "geometry": [Point(0, 0), Point(1, 1)],
+ "other": gpd.GeoSeries([Point(5, 5), Point(6, 6)]),
+ "int": [1, 2],
+ }
+ )
+ temp_file_path = self._write_gpd_feather(gpd_df)
+
+ result = read_feather(temp_file_path, columns=["int", "other"])
+ expected = gpd.read_feather(temp_file_path, columns=["int", "other"])
+
+ assert expected.geometry.name == "other"
+ assert result.geometry.name == "other"
+ self.check_sgpd_df_equals_gpd_df(result, expected)
+
+ def test_read_feather_without_geometry_columns_raises(self):
+ gpd_df = gpd.GeoDataFrame({"geometry": [Point(0, 0)], "int": [1]})
+ temp_file_path = self._write_gpd_feather(gpd_df)
+
+ with pytest.raises(ValueError):
+ read_feather(temp_file_path, columns=["int"])
+
+ def test_read_feather_missing_file_raises(self):
+ with pytest.raises((FileNotFoundError, OSError)):
+ read_feather(self._get_next_temp_file_path("feather"))
+
+ def test_read_feather_preserves_crs(self):
+ gpd_df = gpd.GeoDataFrame(
+ {"geometry": [Point(100000, 200000)], "int": [1]}, crs="EPSG:3857"
+ )
+ temp_file_path = self._write_gpd_feather(gpd_df)
+
+ assert read_feather(temp_file_path).crs == "EPSG:3857"
+
+ def test_read_feather_without_crs(self):
+ gpd_df = gpd.GeoDataFrame({"geometry": [Point(0, 0)], "int": [1]})
+ temp_file_path = self._write_gpd_feather(gpd_df)
+
+ assert read_feather(temp_file_path).crs is None
+
+ def test_read_feather_preserves_index(self):
+ gpd_df = gpd.GeoDataFrame(
+ {"geometry": [Point(0, 0), Point(1, 1)], "int": [1, 2]},
+ index=pd.Index(["a", "b"], name="key"),
+ )
+ temp_file_path = self._write_gpd_feather(gpd_df)
+
+ result = read_feather(temp_file_path)
+
+ pd.testing.assert_index_equal(
+ result.index.to_pandas().sort_values(), gpd_df.index
+ )
+
+ def test_read_feather_accepts_path_object(self):
+ import pathlib
+
+ gpd_df = gpd.GeoDataFrame({"geometry": [Point(0, 0)], "int": [1]})
+ temp_file_path = self._write_gpd_feather(gpd_df)
+
+ result = read_feather(pathlib.Path(temp_file_path))
+
+ self.check_sgpd_df_equals_gpd_df(result, gpd.read_feather(temp_file_path))
+
+ def test_read_feather_empty_file(self):
+ gpd_df = gpd.GeoDataFrame(
+ {
+ "geometry": gpd.GeoSeries([], dtype="geometry"),
+ "int": pd.Series([], dtype="int64"),
+ }
+ )
+ temp_file_path = self._write_gpd_feather(gpd_df)
+
+ result = read_feather(temp_file_path)
+
+ assert isinstance(result, GeoDataFrame)
+ assert len(result) == 0
+ assert list(result.columns) == ["geometry", "int"]
+
+ def test_read_feather_passes_kwargs_to_pyarrow(self):
+ gpd_df = gpd.GeoDataFrame({"geometry": [Point(0, 0)], "int": [1]})
+ temp_file_path = self._write_gpd_feather(gpd_df)
+
+ result = read_feather(temp_file_path, memory_map=True)
+
+ self.check_sgpd_df_equals_gpd_df(result, gpd.read_feather(temp_file_path))
+
+ @pytest.mark.skipif(
+ "to_pandas_kwargs" not in inspect.signature(gpd.read_feather).parameters,
+ reason="to_pandas_kwargs requires a newer GeoPandas",
+ )
+ def test_read_feather_to_pandas_kwargs(self):
+ gpd_df = gpd.GeoDataFrame(
+ {"geometry": [Point(0, 0), Point(1, 1)], "int": [1, 2]}
+ )
+ temp_file_path = self._write_gpd_feather(gpd_df)
+ to_pandas_kwargs = {"types_mapper": {pa.int64(): pd.Int32Dtype()}.get}
+
+ result = read_feather(temp_file_path, to_pandas_kwargs=to_pandas_kwargs)
+
+ assert str(result["int"].dtype) == "Int32"
+
+ def test_read_feather_to_pandas_kwargs_on_older_geopandas(self, monkeypatch):
+ gpd_df = gpd.GeoDataFrame({"geometry": [Point(0, 0)], "int": [1]})
+ temp_file_path = self._write_gpd_feather(gpd_df)
+ original = gpd.read_feather
+
+ def read_feather_without_to_pandas_kwargs(path, columns=None, **kwargs):
+ return original(path, columns=columns, **kwargs)
+
+ monkeypatch.setattr(gpd, "read_feather", read_feather_without_to_pandas_kwargs)
+
+ with pytest.raises(NotImplementedError, match="to_pandas_kwargs"):
+ read_feather(temp_file_path, to_pandas_kwargs={})
+ # Without to_pandas_kwargs, older GeoPandas versions still work
+ self.check_sgpd_df_equals_gpd_df(
+ read_feather(temp_file_path), original(temp_file_path)
+ )
+
def test_to_parquet_writes_crs_of_spark_backed_frame(self):
# The CRS is assigned on the frame while the geometries themselves carry SRID 0
spark_df = self.spark.sql(