You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2026/10/20 02:01:25 UTC
(sedona-db) branch main updated: fix(python/sedonadb-geopandas): compare missing values and signed zeros as pandas does (#1452)
This is an automated email from the ASF dual-hosted git repository.
jiayuasu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/sedona-db.git
The following commit(s) were added to refs/heads/main by this push:
new 9382f19fa fix(python/sedonadb-geopandas): compare missing values and signed zeros as pandas does (#1452)
9382f19fa is described below
commit 9382f19fab921840a9d9a8498543d64bf6651a4c
Author: Jia Yu <ji...@wherobots.com>
AuthorDate: Mon Oct 19 19:01:20 2026 -0700
fix(python/sedonadb-geopandas): compare missing values and signed zeros as pandas does (#1452)
---
.../python/sedonadb_geopandas/_series.py | 109 ++++++++++++++++---
.../tests/test_geopandas_compat.py | 115 +++++++++++++++++++++
2 files changed, 209 insertions(+), 15 deletions(-)
diff --git a/python/sedonadb-geopandas/python/sedonadb_geopandas/_series.py b/python/sedonadb-geopandas/python/sedonadb_geopandas/_series.py
index 9822f5a31..736f5fff1 100644
--- a/python/sedonadb-geopandas/python/sedonadb_geopandas/_series.py
+++ b/python/sedonadb-geopandas/python/sedonadb_geopandas/_series.py
@@ -18,9 +18,10 @@
import math
import numbers
+import operator
import pyarrow as pa
-from sedonadb.expr import lit
+from sedonadb.expr import Expr, lit
from sedonadb_geopandas._temporal import (
coerce_duration_scalar,
@@ -212,12 +213,7 @@ def _numeric_value(other):
raw = value._value
if isinstance(raw, int) and not -(2**63) <= raw < 2**63:
raise OverflowError(f"{raw} overflows the signed 64-bit range")
- arr = pa.array(value)
- if len(arr) != 1:
- raise ValueError(
- f"Can't use a Literal resolving to {len(arr)} values as a single value"
- )
- value = arr[0].as_py()
+ value = _literal_value(value).as_py()
if is_scalar(value):
value = normalize_scalar(value)
try:
@@ -228,6 +224,29 @@ def _numeric_value(other):
return value
+def _literal_value(literal):
+ """The one value `literal` holds, as an Arrow scalar, encodings unwrapped.
+
+ Read through the literal's Arrow form, since SedonaDB accepts one-element
+ containers (an Arrow array, a pandas Series or one-cell frame) as
+ single-value literals; a literal resolving to any other number of values
+ is rejected rather than read as its first.
+ """
+ import pyarrow.compute as pc
+
+ arr = pa.array(literal)
+ if len(arr) != 1:
+ raise ValueError(
+ f"Can't use a Literal resolving to {len(arr)} values as a single value"
+ )
+ while pa.types.is_dictionary(arr.type) or pa.types.is_run_end_encoded(arr.type):
+ if pa.types.is_dictionary(arr.type):
+ arr = arr.dictionary_decode()
+ else:
+ arr = pc.run_end_decode(arr)
+ return arr[0]
+
+
def _operand(df, other):
"""Coerce the right-hand side of an operator into something usable.
@@ -298,22 +317,52 @@ class Series:
# -- element-wise comparisons -> boolean mask --------------------------
def __gt__(self, other):
- return Series(self._df, self._expr > _operand(self._df, other), self._name)
+ return self._compare(other, operator.gt)
def __ge__(self, other):
- return Series(self._df, self._expr >= _operand(self._df, other), self._name)
+ return self._compare(other, operator.ge)
def __lt__(self, other):
- return Series(self._df, self._expr < _operand(self._df, other), self._name)
+ return self._compare(other, operator.lt)
def __le__(self, other):
- return Series(self._df, self._expr <= _operand(self._df, other), self._name)
+ return self._compare(other, operator.le)
def __eq__(self, other):
- return Series(self._df, self._expr == _operand(self._df, other), self._name)
+ return self._compare(other, operator.eq)
def __ne__(self, other):
- return Series(self._df, self._expr != _operand(self._df, other), self._name)
+ return self._compare(other, operator.ne)
+
+ def _compare(self, other, op):
+ """`self <op> other`, compared as pandas compares.
+
+ A missing value on either side (null, or NaN in a float column)
+ compares False, and True for `!=`, so the result is never null and a
+ mask built from it keeps or drops the rows pandas would. The engine
+ alone answers null for a null, orders floats totally (NaN above every
+ number, -0.0 below 0.0), and so differs from pandas on all of these.
+ """
+ from sedonadb.expr import Literal
+ from sedonadb_geopandas._frame import _is_missing
+
+ rhs = _operand(self._df, other)
+ # A literal is missing if the value it holds is.
+ value = _literal_value(other) if isinstance(other, Literal) else other
+ if isinstance(other, Series):
+ missing = self._missing() | other._missing()
+ elif _is_missing(normalize_scalar(value)):
+ missing = self._expr.is_null() | lit(True)
+ else:
+ missing = self._missing()
+ expr = op(
+ _zero_normalized(self._df, self._expr), _zero_normalized(self._df, rhs)
+ )
+ if op is operator.ne:
+ expr = expr | missing
+ else:
+ expr = expr & ~missing
+ return Series(self._df, expr, self._name)
# -- boolean composition of masks --------------------------------------
def __and__(self, other):
@@ -465,7 +514,7 @@ class Series:
"""
expr = self._expr.is_null()
if self._is_floating():
- expr = expr | self._expr.funcs.isnan()
+ expr = expr | self._decoded().funcs.isnan()
return expr.funcs.coalesce(lit(True))
def isna(self):
@@ -545,7 +594,7 @@ class Series:
if matches_null:
expr = expr | self._expr.is_null()
if matches_nan:
- expr = expr | self._expr.funcs.isnan().funcs.coalesce(lit(False))
+ expr = expr | self._decoded().funcs.isnan().funcs.coalesce(lit(False))
return Series(self._df, expr, self._name)
def astype(self, dtype):
@@ -774,6 +823,36 @@ class Series:
return f"<{type(self).__name__} {self._expr!r} (lazy; call .to_pandas())>"
+def _zero_normalized(df, expr):
+ """`expr` (a column expression or a scalar), with -0.0 as 0.0 if it is
+ floating.
+
+ The engine orders floats totally, so -0.0 compares below 0.0, while
+ pandas (IEEE) holds them equal; adding 0.0 turns -0.0 into 0.0. An
+ encoded column is decoded first, since the encoding hides the float.
+ """
+ from sedonadb.expr import Literal
+
+ if isinstance(expr, float):
+ return expr + 0.0
+ if isinstance(expr, pa.Scalar):
+ if pa.types.is_floating(expr.type) and expr.is_valid:
+ return pa.scalar(expr.as_py() + 0.0, type=expr.type)
+ return expr
+ if isinstance(expr, Literal):
+ value = _literal_value(expr)
+ if pa.types.is_floating(value.type):
+ return Literal(_zero_normalized(df, value), expr._ctx)
+ return expr
+ if not isinstance(expr, Expr):
+ return expr
+ stored = pa.schema(df.select(expr.alias("x")).schema).field("x").type
+ dtype = _logical_type(stored)
+ if not pa.types.is_floating(dtype):
+ return expr
+ return (expr if dtype == stored else expr.cast(dtype)) + lit(0.0)
+
+
def _buffer_style(style, names, argument):
"""A buffer style as a GEOS parameter name.
diff --git a/python/sedonadb-geopandas/tests/test_geopandas_compat.py b/python/sedonadb-geopandas/tests/test_geopandas_compat.py
index b91d7028c..ff2dd0227 100644
--- a/python/sedonadb-geopandas/tests/test_geopandas_compat.py
+++ b/python/sedonadb-geopandas/tests/test_geopandas_compat.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import operator
import sys
import geoarrow.pyarrow as ga
@@ -150,6 +151,120 @@ def test_filter_matches_geopandas(cities):
assert_geopandas_expr_equal(cities, lambda df: df[df["pop"] > 150], sort_by="name")
+_COMPARED = """
+SELECT f, g, CAST(i AS BIGINT) AS i, s, CAST(t AS TIMESTAMP) AS t,
+ ST_Point(0, 0) AS geometry
+FROM (VALUES
+ (1.5, 1.5, 1, 'a', '2020-01-01'),
+ (CAST('NaN' AS DOUBLE), CAST('NaN' AS DOUBLE), 2, 'b', '2020-01-02'),
+ (NULL, 2.0, NULL, NULL, NULL),
+ (CAST('-0.0' AS DOUBLE), 0.0, 0, 'a', '2020-01-01')
+) AS v(f, g, i, s, t)
+"""
+
+
+@pytest.mark.parametrize("op", ["gt", "ge", "lt", "le", "eq", "ne"])
+@pytest.mark.parametrize(
+ "left, right",
+ [
+ ("f", 1.5),
+ ("f", 0),
+ ("f", -0.0),
+ ("f", np.nan),
+ ("f", None),
+ ("f", "g"),
+ ("i", 1),
+ ("i", "f"),
+ ("t", pd.Timestamp("2020-01-01")),
+ ],
+)
+def test_comparisons_follow_pandas(op, left, right):
+ # A missing value on either side (null, or NaN in a float column) compares
+ # False, and True for !=; -0.0 equals 0.0. The engine alone answers null
+ # for a null, puts NaN above every number, and -0.0 below 0.0.
+ gdf = GeoDataFrame(sgpd.default_context().sql(_COMPARED))
+ frame = gdf.to_geopandas()
+ ours_right = gdf[right] if right in ("f", "g") else right
+ theirs_right = frame[right] if right in ("f", "g") else right
+ got = getattr(operator, op)(gdf[left], ours_right).to_pandas().tolist()
+ assert got == getattr(operator, op)(frame[left], theirs_right).tolist()
+
+
+@pytest.mark.parametrize("op", ["gt", "ge", "lt", "le", "eq", "ne"])
+@pytest.mark.parametrize("value", [-0.0, 0.0, 1.5, np.nan, None])
+def test_comparisons_with_literals_follow_pandas(op, value):
+ # A lit() operand compares as the value it holds: its -0.0 equals 0.0,
+ # and a missing one compares False (True for !=).
+ gdf = GeoDataFrame(sgpd.default_context().sql(_COMPARED))
+ got = getattr(operator, op)(gdf["f"], lit(value)).to_pandas().tolist()
+ assert got == getattr(operator, op)(gdf.to_geopandas()["f"], value).tolist()
+
+
+@pytest.mark.parametrize("encode", ["dictionary", "run_end"])
+@pytest.mark.parametrize("value", [-0.0, 1.5, np.nan, None])
+def test_encoded_literals_compare_as_their_value(encode, value):
+ plain = pa.array([value], pa.float64())
+ if encode == "dictionary":
+ encoded = plain.dictionary_encode()
+ else:
+ encoded = pa.RunEndEncodedArray.from_arrays(pa.array([1], pa.int32()), plain)
+ gdf = GeoDataFrame(sgpd.default_context().sql(_COMPARED))
+ frame = gdf.to_geopandas()
+ for op in (operator.eq, operator.ne, operator.lt):
+ got = op(gdf["f"], lit(encoded)).to_pandas().tolist()
+ assert got == op(frame["f"], value).tolist()
+
+
+@pytest.mark.parametrize("values", [[-0.0, 1.0], []])
+def test_literals_of_other_than_one_value_are_rejected(values):
+ # Not compared against the first value, nor failing to find one.
+ gdf = GeoDataFrame(sgpd.default_context().sql(_COMPARED))
+ with pytest.raises(ValueError, match=f"resolving to {len(values)} values"):
+ gdf["f"] == lit(pa.array(values, pa.float64()))
+
+
+@pytest.mark.parametrize("type", [pa.float32(), pa.float64()])
+@pytest.mark.parametrize("encode", ["dictionary", "run_end"])
+def test_encoded_float_columns_compare_as_plain_ones(type, encode):
+ # isnan(), behind the missing-value checks, accepts no encoded type.
+ values = pa.array([1.0, np.nan, None, -0.0], type)
+ if encode == "dictionary":
+ encoded = values.dictionary_encode()
+ else:
+ ends = pa.array(range(1, len(values) + 1), pa.int32())
+ encoded = pa.RunEndEncodedArray.from_arrays(ends, values)
+ ctx = sgpd.default_context()
+ ours = GeoDataFrame(ctx.create_data_frame(pa.table({"x": encoded})))["x"]
+ plain = GeoDataFrame(ctx.create_data_frame(pa.table({"x": values})))["x"]
+ for f in [
+ lambda s: s == 1.0,
+ lambda s: s != 1.0,
+ lambda s: s <= 0.0,
+ lambda s: s == s,
+ lambda s: s.isna(),
+ ]:
+ assert f(ours).to_pandas().tolist() == f(plain).to_pandas().tolist()
+ assert ours.isin([np.nan]).to_pandas().tolist() == [False, True, True, False]
+
+
+def test_string_equality_with_missing_values():
+ gdf = GeoDataFrame(sgpd.default_context().sql(_COMPARED))
+ assert (gdf["s"] == "a").to_pandas().tolist() == [True, False, False, True]
+ assert (gdf["s"] != "a").to_pandas().tolist() == [False, True, True, False]
+
+
+def test_masks_with_missing_values_select_the_rows_pandas_does():
+ gdf = GeoDataFrame(sgpd.default_context().sql(_COMPARED))
+ frame = gdf.to_geopandas()
+ for ours, theirs in [
+ (gdf["f"] > 100, frame["f"] > 100),
+ (~(gdf["f"] > 100), ~(frame["f"] > 100)),
+ (gdf["f"] == gdf["f"], frame["f"] == frame["f"]),
+ ((gdf["f"] != 1.5) & (gdf["i"] > 0), (frame["f"] != 1.5) & (frame["i"] > 0)),
+ ]:
+ assert len(gdf[ours].to_geopandas()) == int(theirs.sum())
+
+
def test_to_crs_matches_geopandas(cities):
# Exercises CRS propagation through an operation (asserted via check_crs).
assert_geopandas_expr_equal(