You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by pa...@apache.org on 2025/12/01 15:30:48 UTC

(sedona-db) branch branch-0.2.0 updated (5d17f852 -> 69f89a9a)

This is an automated email from the ASF dual-hosted git repository.

paleolimbot pushed a change to branch branch-0.2.0
in repository https://gitbox.apache.org/repos/asf/sedona-db.git


    from 5d17f852 Remove deprecated mkdocstrings option
     new 2e52ef90 fix(rust/sedona-expr): Resolve filter expression bounding box by name and not by index (#384)
     new 62581633 fix(rust/sedona-expr): Fix GeoParquet pruning when number of final columns is less than the geometry column index (#385)
     new 69f89a9a fix(rust/sedona-geoparquet): Don't use ProjectionExec to create GeoParquet 1.1 bounding box columns (#398)

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 python/sedonadb/python/sedonadb/datasource.py |   5 +-
 python/sedonadb/src/datasource.rs             |   4 +-
 rust/sedona-datasource/src/spec.rs            |  10 +-
 rust/sedona-expr/src/spatial_filter.rs        | 320 ++++++++++++++++++--------
 rust/sedona-expr/src/statistics.rs            |  29 +--
 rust/sedona-geoparquet/src/file_opener.rs     |  19 +-
 rust/sedona-geoparquet/src/writer.rs          | 194 ++++++++++++++--
 7 files changed, 450 insertions(+), 131 deletions(-)


(sedona-db) 01/03: fix(rust/sedona-expr): Resolve filter expression bounding box by name and not by index (#384)

Posted by pa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

paleolimbot pushed a commit to branch branch-0.2.0
in repository https://gitbox.apache.org/repos/asf/sedona-db.git

commit 2e52ef90d150c1b44d8c0e7521f9546c905e6c6c
Author: Dewey Dunnington <de...@dunnington.ca>
AuthorDate: Fri Nov 28 15:38:43 2025 -0600

    fix(rust/sedona-expr): Resolve filter expression bounding box by name and not by index (#384)
---
 python/sedonadb/python/sedonadb/datasource.py |  5 ++++-
 python/sedonadb/src/datasource.rs             |  4 ++--
 rust/sedona-expr/src/spatial_filter.rs        | 26 +++++++++++++-------------
 3 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/python/sedonadb/python/sedonadb/datasource.py b/python/sedonadb/python/sedonadb/datasource.py
index 2171bedf..33acc72d 100644
--- a/python/sedonadb/python/sedonadb/datasource.py
+++ b/python/sedonadb/python/sedonadb/datasource.py
@@ -154,8 +154,11 @@ class PyogrioFormatSpec(ExternalFormatSpec):
 
         if args.filter and args.file_schema is not None:
             geometry_column_indices = args.file_schema.geometry_column_indices
+            file_columns = args.file_schema.names
             if len(geometry_column_indices) == 1:
-                bbox = args.filter.bounding_box(geometry_column_indices[0])
+                bbox = args.filter.bounding_box(
+                    file_columns[geometry_column_indices[0]]
+                )
             else:
                 bbox = None
         else:
diff --git a/python/sedonadb/src/datasource.rs b/python/sedonadb/src/datasource.rs
index 496a3000..348a6a02 100644
--- a/python/sedonadb/src/datasource.rs
+++ b/python/sedonadb/src/datasource.rs
@@ -282,10 +282,10 @@ pub struct PyFilter {
 impl PyFilter {
     fn bounding_box(
         &self,
-        column_index: usize,
+        column_name: &str,
     ) -> Result<Option<(f64, f64, f64, f64)>, PySedonaError> {
         let filter = SpatialFilter::try_from_expr(&self.inner)?;
-        let filter_bbox = filter.filter_bbox(column_index);
+        let filter_bbox = filter.filter_bbox(column_name);
         if filter_bbox.x().is_full() || filter_bbox.y().is_full() {
             Ok(None)
         } else {
diff --git a/rust/sedona-expr/src/spatial_filter.rs b/rust/sedona-expr/src/spatial_filter.rs
index 1160cf2c..733cd746 100644
--- a/rust/sedona-expr/src/spatial_filter.rs
+++ b/rust/sedona-expr/src/spatial_filter.rs
@@ -73,24 +73,24 @@ impl SpatialFilter {
     /// Note that this always succeeds; however, for a non-spatial expression or
     /// a non-spatial expression that is unsupported, the full bounding box is
     /// returned.
-    pub fn filter_bbox(&self, column_index: usize) -> BoundingBox {
+    pub fn filter_bbox(&self, column_name: &str) -> BoundingBox {
         match self {
             SpatialFilter::Intersects(column, bounding_box)
             | SpatialFilter::Covers(column, bounding_box) => {
-                if column.index() == column_index {
+                if column.name() == column_name {
                     return bounding_box.clone();
                 }
             }
             SpatialFilter::And(lhs, rhs) => {
-                let lhs_box = lhs.filter_bbox(column_index);
-                let rhs_box = rhs.filter_bbox(column_index);
+                let lhs_box = lhs.filter_bbox(column_name);
+                let rhs_box = rhs.filter_bbox(column_name);
                 if let Ok(bounds) = lhs_box.intersection(&rhs_box) {
                     return bounds;
                 }
             }
             SpatialFilter::Or(lhs, rhs) => {
-                let mut bounds = lhs.filter_bbox(column_index);
-                bounds.update_box(&rhs.filter_bbox(column_index));
+                let mut bounds = lhs.filter_bbox(column_name);
+                bounds.update_box(&rhs.filter_bbox(column_name));
                 return bounds;
             }
             SpatialFilter::LiteralFalse => {
@@ -1153,25 +1153,25 @@ mod test {
         let bbox_13 = BoundingBox::xy((1, 3), (1, 3));
 
         assert_eq!(
-            SpatialFilter::Intersects(col_zero.clone(), bbox_02.clone()).filter_bbox(0),
+            SpatialFilter::Intersects(col_zero.clone(), bbox_02.clone()).filter_bbox("foofy"),
             bbox_02
         );
 
         assert_eq!(
-            SpatialFilter::Covers(col_zero.clone(), bbox_02.clone()).filter_bbox(0),
+            SpatialFilter::Covers(col_zero.clone(), bbox_02.clone()).filter_bbox("foofy"),
             bbox_02
         );
 
         assert_eq!(
-            SpatialFilter::LiteralFalse.filter_bbox(0),
+            SpatialFilter::LiteralFalse.filter_bbox("foofy"),
             BoundingBox::xy(Interval::empty(), Interval::empty())
         );
         assert_eq!(
-            SpatialFilter::HasZ(col_zero.clone()).filter_bbox(0),
+            SpatialFilter::HasZ(col_zero.clone()).filter_bbox("foofy"),
             BoundingBox::xy(Interval::full(), Interval::full())
         );
         assert_eq!(
-            SpatialFilter::Unknown.filter_bbox(0),
+            SpatialFilter::Unknown.filter_bbox("foofy"),
             BoundingBox::xy(Interval::full(), Interval::full())
         );
 
@@ -1182,7 +1182,7 @@ mod test {
                 Box::new(intersects_02.clone()),
                 Box::new(intersects_13.clone())
             )
-            .filter_bbox(0),
+            .filter_bbox("foofy"),
             BoundingBox::xy((1, 2), (1, 2))
         );
 
@@ -1191,7 +1191,7 @@ mod test {
                 Box::new(intersects_02.clone()),
                 Box::new(intersects_13.clone())
             )
-            .filter_bbox(0),
+            .filter_bbox("foofy"),
             BoundingBox::xy((0, 3), (0, 3))
         );
     }


(sedona-db) 02/03: fix(rust/sedona-expr): Fix GeoParquet pruning when number of final columns is less than the geometry column index (#385)

Posted by pa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

paleolimbot pushed a commit to branch branch-0.2.0
in repository https://gitbox.apache.org/repos/asf/sedona-db.git

commit 625816335f6dfc188a0d3e8b747f38f04a6e1aeb
Author: Dewey Dunnington <de...@dunnington.ca>
AuthorDate: Sun Nov 30 20:21:38 2025 -0600

    fix(rust/sedona-expr): Fix GeoParquet pruning when number of final columns is less than the geometry column index (#385)
    
    Co-authored-by: Peter Nguyen <pe...@gmail.com>
    Co-authored-by: Copilot <17...@users.noreply.github.com>
---
 rust/sedona-datasource/src/spec.rs        |  10 +-
 rust/sedona-expr/src/spatial_filter.rs    | 294 ++++++++++++++++++++++--------
 rust/sedona-expr/src/statistics.rs        |  29 +--
 rust/sedona-geoparquet/src/file_opener.rs |  19 +-
 4 files changed, 255 insertions(+), 97 deletions(-)

diff --git a/rust/sedona-datasource/src/spec.rs b/rust/sedona-datasource/src/spec.rs
index d9b8f1af..2c2ca31a 100644
--- a/rust/sedona-datasource/src/spec.rs
+++ b/rust/sedona-datasource/src/spec.rs
@@ -141,7 +141,15 @@ pub struct OpenReaderArgs {
     /// Filter expressions
     ///
     /// Expressions that may be used for pruning. Implementations need not
-    /// apply these filters.
+    /// apply these filters to produce a correct result (i.e., DataFusion will
+    /// evaluate the filters at a later step regardless of how this implementation
+    /// uses the provided filters).
+    ///
+    /// Note that `Column`s in this [PhysicalExpr] are relative to `file_projection`.
+    /// For example, in a scan with  file_projection `[5, 6]` (i.e., DataFusion is only
+    /// requesting the 6th and 7th columns from the `file_schema` inferred for this object),
+    /// a `Column { index: 1, ... }` refers to the column at index 6 (i.e.,
+    /// `file_schema.field(file_projection[1])`).
     pub filters: Vec<Arc<dyn PhysicalExpr>>,
 }
 
diff --git a/rust/sedona-expr/src/spatial_filter.rs b/rust/sedona-expr/src/spatial_filter.rs
index 733cd746..922b8ded 100644
--- a/rust/sedona-expr/src/spatial_filter.rs
+++ b/rust/sedona-expr/src/spatial_filter.rs
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};
 
 use arrow_schema::{DataType, Schema};
 use datafusion_common::{DataFusionError, Result, ScalarValue};
@@ -30,7 +30,7 @@ use sedona_geometry::{
     bounds::wkb_bounds_xy,
     interval::{Interval, IntervalTrait},
 };
-use sedona_schema::{datatypes::SedonaType, matchers::ArgMatcher};
+use sedona_schema::{datatypes::SedonaType, matchers::ArgMatcher, schema::SedonaSchema};
 
 use crate::{
     statistics::GeoStatistics,
@@ -106,19 +106,24 @@ impl SpatialFilter {
     ///
     /// In other words, returns false if and only if the expression is guaranteed
     /// to be false.
-    pub fn evaluate(&self, table_stats: &[GeoStatistics]) -> bool {
+    pub fn evaluate(&self, table_stats: &TableGeoStatistics) -> Result<bool> {
+        self.evaluate_internal(table_stats)
+    }
+
+    fn evaluate_internal(&self, table_stats: &TableGeoStatistics) -> Result<bool> {
         match self {
-            SpatialFilter::Intersects(column, bounds) => {
-                Self::evaluate_intersects_bbox(&table_stats[column.index()], bounds)
-            }
+            SpatialFilter::Intersects(column, bounds) => Ok(Self::evaluate_intersects_bbox(
+                table_stats.get(column)?,
+                bounds,
+            )),
             SpatialFilter::Covers(column, bounds) => {
-                Self::evaluate_covers_bbox(&table_stats[column.index()], bounds)
+                Ok(Self::evaluate_covers_bbox(table_stats.get(column)?, bounds))
             }
-            SpatialFilter::HasZ(column) => Self::evaluate_has_z(&table_stats[column.index()]),
+            SpatialFilter::HasZ(column) => Ok(Self::evaluate_has_z(table_stats.get(column)?)),
             SpatialFilter::And(lhs, rhs) => Self::evaluate_and(lhs, rhs, table_stats),
             SpatialFilter::Or(lhs, rhs) => Self::evaluate_or(lhs, rhs, table_stats),
-            SpatialFilter::LiteralFalse => false,
-            SpatialFilter::Unknown => true,
+            SpatialFilter::LiteralFalse => Ok(false),
+            SpatialFilter::Unknown => Ok(true),
         }
     }
 
@@ -161,16 +166,16 @@ impl SpatialFilter {
         true
     }
 
-    fn evaluate_and(lhs: &Self, rhs: &Self, table_stats: &[GeoStatistics]) -> bool {
-        let maybe_lhs = lhs.evaluate(table_stats);
-        let maybe_rhs = rhs.evaluate(table_stats);
-        maybe_lhs && maybe_rhs
+    fn evaluate_and(lhs: &Self, rhs: &Self, table_stats: &TableGeoStatistics) -> Result<bool> {
+        let maybe_lhs = lhs.evaluate_internal(table_stats)?;
+        let maybe_rhs = rhs.evaluate_internal(table_stats)?;
+        Ok(maybe_lhs && maybe_rhs)
     }
 
-    fn evaluate_or(lhs: &Self, rhs: &Self, table_stats: &[GeoStatistics]) -> bool {
-        let maybe_lhs = lhs.evaluate(table_stats);
-        let maybe_rhs = rhs.evaluate(table_stats);
-        maybe_lhs || maybe_rhs
+    fn evaluate_or(lhs: &Self, rhs: &Self, table_stats: &TableGeoStatistics) -> Result<bool> {
+        let maybe_lhs = lhs.evaluate_internal(table_stats)?;
+        let maybe_rhs = rhs.evaluate_internal(table_stats)?;
+        Ok(maybe_lhs || maybe_rhs)
     }
 
     /// Construct a SpatialPredicate from a [PhysicalExpr]
@@ -389,6 +394,81 @@ impl SpatialFilter {
     }
 }
 
+/// Table GeoStatistics
+///
+/// Enables providing a collection of GeoStatistics to [SpatialFilter::evaluate]
+/// such that attempts to access out-of-bounds values results in a readable
+/// error.
+pub enum TableGeoStatistics {
+    /// Provide statistics for every Column in the table. These must be
+    /// [GeoStatistics::unspecified] for non-spatial columns.
+    ///
+    /// These are resolved using [Column::index].
+    ByPosition(Vec<GeoStatistics>),
+
+    /// Provide statistics for specific named columns. Columns not included
+    /// are treated as [GeoStatistics::unspecified].
+    ///
+    /// These are resolved using [Column::name]. This may be used for logical
+    /// expressions (where columns are resolved by name) or as a workaround
+    /// for physical expressions where the index is relative to a projected
+    /// schema <https://github.com/apache/sedona-db/issues/389>.
+    ByName(HashMap<String, GeoStatistics>),
+}
+
+impl TableGeoStatistics {
+    /// Construct TableGeoStatistics with no columns
+    pub fn empty() -> Self {
+        TableGeoStatistics::ByPosition(vec![])
+    }
+
+    /// Construct TableGeoStatistics from a slice of all column statistics and a schema
+    pub fn try_from_stats_and_schema(
+        column_stats: &[GeoStatistics],
+        schema: &Schema,
+    ) -> Result<Self> {
+        let mut stats_map = HashMap::new();
+        for i in schema.geometry_column_indices()? {
+            stats_map.insert(schema.field(i).name().to_string(), column_stats[i].clone());
+        }
+        Ok(Self::ByName(stats_map))
+    }
+
+    /// For a given [Column], obtain [GeoStatistics]
+    ///
+    /// This will error if the provided statistics have an index out of bounds.
+    /// Names that cannot be resolved will be treated as unspecified.
+    fn get(&self, column: &Column) -> Result<&GeoStatistics> {
+        match self {
+            Self::ByPosition(items) => {
+                if column.index() >= items.len() {
+                    sedona_internal_err!(
+                        "Can't obtain GeoStatistics for column at index {} from schema with {} columns",
+                        column.index(),
+                        items.len()
+                    )
+                } else {
+                    Ok(&items[column.index()])
+                }
+            }
+            Self::ByName(items) => {
+                if let Some(item) = items.get(column.name()) {
+                    Ok(item)
+                } else {
+                    Ok(&GeoStatistics::UNSPECIFIED)
+                }
+            }
+        }
+    }
+}
+
+// Useful for testing (create from a single GeoStatistics)
+impl From<GeoStatistics> for TableGeoStatistics {
+    fn from(value: GeoStatistics) -> Self {
+        TableGeoStatistics::ByPosition(vec![value])
+    }
+}
+
 /// Internal utility to help match physical expression types
 enum ArgRef<'a> {
     Col(Column),
@@ -492,24 +572,28 @@ mod test {
         );
         let bounds = literal_bounds(&literal).unwrap();
 
-        let stats_no_info = [GeoStatistics::unspecified()];
-        let stats_intersecting = [
-            GeoStatistics::unspecified().with_bbox(Some(BoundingBox::xy((0.5, 1.5), (1.5, 2.5))))
-        ];
-        let col0 = Column::new("col0", 0);
-
-        assert!(SpatialFilter::Intersects(col0.clone(), bounds.clone()).evaluate(&stats_no_info));
-        assert!(
-            SpatialFilter::Intersects(col0.clone(), bounds.clone()).evaluate(&stats_intersecting)
+        let stats_no_info = TableGeoStatistics::from(GeoStatistics::unspecified());
+        let stats_intersecting = TableGeoStatistics::from(
+            GeoStatistics::unspecified().with_bbox(Some(BoundingBox::xy((0.5, 1.5), (1.5, 2.5)))),
         );
+        let col0 = Column::new("col0", 0);
 
-        let stats_empty_bbox = [GeoStatistics::unspecified()
-            .with_bbox(Some(BoundingBox::xy(Interval::empty(), Interval::empty())))];
+        assert!(SpatialFilter::Intersects(col0.clone(), bounds.clone())
+            .evaluate(&stats_no_info)
+            .unwrap());
+        assert!(SpatialFilter::Intersects(col0.clone(), bounds.clone())
+            .evaluate(&stats_intersecting)
+            .unwrap());
 
-        assert!(
-            !SpatialFilter::Intersects(col0.clone(), bounds.clone()).evaluate(&stats_empty_bbox)
+        let stats_empty_bbox = TableGeoStatistics::from(
+            GeoStatistics::unspecified()
+                .with_bbox(Some(BoundingBox::xy(Interval::empty(), Interval::empty()))),
         );
 
+        assert!(!SpatialFilter::Intersects(col0.clone(), bounds.clone())
+            .evaluate(&stats_empty_bbox)
+            .unwrap());
+
         let unrelated_literal = Literal::new(ScalarValue::Null);
 
         let err = literal_bounds(&unrelated_literal).unwrap_err();
@@ -527,18 +611,25 @@ mod test {
         );
         let bounds = literal_bounds(&literal).unwrap();
 
-        let stats_no_info = [GeoStatistics::unspecified()];
-        let stats_covered =
-            [GeoStatistics::unspecified().with_bbox(Some(BoundingBox::xy((0, 4), (0, 4))))];
-        let stats_not_covered = [
-            GeoStatistics::unspecified().with_bbox(Some(BoundingBox::xy((3.0, 3.0), (5.0, 5.0))))
-        ];
+        let stats_no_info = TableGeoStatistics::from(GeoStatistics::unspecified());
+        let stats_covered = TableGeoStatistics::from(
+            GeoStatistics::unspecified().with_bbox(Some(BoundingBox::xy((0, 4), (0, 4)))),
+        );
+        let stats_not_covered = TableGeoStatistics::from(
+            GeoStatistics::unspecified().with_bbox(Some(BoundingBox::xy((3.0, 3.0), (5.0, 5.0)))),
+        );
         let col0 = Column::new("col0", 0);
 
         // Covers should return true when column bbox is fully contained in literal bounds
-        assert!(SpatialFilter::Covers(col0.clone(), bounds.clone()).evaluate(&stats_no_info));
-        assert!(SpatialFilter::Covers(col0.clone(), bounds.clone()).evaluate(&stats_covered));
-        assert!(!SpatialFilter::Covers(col0.clone(), bounds.clone()).evaluate(&stats_not_covered));
+        assert!(SpatialFilter::Covers(col0.clone(), bounds.clone())
+            .evaluate(&stats_no_info)
+            .unwrap());
+        assert!(SpatialFilter::Covers(col0.clone(), bounds.clone())
+            .evaluate(&stats_covered)
+            .unwrap());
+        assert!(!SpatialFilter::Covers(col0.clone(), bounds.clone())
+            .evaluate(&stats_not_covered)
+            .unwrap());
     }
 
     #[test]
@@ -546,73 +637,77 @@ mod test {
         let col0 = Column::new("col0", 0);
         let has_z = SpatialFilter::HasZ(col0.clone());
 
-        let stats_z_geometry_types = [GeoStatistics::unspecified()
-            .try_with_str_geometry_types(Some(&["POINT Z"]))
-            .unwrap()];
-        let stats_z_bbox = [
-            GeoStatistics::unspecified().with_bbox(Some(BoundingBox::xyzm(
-                (0, 1),
-                (2, 3),
-                Some((4, 5).into()),
-                None,
-            ))),
-        ];
-        let stats_no_info = [GeoStatistics::unspecified()];
-
-        assert!(has_z.evaluate(&stats_z_geometry_types));
-        assert!(has_z.evaluate(&stats_z_bbox));
-        assert!(has_z.evaluate(&stats_no_info));
-
-        let stats_no_z_geometry_types = [GeoStatistics::unspecified()
-            .try_with_str_geometry_types(Some(&["POINT"]))
-            .unwrap()];
-        let stats_no_z_bbox = [
-            GeoStatistics::unspecified().with_bbox(Some(BoundingBox::xyzm(
-                (0, 1),
-                (2, 3),
-                Some(Interval::empty()),
-                None,
-            ))),
-        ];
-
-        assert!(!has_z.evaluate(&stats_no_z_geometry_types));
-        assert!(!has_z.evaluate(&stats_no_z_bbox));
+        let stats_z_geometry_types = TableGeoStatistics::from(
+            GeoStatistics::unspecified()
+                .try_with_str_geometry_types(Some(&["POINT Z"]))
+                .unwrap(),
+        );
+        let stats_z_bbox = TableGeoStatistics::from(GeoStatistics::unspecified().with_bbox(Some(
+            BoundingBox::xyzm((0, 1), (2, 3), Some((4, 5).into()), None),
+        )));
+        let stats_no_info = TableGeoStatistics::from(GeoStatistics::unspecified());
+
+        assert!(has_z.evaluate(&stats_z_geometry_types).unwrap());
+        assert!(has_z.evaluate(&stats_z_bbox).unwrap());
+        assert!(has_z.evaluate(&stats_no_info).unwrap());
+
+        let stats_no_z_geometry_types = TableGeoStatistics::from(
+            GeoStatistics::unspecified()
+                .try_with_str_geometry_types(Some(&["POINT"]))
+                .unwrap(),
+        );
+        let stats_no_z_bbox =
+            TableGeoStatistics::from(GeoStatistics::unspecified().with_bbox(Some(
+                BoundingBox::xyzm((0, 1), (2, 3), Some(Interval::empty()), None),
+            )));
+
+        assert!(!has_z.evaluate(&stats_no_z_geometry_types).unwrap());
+        assert!(!has_z.evaluate(&stats_no_z_bbox).unwrap());
     }
 
     #[test]
     fn predicate_other() {
-        assert!(!SpatialFilter::LiteralFalse.evaluate(&[]));
-        assert!(SpatialFilter::Unknown.evaluate(&[]));
+        assert!(!SpatialFilter::LiteralFalse
+            .evaluate(&TableGeoStatistics::empty())
+            .unwrap());
+        assert!(SpatialFilter::Unknown
+            .evaluate(&TableGeoStatistics::empty())
+            .unwrap());
 
         assert!(SpatialFilter::And(
             Box::new(SpatialFilter::Unknown),
             Box::new(SpatialFilter::Unknown)
         )
-        .evaluate(&[]));
+        .evaluate(&TableGeoStatistics::empty())
+        .unwrap());
 
         assert!(!SpatialFilter::And(
             Box::new(SpatialFilter::Unknown),
             Box::new(SpatialFilter::LiteralFalse)
         )
-        .evaluate(&[]));
+        .evaluate(&TableGeoStatistics::empty())
+        .unwrap());
 
         assert!(SpatialFilter::Or(
             Box::new(SpatialFilter::Unknown),
             Box::new(SpatialFilter::Unknown)
         )
-        .evaluate(&[]));
+        .evaluate(&TableGeoStatistics::empty())
+        .unwrap());
 
         assert!(SpatialFilter::Or(
             Box::new(SpatialFilter::Unknown),
             Box::new(SpatialFilter::LiteralFalse)
         )
-        .evaluate(&[]));
+        .evaluate(&TableGeoStatistics::empty())
+        .unwrap());
 
         assert!(!SpatialFilter::Or(
             Box::new(SpatialFilter::LiteralFalse),
             Box::new(SpatialFilter::LiteralFalse)
         )
-        .evaluate(&[]));
+        .evaluate(&TableGeoStatistics::empty())
+        .unwrap());
     }
 
     #[test]
@@ -1146,6 +1241,49 @@ mod test {
         }
     }
 
+    #[test]
+    fn table_geo_stats_position() {
+        let column_stats =
+            GeoStatistics::unspecified().with_bbox(Some(BoundingBox::xy((0.5, 1.5), (1.5, 2.5))));
+        let table_stats = TableGeoStatistics::from(column_stats.clone());
+
+        assert_eq!(
+            table_stats.get(&Column::new("col0", 0)).unwrap(),
+            &column_stats
+        );
+        assert!(table_stats.get(&Column::new("col1", 1)).is_err());
+    }
+
+    #[test]
+    fn table_geo_stats_name() {
+        let geo_stats =
+            GeoStatistics::unspecified().with_bbox(Some(BoundingBox::xy((0.5, 1.5), (1.5, 2.5))));
+        let schema = Schema::new(vec![
+            Field::new("col0", DataType::Binary, true),
+            WKB_GEOMETRY.to_storage_field("col1", true).unwrap(),
+        ]);
+        let table_stats = TableGeoStatistics::try_from_stats_and_schema(
+            &[GeoStatistics::UNSPECIFIED, geo_stats.clone()],
+            &schema,
+        )
+        .unwrap();
+
+        assert_eq!(
+            table_stats.get(&Column::new("col0", usize::MAX)).unwrap(),
+            &GeoStatistics::UNSPECIFIED
+        );
+        assert_eq!(
+            table_stats.get(&Column::new("col1", usize::MAX)).unwrap(),
+            &geo_stats
+        );
+        assert_eq!(
+            table_stats
+                .get(&Column::new("col_not_in_schema", usize::MAX))
+                .unwrap(),
+            &GeoStatistics::UNSPECIFIED
+        );
+    }
+
     #[test]
     fn bounding_box() {
         let col_zero = Column::new("foofy", 0);
diff --git a/rust/sedona-expr/src/statistics.rs b/rust/sedona-expr/src/statistics.rs
index eafa3d1b..08c0dd9c 100644
--- a/rust/sedona-expr/src/statistics.rs
+++ b/rust/sedona-expr/src/statistics.rs
@@ -55,21 +55,24 @@ pub struct GeoStatistics {
 }
 
 impl GeoStatistics {
+    /// Statistics representing unspecified information
+    pub const UNSPECIFIED: GeoStatistics = Self {
+        bbox: None,
+        geometry_types: None,
+        total_geometries: None,
+        total_size_bytes: None,
+        total_points: None,
+        puntal_count: None,
+        lineal_count: None,
+        polygonal_count: None,
+        collection_count: None,
+        total_envelope_width: None,
+        total_envelope_height: None,
+    };
+
     /// Create statistics representing unspecified information
     pub fn unspecified() -> Self {
-        Self {
-            bbox: None,
-            geometry_types: None,
-            total_geometries: None,
-            total_size_bytes: None,
-            total_points: None,
-            puntal_count: None,
-            lineal_count: None,
-            polygonal_count: None,
-            collection_count: None,
-            total_envelope_width: None,
-            total_envelope_height: None,
-        }
+        Self::UNSPECIFIED.clone()
     }
 
     /// Create statistics representing empty information (with zero values instead of None)
diff --git a/rust/sedona-geoparquet/src/file_opener.rs b/rust/sedona-geoparquet/src/file_opener.rs
index 183fe0a8..d634b92a 100644
--- a/rust/sedona-geoparquet/src/file_opener.rs
+++ b/rust/sedona-geoparquet/src/file_opener.rs
@@ -30,7 +30,10 @@ use parquet::file::{
     metadata::{ParquetMetaData, RowGroupMetaData},
     statistics::Statistics,
 };
-use sedona_expr::{spatial_filter::SpatialFilter, statistics::GeoStatistics};
+use sedona_expr::{
+    spatial_filter::{SpatialFilter, TableGeoStatistics},
+    statistics::GeoStatistics,
+};
 use sedona_geometry::bounding_box::BoundingBox;
 use sedona_schema::{datatypes::SedonaType, matchers::ArgMatcher};
 
@@ -175,8 +178,10 @@ fn filter_access_plan_using_geoparquet_file_metadata(
     metadata: &GeoParquetMetadata,
     metrics: &GeoParquetFileOpenerMetrics,
 ) -> Result<()> {
-    let table_geo_stats = geoparquet_file_geo_stats(file_schema, metadata)?;
-    if !spatial_filter.evaluate(&table_geo_stats) {
+    let column_geo_stats = geoparquet_file_geo_stats(file_schema, metadata)?;
+    let table_geo_stats =
+        TableGeoStatistics::try_from_stats_and_schema(&column_geo_stats, file_schema)?;
+    if !spatial_filter.evaluate(&table_geo_stats)? {
         metrics.files_ranges_spatial_pruned.add(1);
         for i in access_plan.row_group_indexes() {
             access_plan.skip(i);
@@ -214,11 +219,15 @@ fn filter_access_plan_using_geoparquet_covering(
     // Iterate through the row groups
     for i in row_group_indices_to_scan {
         // Generate row group statistics based on the covering statistics
-        let row_group_geo_stats =
+        let row_group_column_geo_stats =
             row_group_covering_geo_stats(parquet_metadata.row_group(i), &covering_specs);
+        let row_group_geo_stats = TableGeoStatistics::try_from_stats_and_schema(
+            &row_group_column_geo_stats,
+            file_schema,
+        )?;
 
         // Evaluate predicate!
-        if !spatial_filter.evaluate(&row_group_geo_stats) {
+        if !spatial_filter.evaluate(&row_group_geo_stats)? {
             metrics.row_groups_spatial_pruned.add(1);
             access_plan.skip(i);
         } else {


(sedona-db) 03/03: fix(rust/sedona-geoparquet): Don't use ProjectionExec to create GeoParquet 1.1 bounding box columns (#398)

Posted by pa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

paleolimbot pushed a commit to branch branch-0.2.0
in repository https://gitbox.apache.org/repos/asf/sedona-db.git

commit 69f89a9a0cced55a3792eb6dfe1f76e9dd01033c
Author: Dewey Dunnington <de...@dunnington.ca>
AuthorDate: Mon Dec 1 09:07:35 2025 -0600

    fix(rust/sedona-geoparquet): Don't use ProjectionExec to create GeoParquet 1.1 bounding box columns (#398)
---
 rust/sedona-geoparquet/src/writer.rs | 194 +++++++++++++++++++++++++++++++----
 1 file changed, 176 insertions(+), 18 deletions(-)

diff --git a/rust/sedona-geoparquet/src/writer.rs b/rust/sedona-geoparquet/src/writer.rs
index 2b2ec9a4..8ea1a264 100644
--- a/rust/sedona-geoparquet/src/writer.rs
+++ b/rust/sedona-geoparquet/src/writer.rs
@@ -15,28 +15,35 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::{collections::HashMap, sync::Arc};
+use std::{any::Any, collections::HashMap, fmt, sync::Arc};
 
 use arrow_array::{
     builder::{Float32Builder, NullBufferBuilder},
-    ArrayRef, StructArray,
+    ArrayRef, RecordBatch, StructArray,
 };
-use arrow_schema::{DataType, Field, Fields};
+use arrow_schema::{DataType, Field, Fields, Schema, SchemaRef};
+use async_trait::async_trait;
 use datafusion::{
     config::TableParquetOptions,
     datasource::{
-        file_format::parquet::ParquetSink, physical_plan::FileSinkConfig, sink::DataSinkExec,
+        file_format::parquet::ParquetSink,
+        physical_plan::FileSinkConfig,
+        sink::{DataSink, DataSinkExec},
     },
 };
 use datafusion_common::{
     config::ConfigOptions, exec_datafusion_err, exec_err, not_impl_err, DataFusionError, Result,
 };
+use datafusion_execution::{SendableRecordBatchStream, TaskContext};
 use datafusion_expr::{dml::InsertOp, ColumnarValue, ScalarUDF, Volatility};
 use datafusion_physical_expr::{
     expressions::Column, LexRequirement, PhysicalExpr, ScalarFunctionExpr,
 };
-use datafusion_physical_plan::{projection::ProjectionExec, ExecutionPlan};
+use datafusion_physical_plan::{
+    stream::RecordBatchStreamAdapter, DisplayAs, DisplayFormatType, ExecutionPlan,
+};
 use float_next_after::NextAfter;
+use futures::StreamExt;
 use geo_traits::GeometryTrait;
 use sedona_common::sedona_internal_err;
 use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
@@ -58,7 +65,7 @@ use crate::{
 };
 
 pub fn create_geoparquet_writer_physical_plan(
-    mut input: Arc<dyn ExecutionPlan>,
+    input: Arc<dyn ExecutionPlan>,
     mut conf: FileSinkConfig,
     order_requirements: Option<LexRequirement>,
     options: &TableGeoParquetOptions,
@@ -76,6 +83,8 @@ pub fn create_geoparquet_writer_physical_plan(
     // We have geometry and/or geography! Collect the GeoParquetMetadata we'll need to write
     let mut metadata = GeoParquetMetadata::default();
     let mut bbox_columns = HashMap::new();
+    let mut bbox_projection = None;
+    let mut parquet_output_schema = conf.output_schema().clone();
 
     // Check the version
     match options.geoparquet_version {
@@ -84,9 +93,10 @@ pub fn create_geoparquet_writer_physical_plan(
         }
         GeoParquetVersion::V1_1 => {
             metadata.version = "1.1.0".to_string();
-            (input, bbox_columns) = project_bboxes(input, options.overwrite_bbox_columns)?;
-            conf.output_schema = input.schema();
-            output_geometry_column_indices = input.schema().geometry_column_indices()?;
+            (bbox_projection, bbox_columns) =
+                project_bboxes(&input, options.overwrite_bbox_columns)?;
+            parquet_output_schema = compute_final_schema(&bbox_projection, &input.schema())?;
+            output_geometry_column_indices = conf.output_schema.geometry_column_indices()?;
         }
         _ => {
             return not_impl_err!(
@@ -168,10 +178,78 @@ pub fn create_geoparquet_writer_physical_plan(
     );
 
     // Create the sink
-    let sink = Arc::new(ParquetSink::new(conf, parquet_options));
+    let sink_input_schema = conf.output_schema;
+    conf.output_schema = parquet_output_schema.clone();
+    let sink = Arc::new(GeoParquetSink {
+        inner: ParquetSink::new(conf, parquet_options),
+        projection: bbox_projection,
+        sink_input_schema,
+        parquet_output_schema,
+    });
     Ok(Arc::new(DataSinkExec::new(input, sink, order_requirements)) as _)
 }
 
+/// Implementation of [DataSink] that computes GeoParquet 1.1 bbox columns
+/// if needed. This is used instead of a ProjectionExec because DataFusion's
+/// optimizer rules seem to rearrange the projection in ways that cause
+/// the plan to fail <https://github.com/apache/sedona-db/issues/379>.
+#[derive(Debug)]
+struct GeoParquetSink {
+    inner: ParquetSink,
+    projection: Option<Vec<(Arc<dyn PhysicalExpr>, String)>>,
+    sink_input_schema: SchemaRef,
+    parquet_output_schema: SchemaRef,
+}
+
+impl DisplayAs for GeoParquetSink {
+    fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.inner.fmt_as(t, f)
+    }
+}
+
+#[async_trait]
+impl DataSink for GeoParquetSink {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> &SchemaRef {
+        &self.sink_input_schema
+    }
+
+    async fn write_all(
+        &self,
+        data: SendableRecordBatchStream,
+        context: &Arc<TaskContext>,
+    ) -> Result<u64> {
+        if let Some(projection) = &self.projection {
+            // If we have a projection, apply it here
+            let schema = self.parquet_output_schema.clone();
+            let projection = projection.clone();
+
+            let data = Box::pin(RecordBatchStreamAdapter::new(
+                schema.clone(),
+                data.map(move |batch_result| {
+                    let schema = schema.clone();
+
+                    batch_result.and_then(|batch| {
+                        let mut columns = Vec::with_capacity(projection.len());
+                        for (expr, _) in &projection {
+                            let col = expr.evaluate(&batch)?;
+                            columns.push(col.into_array(batch.num_rows())?);
+                        }
+                        Ok(RecordBatch::try_new(schema.clone(), columns)?)
+                    })
+                }),
+            ));
+
+            self.inner.write_all(data, context).await
+        } else {
+            self.inner.write_all(data, context).await
+        }
+    }
+}
+
 /// Create a regular Parquet writer like DataFusion would otherwise do.
 fn create_inner_writer(
     input: Arc<dyn ExecutionPlan>,
@@ -184,6 +262,11 @@ fn create_inner_writer(
     Ok(Arc::new(DataSinkExec::new(input, sink, order_requirements)) as _)
 }
 
+type ProjectBboxesResult = (
+    Option<Vec<(Arc<dyn PhysicalExpr>, String)>>,
+    HashMap<String, String>,
+);
+
 /// Create a projection that inserts a bbox column for every geometry column
 ///
 /// This implements creating the GeoParquet 1.1 bounding box columns,
@@ -206,9 +289,9 @@ fn create_inner_writer(
 /// "some_col_bbox", it is unlikely that replacing it would have unintended
 /// consequences.
 fn project_bboxes(
-    input: Arc<dyn ExecutionPlan>,
+    input: &Arc<dyn ExecutionPlan>,
     overwrite_bbox_columns: bool,
-) -> Result<(Arc<dyn ExecutionPlan>, HashMap<String, String>)> {
+) -> Result<ProjectBboxesResult> {
     let input_schema = input.schema();
     let matcher = ArgMatcher::is_geometry();
     let bbox_udf: Arc<ScalarUDF> = Arc::new(geoparquet_bbox_udf().into());
@@ -245,7 +328,7 @@ fn project_bboxes(
     // If we don't need to create any bbox columns, don't add an additional
     // projection at the end of the input plan
     if bbox_exprs.is_empty() {
-        return Ok((input, HashMap::new()));
+        return Ok((None, HashMap::new()));
     }
 
     // Create the projection expressions
@@ -275,13 +358,34 @@ Use overwrite_bbox_columns = True if this is what was intended.",
         exprs.push((column, f.name().clone()));
     }
 
-    // Create the projection
-    let exec = ProjectionExec::try_new(exprs, input)?;
-
     // Flip the bbox_column_names into the form our caller needs it
     let bbox_column_names_by_field = bbox_column_names.drain().map(|(k, v)| (v, k)).collect();
 
-    Ok((Arc::new(exec), bbox_column_names_by_field))
+    Ok((Some(exprs), bbox_column_names_by_field))
+}
+
+fn compute_final_schema(
+    bbox_projection: &Option<Vec<(Arc<dyn PhysicalExpr>, String)>>,
+    initial_schema: &SchemaRef,
+) -> Result<SchemaRef> {
+    if let Some(bbox_projection) = bbox_projection {
+        let new_fields = bbox_projection
+            .iter()
+            .map(|(expr, name)| -> Result<Field> {
+                let return_field_ref = expr.return_field(initial_schema)?;
+                Ok(Field::new(
+                    name,
+                    return_field_ref.data_type().clone(),
+                    return_field_ref.is_nullable(),
+                )
+                .with_metadata(return_field_ref.metadata().clone()))
+            })
+            .collect::<Result<Vec<_>>>()?;
+
+        Ok(Arc::new(Schema::new(new_fields)))
+    } else {
+        Ok(initial_schema.clone())
+    }
 }
 
 fn geoparquet_bbox_udf() -> SedonaScalarUDF {
@@ -419,7 +523,7 @@ mod test {
     };
     use datafusion_common::cast::{as_float32_array, as_struct_array};
     use datafusion_common::ScalarValue;
-    use datafusion_expr::{Expr, LogicalPlanBuilder};
+    use datafusion_expr::{Cast, Expr, LogicalPlanBuilder};
     use sedona_schema::datatypes::WKB_GEOMETRY;
     use sedona_testing::create::create_array;
     use sedona_testing::data::test_geoparquet;
@@ -745,6 +849,60 @@ mod test {
             .unwrap();
     }
 
+    #[tokio::test]
+    async fn geoparquet_1_1_with_sort_by_expr() {
+        let example = test_geoparquet("ns-water", "water-point");
+
+        // Requires submodules/download-assets.py which not all contributors need
+        let example = match example {
+            Ok(path) => path,
+            Err(err) => {
+                println!("ns-water/water-point is not available: {err}");
+                return;
+            }
+        };
+
+        let ctx = setup_context();
+        let fns = sedona_functions::register::default_function_set();
+
+        let geometry_udf: ScalarUDF = fns.scalar_udf("sd_format").unwrap().clone().into();
+        let bbox_udf: ScalarUDF = geoparquet_bbox_udf().into();
+
+        let df = ctx
+            .table(&example)
+            .await
+            .unwrap()
+            .sort_by(vec![geometry_udf.call(vec![col("geometry")])])
+            .unwrap()
+            .select(vec![
+                Expr::Cast(Cast::new(
+                    geometry_udf.call(vec![col("geometry")]).alias("txt").into(),
+                    DataType::Utf8View,
+                )),
+                col("geometry"),
+            ])
+            .unwrap();
+
+        let mut options = TableGeoParquetOptions::new();
+        options.geoparquet_version = GeoParquetVersion::V1_1;
+
+        let df_batches_with_bbox = df
+            .clone()
+            .select(vec![
+                col("txt"),
+                bbox_udf.call(vec![col("geometry")]).alias("bbox"),
+                col("geometry"),
+            ])
+            .unwrap()
+            .collect()
+            .await
+            .unwrap();
+
+        test_write_dataframe(ctx, df, df_batches_with_bbox, options, vec![])
+            .await
+            .unwrap();
+    }
+
     #[test]
     fn float_bbox() {
         let tester = ScalarUdfTester::new(geoparquet_bbox_udf().into(), vec![WKB_GEOMETRY]);