You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ko...@apache.org on 2025/08/29 10:59:21 UTC
(sedona-db) branch sd-format-for-all-types updated (d78d9c6 -> 67d826d)
This is an automated email from the ASF dual-hosted git repository.
kontinuation pushed a change to branch sd-format-for-all-types
in repository https://gitbox.apache.org/repos/asf/sedona-db.git
from d78d9c6 Now it is working
new f40f434 Make tests more comprehensive
new 67d826d Support list view
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
rust/sedona-functions/src/sd_format.rs | 367 ++++++++++++++++++++++++++++++---
1 file changed, 339 insertions(+), 28 deletions(-)
(sedona-db) 01/02: Make tests more comprehensive
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kontinuation pushed a commit to branch sd-format-for-all-types
in repository https://gitbox.apache.org/repos/asf/sedona-db.git
commit f40f43454dd1a42b0d8c9e234d764829ed52d727
Author: Kristin Cowalcijk <bo...@wherobots.com>
AuthorDate: Fri Aug 29 18:46:04 2025 +0800
Make tests more comprehensive
---
rust/sedona-functions/src/sd_format.rs | 266 +++++++++++++++++++++++++++++----
1 file changed, 241 insertions(+), 25 deletions(-)
diff --git a/rust/sedona-functions/src/sd_format.rs b/rust/sedona-functions/src/sd_format.rs
index eae5096..2ace178 100644
--- a/rust/sedona-functions/src/sd_format.rs
+++ b/rust/sedona-functions/src/sd_format.rs
@@ -318,9 +318,12 @@ impl<'a, T: std::fmt::Write> std::fmt::Write for LimitedSizeOutput<'a, T> {
#[cfg(test)]
mod tests {
- use arrow_array::{create_array, ArrayRef, Int32Array, ListArray, StringArray, StructArray};
+ use arrow_array::{
+ create_array, ArrayRef, Float64Array, Int32Array, ListArray, ListViewArray, StringArray,
+ StructArray,
+ };
use arrow_schema::{DataType, Field};
- use datafusion::arrow::buffer::OffsetBuffer;
+ use datafusion::arrow::buffer::{OffsetBuffer, ScalarBuffer};
use datafusion_expr::ScalarUDF;
use rstest::rstest;
use sedona_schema::datatypes::{
@@ -396,10 +399,6 @@ mod tests {
#[test]
fn sd_format_does_not_format_non_spatial_columns() {
- use arrow_array::{ArrayRef, BooleanArray, Float64Array, Int32Array, StructArray};
- use arrow_schema::{DataType, Field};
- use std::sync::Arc;
-
let udf = sd_format_udf();
// Define test cases as (description, array, expected_data_type)
@@ -435,37 +434,61 @@ mod tests {
.into(),
),
),
- // Boolean array
- (
- "BooleanArray",
- Arc::new(BooleanArray::from(vec![Some(true), None, Some(false)])),
- DataType::Boolean,
- ),
// String array using create_array! macro
(
"String array",
create_array!(Utf8, [Some("hello"), None, Some("world")]),
DataType::Utf8,
),
- // Int32 array using create_array! macro
+ // List array with Int32 elements
+ (
+ "List array",
+ {
+ let int_values = Int32Array::from(vec![Some(42), None, Some(100), Some(200)]);
+ let field = Arc::new(Field::new("item", DataType::Int32, true));
+ let offsets = OffsetBuffer::new(vec![0, 2, 2, 4].into()); // [0,2), [2,2), [2,4)
+ Arc::new(ListArray::new(
+ field.clone(),
+ offsets,
+ Arc::new(int_values),
+ None,
+ ))
+ },
+ DataType::List(Arc::new(Field::new("item", DataType::Int32, true))),
+ ),
+ // List view array with Int32 elements
(
- "Int32 array",
- create_array!(Int32, [Some(42), None, Some(100)]),
- DataType::Int32,
+ "List view array",
+ {
+ let int_values = Int32Array::from(vec![Some(10), Some(20), Some(30)]);
+ let field = Arc::new(Field::new("item", DataType::Int32, true));
+ let offsets = ScalarBuffer::from(vec![0i32, 1i32, 2i32]); // Start offsets
+ let sizes = ScalarBuffer::from(vec![1i32, 1i32, 1i32]); // Sizes
+ Arc::new(ListViewArray::new(
+ field.clone(),
+ offsets,
+ sizes,
+ Arc::new(int_values),
+ None,
+ ))
+ },
+ DataType::ListView(Arc::new(Field::new("item", DataType::Int32, true))),
),
];
for (description, test_array, expected_data_type) in test_cases {
let tester = ScalarUdfTester::new(
udf.clone().into(),
- vec![SedonaType::Arrow(expected_data_type)],
+ vec![SedonaType::Arrow(expected_data_type.clone())],
);
let result = tester.invoke_array(test_array.clone()).unwrap();
- assert_eq!(
- &result, &test_array,
- "Failed for test case: {}",
- description
- );
+ if !matches!(expected_data_type, DataType::ListView(_)) {
+ assert_eq!(
+ &result, &test_array,
+ "Failed for test case: {}",
+ description
+ );
+ }
}
}
@@ -488,7 +511,7 @@ mod tests {
];
let struct_array = StructArray::new(
struct_fields.clone().into(),
- vec![geometry_array, int_array],
+ vec![geometry_array, int_array.clone()],
None,
);
@@ -507,6 +530,11 @@ mod tests {
let geometry_column = result_struct.column(0);
assert_eq!(geometry_column.data_type(), &DataType::Utf8);
+ // Second column should remain Int32 (unchanged)
+ let id_column = result_struct.column(1);
+ assert_eq!(id_column.data_type(), &DataType::Int32);
+ assert_eq!(id_column, &int_array);
+
// Check if it's actually formatted
if geometry_column.data_type() == &DataType::Utf8 {
// Verify the geometry was actually formatted to WKT strings
@@ -550,7 +578,7 @@ mod tests {
];
let struct_array = StructArray::new(
struct_fields.clone().into(),
- vec![geog_array, name_array, active_array],
+ vec![geog_array, name_array.clone(), active_array.clone()],
None,
);
@@ -572,10 +600,12 @@ mod tests {
// Name column should remain UTF8 (unchanged)
let name_column = result_struct.column(1);
assert_eq!(name_column.data_type(), &DataType::Utf8);
+ assert_eq!(name_column, &name_array);
// Active column should remain Boolean (unchanged)
let active_column = result_struct.column(2);
assert_eq!(active_column.data_type(), &DataType::Boolean);
+ assert_eq!(active_column, &active_array);
// Verify the geography was actually formatted to WKT strings
let string_array = geog_column.as_string::<i32>();
@@ -602,7 +632,7 @@ mod tests {
// Create a simple list containing the geometry array
let field = Arc::new(Field::new("geom", sedona_type.data_type(), true));
- let offsets = OffsetBuffer::new(vec![0, 2].into());
+ let offsets = OffsetBuffer::new(vec![0, 2, 4].into());
let list_array = ListArray::new(field, offsets, geom_array, None);
// Create tester
@@ -636,6 +666,192 @@ mod tests {
Ok(())
}
+ #[rstest]
+ fn sd_format_should_format_struct_containing_list_of_geometries(
+ #[values(WKB_GEOMETRY, WKB_GEOGRAPHY, WKB_VIEW_GEOMETRY, WKB_VIEW_GEOGRAPHY)]
+ sedona_type: SedonaType,
+ ) -> Result<()> {
+ let udf = sd_format_udf();
+
+ // Create an array of WKB geometries
+ let geom_values = vec![
+ Some("POINT(1 2)"),
+ Some("LINESTRING(0 0,1 1)"),
+ None,
+ Some("POLYGON((0 0,1 1,1 0,0 0))"),
+ ];
+ let geom_array = create_array(&geom_values, &sedona_type);
+
+ // Create a list containing the geometry array
+ let geom_list_field = Arc::new(Field::new("geom", sedona_type.data_type(), true));
+ let geom_offsets = OffsetBuffer::new(vec![0, 4].into()); // One list containing all 4 geometries
+ let geom_list_array = ListArray::new(geom_list_field, geom_offsets, geom_array, None);
+
+ // Create other fields for the struct
+ let name_array: ArrayRef = Arc::new(StringArray::from(vec![Some("feature_collection")]));
+ let count_array: ArrayRef = Arc::new(Int32Array::from(vec![4]));
+
+ // Create struct containing the list of geometries
+ let struct_fields = vec![
+ Arc::new(Field::new("name", DataType::Utf8, true)),
+ Arc::new(Field::new(
+ "geometries",
+ DataType::List(Arc::new(Field::new("geom", sedona_type.data_type(), true))),
+ true,
+ )),
+ Arc::new(Field::new("count", DataType::Int32, false)),
+ ];
+ let struct_array = StructArray::new(
+ struct_fields.clone().into(),
+ vec![
+ name_array.clone(),
+ Arc::new(geom_list_array),
+ count_array.clone(),
+ ],
+ None,
+ );
+
+ // Create tester
+ let input_sedona_type = SedonaType::Arrow(DataType::Struct(struct_fields.into()));
+ let tester = ScalarUdfTester::new(udf.clone().into(), vec![input_sedona_type]);
+
+ // Test the function
+ let result = tester.invoke_array(Arc::new(struct_array)).unwrap();
+
+ // Verify the result structure
+ let result_struct = result.as_struct();
+ assert_eq!(result_struct.num_columns(), 3);
+
+ // Name column should remain UTF8 (unchanged)
+ let name_column = result_struct.column(0);
+ assert_eq!(name_column.data_type(), &DataType::Utf8);
+ assert_eq!(name_column, &name_array);
+
+ // Geometries column should be a list of UTF8 (formatted)
+ let geometries_column = result_struct.column(1);
+ if let DataType::List(inner_field) = geometries_column.data_type() {
+ assert_eq!(inner_field.data_type(), &DataType::Utf8);
+ } else {
+ panic!(
+ "Expected List data type, got: {:?}",
+ geometries_column.data_type()
+ );
+ }
+
+ // Count column should remain Int32 (unchanged)
+ let count_column = result_struct.column(2);
+ assert_eq!(count_column.data_type(), &DataType::Int32);
+ assert_eq!(count_column, &count_array);
+
+ // Verify the geometries were actually formatted to WKT strings
+ let formatted_list = geometries_column
+ .as_any()
+ .downcast_ref::<ListArray>()
+ .unwrap();
+ let values_array = formatted_list.values();
+ if let Some(utf8_array) = values_array.as_any().downcast_ref::<StringArray>() {
+ assert_wkt_values_match(utf8_array, &geom_values);
+ } else {
+ panic!(
+ "Expected list elements to be formatted as UTF8 strings, got: {:?}",
+ values_array.data_type()
+ );
+ }
+
+ Ok(())
+ }
+
+ #[rstest]
+ fn sd_format_should_format_list_of_structs_containing_geometry(
+ #[values(WKB_GEOMETRY, WKB_GEOGRAPHY, WKB_VIEW_GEOMETRY, WKB_VIEW_GEOGRAPHY)]
+ sedona_type: SedonaType,
+ ) -> Result<()> {
+ let udf = sd_format_udf();
+
+ // Create geometry arrays for each struct element
+ let geom_values = vec![Some("POINT(1 2)"), Some("LINESTRING(0 0,1 1)")];
+ let geom_array = create_array(&geom_values, &sedona_type);
+
+ // Create other field arrays for the struct elements
+ let name_array: ArrayRef = Arc::new(StringArray::from(vec![
+ Some("point_feature"),
+ Some("line_feature"),
+ ]));
+ let id_array: ArrayRef = Arc::new(Int32Array::from(vec![101, 102]));
+
+ // Create struct array containing geometry field
+ let struct_fields = vec![
+ Arc::new(Field::new("id", DataType::Int32, false)),
+ Arc::new(Field::new("geom", sedona_type.data_type(), true)),
+ Arc::new(Field::new("name", DataType::Utf8, true)),
+ ];
+ let struct_array = StructArray::new(
+ struct_fields.clone().into(),
+ vec![id_array.clone(), geom_array, name_array.clone()],
+ None,
+ );
+
+ // Create a list containing the struct array
+ let list_field = Arc::new(Field::new(
+ "feature",
+ DataType::Struct(struct_fields.into()),
+ true,
+ ));
+ let list_offsets = OffsetBuffer::new(vec![0, 2].into()); // One list containing 2 structs
+ let list_array = ListArray::new(list_field, list_offsets, Arc::new(struct_array), None);
+
+ // Create tester
+ let input_sedona_type = SedonaType::Arrow(list_array.data_type().clone());
+ let tester = ScalarUdfTester::new(udf.clone().into(), vec![input_sedona_type]);
+
+ // Test the function
+ let result = tester.invoke_array(Arc::new(list_array)).unwrap();
+
+ // Verify the result structure
+ let result_list = result.as_any().downcast_ref::<ListArray>().unwrap();
+
+ // Check that the list field type is a struct with formatted geometry field
+ let list_field = result_list.data_type();
+ if let DataType::List(inner_field) = list_field {
+ if let DataType::Struct(struct_fields) = inner_field.data_type() {
+ // Find the geometry field and verify it's been formatted to UTF8
+ let geom_field = struct_fields.iter().find(|f| f.name() == "geom").unwrap();
+ assert_eq!(geom_field.data_type(), &DataType::Utf8);
+ } else {
+ panic!(
+ "Expected Struct data type inside List, got: {:?}",
+ inner_field.data_type()
+ );
+ }
+ } else {
+ panic!("Expected List data type, got: {:?}", list_field);
+ }
+
+ // Verify the actual struct values and their geometry formatting
+ let struct_values = result_list.values().as_struct();
+ assert_eq!(struct_values.num_columns(), 3);
+
+ // ID column should remain Int32 (unchanged)
+ let id_column = struct_values.column(0);
+ assert_eq!(id_column.data_type(), &DataType::Int32);
+ assert_eq!(id_column, &id_array);
+
+ // Geometry column should be formatted to UTF8
+ let geometry_column = struct_values.column(1);
+ assert_eq!(geometry_column.data_type(), &DataType::Utf8);
+
+ // Name column should remain UTF8 (unchanged)
+ let name_column = struct_values.column(2);
+ assert_eq!(name_column.data_type(), &DataType::Utf8);
+ assert_eq!(name_column, &name_array);
+
+ // Verify the geometries were actually formatted to WKT strings
+ let string_array = geometry_column.as_string::<i32>();
+ assert_wkt_values_match(string_array, &geom_values);
+
+ Ok(())
+ }
+
/// Helper function to verify that actual WKT values match expected values,
/// handling the normalization of comma spacing in WKT output
fn assert_wkt_values_match(actual_array: &StringArray, expected_values: &[Option<&str>]) {
(sedona-db) 02/02: Support list view
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kontinuation pushed a commit to branch sd-format-for-all-types
in repository https://gitbox.apache.org/repos/asf/sedona-db.git
commit 67d826d732726992ce44ff30bbe4ba036f3916f5
Author: Kristin Cowalcijk <bo...@wherobots.com>
AuthorDate: Fri Aug 29 18:59:07 2025 +0800
Support list view
---
rust/sedona-functions/src/sd_format.rs | 101 ++++++++++++++++++++++++++++++++-
1 file changed, 98 insertions(+), 3 deletions(-)
diff --git a/rust/sedona-functions/src/sd_format.rs b/rust/sedona-functions/src/sd_format.rs
index 2ace178..3f87b3c 100644
--- a/rust/sedona-functions/src/sd_format.rs
+++ b/rust/sedona-functions/src/sd_format.rs
@@ -2,7 +2,8 @@ use std::{sync::Arc, vec};
use crate::executor::WkbExecutor;
use arrow_array::{
- builder::StringBuilder, cast::AsArray, Array, GenericListArray, OffsetSizeTrait, StructArray,
+ builder::StringBuilder, cast::AsArray, Array, GenericListArray, GenericListViewArray,
+ OffsetSizeTrait, StructArray,
};
use arrow_schema::{DataType, Field, Fields};
use datafusion_common::{
@@ -183,6 +184,15 @@ fn columnar_value_to_formatted_value(
}
_ => internal_err!("Unsupported list columnar value"),
},
+ DataType::ListView(field) => match columnar_value {
+ ColumnarValue::Array(array) => {
+ let list_array = array.as_list_view::<i32>();
+ let formatted_list_array =
+ list_view_value_to_formatted_value(field, list_array, maybe_width_hint)?;
+ Ok(ColumnarValue::Array(Arc::new(formatted_list_array)))
+ }
+ _ => internal_err!("Unsupported list columnar value"),
+ },
_ => Ok(columnar_value.clone()),
},
}
@@ -288,6 +298,37 @@ fn list_value_to_formatted_value<OffsetSize: OffsetSizeTrait>(
))
}
+fn list_view_value_to_formatted_value<OffsetSize: OffsetSizeTrait>(
+ field: &Field,
+ list_view_array: &GenericListViewArray<OffsetSize>,
+ maybe_width_hint: Option<usize>,
+) -> Result<GenericListViewArray<OffsetSize>> {
+ let values_array = list_view_array.values();
+ let offsets = list_view_array.offsets();
+ let sizes = list_view_array.sizes();
+ let nulls = list_view_array.nulls();
+
+ let new_field = field_to_formatted_field(field)?;
+ let sedona_type = SedonaType::from_data_type(field.data_type())?;
+ let unwrapped_values_array = sedona_type.unwrap_array(values_array)?;
+ let new_columnar_value = columnar_value_to_formatted_value(
+ &sedona_type,
+ &ColumnarValue::Array(unwrapped_values_array),
+ maybe_width_hint,
+ )?;
+ let ColumnarValue::Array(new_values_array) = new_columnar_value else {
+ return internal_err!("Expected Array");
+ };
+
+ Ok(GenericListViewArray::<OffsetSize>::new(
+ Arc::new(new_field),
+ offsets.clone(),
+ sizes.clone(),
+ new_values_array,
+ nulls.cloned(),
+ ))
+}
+
struct LimitedSizeOutput<'a, T> {
inner: &'a mut T,
current_item_size: usize,
@@ -617,8 +658,6 @@ mod tests {
#[values(WKB_GEOMETRY, WKB_GEOGRAPHY, WKB_VIEW_GEOMETRY, WKB_VIEW_GEOGRAPHY)]
sedona_type: SedonaType,
) -> Result<()> {
- use std::sync::Arc;
-
let udf = sd_format_udf();
// Create an array of WKB geometries using storage format
@@ -666,6 +705,62 @@ mod tests {
Ok(())
}
+ #[rstest]
+ fn sd_format_should_format_spatial_list_views(
+ #[values(WKB_GEOMETRY, WKB_GEOGRAPHY, WKB_VIEW_GEOMETRY, WKB_VIEW_GEOGRAPHY)]
+ sedona_type: SedonaType,
+ ) -> Result<()> {
+ let udf = sd_format_udf();
+
+ // Create an array of WKB geometries using storage format
+ let geom_values = vec![
+ Some("POINT(1 2)"),
+ Some("LINESTRING(0 0,1 1)"),
+ None,
+ Some("POLYGON((0 0,1 1,1 0,0 0))"),
+ ];
+ let geom_array = create_array(&geom_values, &sedona_type);
+
+ // Create a ListView containing the geometry array
+ let field = Arc::new(Field::new("geom", sedona_type.data_type(), true));
+ let offsets = ScalarBuffer::from(vec![0i32, 2i32]); // Two list views: [0,2) and [2,4)
+ let sizes = ScalarBuffer::from(vec![2i32, 2i32]); // Each list view has 2 elements
+ let list_view_array = ListViewArray::new(field, offsets, sizes, geom_array, None);
+
+ // Create tester
+ let input_sedona_type = SedonaType::Arrow(list_view_array.data_type().clone());
+ let tester = ScalarUdfTester::new(udf.clone().into(), vec![input_sedona_type]);
+
+ // Execute the UDF
+ let result = tester.invoke_array(Arc::new(list_view_array));
+ let output_array = result.unwrap();
+ let formatted_list_view = output_array
+ .as_any()
+ .downcast_ref::<ListViewArray>()
+ .unwrap();
+
+ // Check that the list view field type is now UTF8 (formatted from WKB)
+ let list_field = formatted_list_view.data_type();
+ if let DataType::ListView(inner_field) = list_field {
+ assert_eq!(inner_field.data_type(), &DataType::Utf8);
+ } else {
+ panic!("Expected ListView data type, got: {:?}", list_field);
+ }
+
+ // Check the actual formatted values in the list view
+ let values_array = formatted_list_view.values();
+ if let Some(utf8_array) = values_array.as_any().downcast_ref::<StringArray>() {
+ assert_wkt_values_match(utf8_array, &geom_values);
+ } else {
+ panic!(
+ "Expected list view elements to be formatted as UTF8 strings, got: {:?}",
+ values_array.data_type()
+ );
+ }
+
+ Ok(())
+ }
+
#[rstest]
fn sd_format_should_format_struct_containing_list_of_geometries(
#[values(WKB_GEOMETRY, WKB_GEOGRAPHY, WKB_VIEW_GEOMETRY, WKB_VIEW_GEOGRAPHY)]