You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by im...@apache.org on 2026/01/03 23:58:18 UTC
(sedona) 11/24: SEDONA-748 add working example
This is an automated email from the ASF dual-hosted git repository.
imbruced pushed a commit to branch arrow-worker
in repository https://gitbox.apache.org/repos/asf/sedona.git
commit 7371c25a7d90c6582dcb4618d869ad7c15f93fce
Author: pawelkocinski <pa...@gmail.com>
AuthorDate: Thu Nov 13 18:48:45 2025 +0100
SEDONA-748 add working example
---
.../spark/sql/execution/python/SedonaPythonArrowOutput.scala | 10 +++++++++-
.../test/scala/org/apache/spark/sql/udf/StrategySuite.scala | 1 +
.../scala/org/apache/spark/sql/udf/TestScalarPandasUDF.scala | 2 +-
3 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/python/SedonaPythonArrowOutput.scala b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/python/SedonaPythonArrowOutput.scala
index f2c8543537..316cb32c3e 100644
--- a/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/python/SedonaPythonArrowOutput.scala
+++ b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/python/SedonaPythonArrowOutput.scala
@@ -121,7 +121,15 @@ private[python] trait SedonaPythonArrowOutput[OUT <: AnyRef] { self: SedonaBaseP
null.asInstanceOf[OUT]
}
}
- } catch handleException
+ } catch {
+ case e: Exception =>
+ // If an exception happens, make sure to close the reader to release resources.
+ if (reader != null) {
+ reader.close(false)
+ }
+ allocator.close()
+ throw e
+ }
}
}
}
diff --git a/spark/spark-3.5/src/test/scala/org/apache/spark/sql/udf/StrategySuite.scala b/spark/spark-3.5/src/test/scala/org/apache/spark/sql/udf/StrategySuite.scala
index 48e8e1c878..3396defd01 100644
--- a/spark/spark-3.5/src/test/scala/org/apache/spark/sql/udf/StrategySuite.scala
+++ b/spark/spark-3.5/src/test/scala/org/apache/spark/sql/udf/StrategySuite.scala
@@ -47,6 +47,7 @@ class StrategySuite extends AnyFunSuite with Matchers {
// spark.sql("select 1").show()
val df = spark.read.format("geoparquet")
.load("/Users/pawelkocinski/Desktop/projects/sedona-production/apache-sedona-book/data/warehouse/buildings")
+ .limit(10)
.select(
// geometryToNonGeometryFunction(col("geometry")),
geometryToGeometryFunction(col("geometry")),
diff --git a/spark/spark-3.5/src/test/scala/org/apache/spark/sql/udf/TestScalarPandasUDF.scala b/spark/spark-3.5/src/test/scala/org/apache/spark/sql/udf/TestScalarPandasUDF.scala
index 2a3c1dbb2b..83f87f401e 100644
--- a/spark/spark-3.5/src/test/scala/org/apache/spark/sql/udf/TestScalarPandasUDF.scala
+++ b/spark/spark-3.5/src/test/scala/org/apache/spark/sql/udf/TestScalarPandasUDF.scala
@@ -146,7 +146,7 @@ object ScalarUDF {
private val workerEnv = new java.util.HashMap[String, String]()
workerEnv.put("PYTHONPATH", s"$pysparkPythonPath:$pythonPath")
- SparkEnv.get.conf.set(PYTHON_WORKER_MODULE, "sedonaworker.worker")
+ SparkEnv.get.conf.set(PYTHON_WORKER_MODULE, "sedonaworker.work")
SparkEnv.get.conf.set(PYTHON_USE_DAEMON, false)
//
// val geometryToNonGeometryFunction: UserDefinedPythonFunction = UserDefinedPythonFunction(