You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2026/02/10 08:09:10 UTC
(sedona) branch fix/2608-rasterudt-delta-write created (now cb6c34c21f)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a change to branch fix/2608-rasterudt-delta-write
in repository https://gitbox.apache.org/repos/asf/sedona.git
at cb6c34c21f [GH-2608] Fix RasterUDT JSON schema serialization by stripping $ suffix
This branch includes the following new commits:
new 1b6629df63 [GH-2608] Add failing test for RasterUDT JSON schema round-trip
new cb6c34c21f [GH-2608] Fix RasterUDT JSON schema serialization by stripping $ suffix
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(sedona) 02/02: [GH-2608] Fix RasterUDT JSON schema serialization by stripping $ suffix
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch fix/2608-rasterudt-delta-write
in repository https://gitbox.apache.org/repos/asf/sedona.git
commit cb6c34c21f966bc7e19b631afdb5aaa42e6051bb
Author: Jia Yu <ji...@apache.org>
AuthorDate: Tue Feb 10 00:08:52 2026 -0800
[GH-2608] Fix RasterUDT JSON schema serialization by stripping $ suffix
RasterUDT is a Scala case object whose getClass.getName returns
'RasterUDT$' with a trailing $ sign. Spark's UserDefinedType.jsonValue
uses this class name in the JSON schema. When Delta/Parquet tries to
reconstruct the UDT via Class.forName(...).getConstructor().newInstance(),
it fails because the singleton object's constructor is private.
This fix adds a jsonValue override (identical to the existing fix in
GeometryUDT and GeographyUDT) that strips the trailing $ from the
class name, allowing correct round-trip serialization.
Closes #2608
Closes #2347
---
.../scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala
index ee05cd1521..a34b372211 100644
--- a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala
+++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/RasterUDT.scala
@@ -21,6 +21,8 @@ package org.apache.spark.sql.sedona_sql.UDT
import org.apache.sedona.common.raster.serde.Serde
import org.apache.spark.sql.types.{BinaryType, DataType, UserDefinedType}
import org.geotools.coverage.grid.GridCoverage2D
+import org.json4s.JsonDSL._
+import org.json4s.JsonAST.JValue
class RasterUDT extends UserDefinedType[GridCoverage2D] {
override def sqlType: DataType = BinaryType
@@ -41,6 +43,13 @@ class RasterUDT extends UserDefinedType[GridCoverage2D] {
override def userClass: Class[GridCoverage2D] = classOf[GridCoverage2D]
+ override private[sql] def jsonValue: JValue = {
+ super.jsonValue mapField {
+ case ("class", _) => "class" -> this.getClass.getName.stripSuffix("$")
+ case other: Any => other
+ }
+ }
+
override def equals(other: Any): Boolean = other match {
case _: UserDefinedType[_] => other.isInstanceOf[RasterUDT]
case _ => false
(sedona) 01/02: [GH-2608] Add failing test for RasterUDT JSON schema round-trip
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch fix/2608-rasterudt-delta-write
in repository https://gitbox.apache.org/repos/asf/sedona.git
commit 1b6629df637cca06aa179dd2c42a089abd246110
Author: Jia Yu <ji...@apache.org>
AuthorDate: Mon Feb 9 23:45:55 2026 -0800
[GH-2608] Add failing test for RasterUDT JSON schema round-trip
---
.../org/apache/sedona/sql/RasterUDTSuite.scala | 61 +++++++++++++++++++++-
1 file changed, 59 insertions(+), 2 deletions(-)
diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/RasterUDTSuite.scala b/spark/common/src/test/scala/org/apache/sedona/sql/RasterUDTSuite.scala
index 45f6696963..010462d19d 100644
--- a/spark/common/src/test/scala/org/apache/sedona/sql/RasterUDTSuite.scala
+++ b/spark/common/src/test/scala/org/apache/sedona/sql/RasterUDTSuite.scala
@@ -19,9 +19,14 @@
package org.apache.sedona.sql
import org.apache.spark.sql.sedona_sql.UDT.RasterUDT
-import org.scalatest.funspec.AnyFunSpec
+import org.apache.spark.sql.types.{DataType, StructField, StructType}
+import org.junit.rules.TemporaryFolder
+import org.scalatest.BeforeAndAfter
+
+class RasterUDTSuite extends TestBaseScala with BeforeAndAfter {
+
+ var tempFolder: TemporaryFolder = new TemporaryFolder
-class RasterUDTSuite extends AnyFunSpec {
describe("RasterUDT Test") {
it("Case object and new instance should be equals") {
assert(RasterUDT == RasterUDT)
@@ -35,5 +40,57 @@ class RasterUDTSuite extends AnyFunSpec {
val udt = new RasterUDT
assert(udt.hashCode() == RasterUDT.hashCode())
}
+
+ it("Should be able to render and parse JSON schema with RasterUDT") {
+ // This reproduces the Delta write bug (#2608):
+ // Delta and Parquet serialize the schema to JSON, then deserialize it.
+ // Without a jsonValue override, the class name includes a '$' suffix
+ // from the Scala case object, causing ClassNotFoundException on read.
+ val rasterDf = sparkSession.sql(
+ "SELECT RS_MakeEmptyRaster(1, 10, 10, 0, 0, 1) as raster")
+ assert(
+ DataType
+ .fromJson(rasterDf.schema.json)
+ .asInstanceOf[StructType]
+ .equals(rasterDf.schema))
+ }
+
+ it("Should write and read raster DataFrame in Parquet format") {
+ // Parquet also serializes schema as JSON, triggering the same bug as Delta.
+ tempFolder.create()
+ val rasterDf = sparkSession.sql(
+ "SELECT RS_MakeEmptyRaster(1, 10, 10, 0, 0, 1) as raster")
+
+ rasterDf.write.parquet(tempFolder.getRoot.getPath + "/raster_parquet")
+
+ val readDf =
+ sparkSession.read.parquet(tempFolder.getRoot.getPath + "/raster_parquet")
+ assert(readDf.schema.fields(0).dataType.isInstanceOf[RasterUDT])
+ assert(readDf.count() == 1)
+ }
+
+ it("RS_Union_Aggr output should write and read Parquet successfully") {
+ // Users reported (#2608) that RS_Union_Aggr output can be written to Delta/Parquet
+ // while RS_MakeEmptyRaster output cannot. RS_Union_Aggr uses ExpressionEncoder
+ // which resolves the UDT via UDTRegistration (class name without '$' suffix),
+ // whereas InferredExpression-based functions use the case object singleton
+ // whose getClass.getName includes '$'.
+ tempFolder.create()
+ val rasterDf = sparkSession.sql(
+ """SELECT RS_Union_Aggr(raster) as raster FROM (
+ | SELECT RS_MakeEmptyRaster(1, 10, 10, 0, 0, 1) as raster
+ |)""".stripMargin)
+
+ rasterDf.write.parquet(tempFolder.getRoot.getPath + "/union_aggr_parquet")
+
+ val readDf =
+ sparkSession.read.parquet(tempFolder.getRoot.getPath + "/union_aggr_parquet")
+ assert(readDf.schema.fields(0).dataType.isInstanceOf[RasterUDT])
+ assert(readDf.count() == 1)
+ }
+ }
+
+ after {
+ tempFolder.delete()
}
}