You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ma...@apache.org on 2021/03/29 20:47:49 UTC
[incubator-sedona] 03/03: Added WKBGeometrySerde to handle special
geometry types that the Shape serde not handled well
This is an automated email from the ASF dual-hosted git repository.
malka pushed a commit to branch Sedona-17_Shape&WKBSerDe
in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git
commit 5f8a441c26256c5f30710037bb1bbf59bc2d6f53
Author: Netanel Malka <ne...@gmail.com>
AuthorDate: Mon Mar 29 23:47:23 2021 +0300
Added WKBGeometrySerde to handle special geometry types that the Shape serde not handled well
---
.../sedona/core/serde/SedonaKryoRegistrator.java | 16 +-----
.../core/serde/SedonaKryoRegistratorHelper.java | 8 +--
.../core/serde/SedonaWKBKryoRegistrator.java | 7 +--
.../WKB/WKBGeometrySerde.java} | 59 +++++++++++++++-------
.../shape/ShapeGeometrySerde.java} | 8 +--
.../core/serde/spatialindex/SpatialIndexSerde.java | 2 -
.../jts/index/quadtree/IndexSerde.java | 6 +--
.../locationtech/jts/index/strtree/IndexSerde.java | 6 +--
...ySerdeTest.java => ShapeGeometrySerdeTest.java} | 6 +--
9 files changed, 61 insertions(+), 57 deletions(-)
diff --git a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java
index 5843b7d..dfb5962 100644
--- a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java
+++ b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java
@@ -20,21 +20,9 @@
package org.apache.sedona.core.serde;
import com.esotericsoftware.kryo.Kryo;
-import org.apache.log4j.Logger;
-import org.apache.sedona.core.geometryObjects.Circle;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
+import org.apache.sedona.core.serde.shape.ShapeGeometrySerde;
import org.apache.sedona.core.serde.spatialindex.SpatialIndexSerde;
import org.apache.spark.serializer.KryoRegistrator;
-import org.locationtech.jts.geom.Envelope;
-import org.locationtech.jts.geom.GeometryCollection;
-import org.locationtech.jts.geom.LineString;
-import org.locationtech.jts.geom.MultiLineString;
-import org.locationtech.jts.geom.MultiPoint;
-import org.locationtech.jts.geom.MultiPolygon;
-import org.locationtech.jts.geom.Point;
-import org.locationtech.jts.geom.Polygon;
-import org.locationtech.jts.index.quadtree.Quadtree;
-import org.locationtech.jts.index.strtree.STRtree;
/**
* Register Kryo classes using the Geometry Serde(using the ShapeFile serialization)
@@ -45,7 +33,7 @@ public class SedonaKryoRegistrator
@Override
public void registerClasses(Kryo kryo) {
- GeometrySerde serializer = new GeometrySerde();
+ ShapeGeometrySerde serializer = new ShapeGeometrySerde();
SpatialIndexSerde indexSerializer = new SpatialIndexSerde();
SedonaKryoRegistratorHelper.registerClasses(kryo, serializer, indexSerializer);
diff --git a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistratorHelper.java b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistratorHelper.java
index c32a3dc..072d344 100644
--- a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistratorHelper.java
+++ b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistratorHelper.java
@@ -23,8 +23,8 @@ import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import org.apache.log4j.Logger;
import org.apache.sedona.core.geometryObjects.Circle;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
-import org.apache.sedona.core.geometryObjects.WKBGeometrySerde;
+import org.apache.sedona.core.serde.shape.ShapeGeometrySerde;
+import org.apache.sedona.core.serde.WKB.WKBGeometrySerde;
import org.apache.sedona.core.serde.spatialindex.SpatialIndexSerde;
import org.locationtech.jts.geom.*;
import org.locationtech.jts.index.quadtree.Quadtree;
@@ -43,8 +43,8 @@ public class SedonaKryoRegistratorHelper {
Serializer serializer;
if (geometrySerdeType instanceof WKBGeometrySerde) {
serializer = new WKBGeometrySerde();
- } else if (geometrySerdeType instanceof GeometrySerde) {
- serializer = new GeometrySerde();
+ } else if (geometrySerdeType instanceof ShapeGeometrySerde) {
+ serializer = new ShapeGeometrySerde();
} else
throw new UnsupportedOperationException(String.format("Geometry Serde: %s is not supported",
geometrySerdeType.getClass().getName())
diff --git a/core/src/main/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistrator.java b/core/src/main/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistrator.java
index 939c262..adc5047 100644
--- a/core/src/main/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistrator.java
+++ b/core/src/main/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistrator.java
@@ -20,14 +20,9 @@
package org.apache.sedona.core.serde;
import com.esotericsoftware.kryo.Kryo;
-import org.apache.log4j.Logger;
-import org.apache.sedona.core.geometryObjects.Circle;
import org.apache.sedona.core.serde.spatialindex.SpatialIndexSerde;
-import org.apache.sedona.core.geometryObjects.WKBGeometrySerde;
+import org.apache.sedona.core.serde.WKB.WKBGeometrySerde;
import org.apache.spark.serializer.KryoRegistrator;
-import org.locationtech.jts.geom.*;
-import org.locationtech.jts.index.quadtree.Quadtree;
-import org.locationtech.jts.index.strtree.STRtree;
/**
* Register Kryo classes using the WKB Geometry Serde and SpatialIndexSerde for index objects
diff --git a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java b/core/src/main/java/org/apache/sedona/core/serde/WKB/WKBGeometrySerde.java
similarity index 80%
copy from core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
copy to core/src/main/java/org/apache/sedona/core/serde/WKB/WKBGeometrySerde.java
index 213e8f2..0543388 100755
--- a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
+++ b/core/src/main/java/org/apache/sedona/core/serde/WKB/WKBGeometrySerde.java
@@ -17,7 +17,9 @@
* under the License.
*/
-package org.apache.sedona.core.geometryObjects;
+package org.apache.sedona.core.serde.WKB;
+
+import java.util.Objects;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Registration;
@@ -25,7 +27,8 @@ import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import org.apache.log4j.Logger;
-import org.apache.sedona.core.formatMapper.shapefileParser.parseUtils.shp.ShapeSerde;
+import org.apache.sedona.core.geometryObjects.Circle;
+import org.apache.sedona.core.serde.shape.ShapeGeometrySerde;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryCollection;
@@ -36,21 +39,26 @@ import org.locationtech.jts.geom.MultiPoint;
import org.locationtech.jts.geom.MultiPolygon;
import org.locationtech.jts.geom.Point;
import org.locationtech.jts.geom.Polygon;
+import org.locationtech.jts.io.ParseException;
+import org.locationtech.jts.io.WKBReader;
+import org.locationtech.jts.io.WKBWriter;
/**
- * Provides methods to efficiently serialize and deserialize geometry types.
+ * Provides methods to efficiently serialize and deserialize geometry types using the WKB format
* <p>
- * Supports Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon,
- * GeometryCollection, Circle and Envelope types.
+ * Supports Point, LineString, Polygon, MultiPoint, MultiLineString,
+ * MultiPolygon, GeometryCollection, Circle and Envelope types.
* <p>
- * First byte contains {@link Type#id}. Then go type-specific bytes, followed
- * by user-data attached to the geometry.
+ * First byte contains {@link Type#id}. Then go type-specific bytes, followed by
+ * user-data attached to the geometry.
*/
-public class GeometrySerde
+public class WKBGeometrySerde
extends Serializer
{
- private static final Logger log = Logger.getLogger(GeometrySerde.class);
+ // TODO - Add tests for WKB serialization
+ // TODO - Consolidate GeometrySerde to eliminate code repetition
+ private static final Logger log = Logger.getLogger(ShapeGeometrySerde.class);
private static final GeometryFactory geometryFactory = new GeometryFactory();
@Override
@@ -99,13 +107,16 @@ public class GeometrySerde
private void writeGeometry(Kryo kryo, Output out, Geometry geometry)
{
- byte[] data = ShapeSerde.serialize(geometry);
+ WKBWriter writer = new WKBWriter(2, 2, true);
+ byte[] data = writer.write(geometry);
+
+ // write geometry length size to read bytes until userData
+ out.writeInt(data.length, true);
out.write(data, 0, data.length);
writeUserData(kryo, out, geometry);
}
- private void writeUserData(Kryo kryo, Output out, Geometry geometry)
- {
+ private void writeUserData(Kryo kryo, Output out, Geometry geometry) {
out.writeBoolean(geometry.getUserData() != null);
if (geometry.getUserData() != null) {
kryo.writeClass(out, geometry.getUserData().getClass());
@@ -118,7 +129,7 @@ public class GeometrySerde
{
byte typeId = input.readByte();
Type geometryType = Type.fromId(typeId);
- switch (geometryType) {
+ switch (Objects.requireNonNull(geometryType)) {
case SHAPE:
return readGeometry(kryo, input);
case CIRCLE: {
@@ -153,8 +164,7 @@ public class GeometrySerde
}
}
- private Object readUserData(Kryo kryo, Input input)
- {
+ private Object readUserData(Kryo kryo, Input input) {
Object userData = null;
if (input.readBoolean()) {
Registration clazz = kryo.readClass(input);
@@ -163,10 +173,21 @@ public class GeometrySerde
return userData;
}
- private Geometry readGeometry(Kryo kryo, Input input)
- {
- Geometry geometry = ShapeSerde.deserialize(input, geometryFactory);
- geometry.setUserData(readUserData(kryo, input));
+ private Geometry readGeometry(Kryo kryo, Input input) {
+ WKBReader reader = new WKBReader();
+ Geometry geometry;
+
+ int geometryBytesLength = input.readInt(true);
+ byte[] bytes = input.readBytes(geometryBytesLength);
+
+ try {
+ geometry = reader.read(bytes);
+ geometry.setUserData(readUserData(kryo, input));
+ } catch (ParseException e) {
+ log.error("Cannot parse geometry bytes", e);
+ return null;
+ }
+
return geometry;
}
diff --git a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java b/core/src/main/java/org/apache/sedona/core/serde/shape/ShapeGeometrySerde.java
similarity index 96%
rename from core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
rename to core/src/main/java/org/apache/sedona/core/serde/shape/ShapeGeometrySerde.java
index 213e8f2..c5551ed 100755
--- a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
+++ b/core/src/main/java/org/apache/sedona/core/serde/shape/ShapeGeometrySerde.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.sedona.core.geometryObjects;
+package org.apache.sedona.core.serde.shape;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Registration;
@@ -26,6 +26,7 @@ import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import org.apache.log4j.Logger;
import org.apache.sedona.core.formatMapper.shapefileParser.parseUtils.shp.ShapeSerde;
+import org.apache.sedona.core.geometryObjects.Circle;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryCollection;
@@ -46,11 +47,11 @@ import org.locationtech.jts.geom.Polygon;
* First byte contains {@link Type#id}. Then go type-specific bytes, followed
* by user-data attached to the geometry.
*/
-public class GeometrySerde
+public class ShapeGeometrySerde
extends Serializer
{
- private static final Logger log = Logger.getLogger(GeometrySerde.class);
+ private static final Logger log = Logger.getLogger(ShapeGeometrySerde.class);
private static final GeometryFactory geometryFactory = new GeometryFactory();
@Override
@@ -165,6 +166,7 @@ public class GeometrySerde
private Geometry readGeometry(Kryo kryo, Input input)
{
+ //TODO Find the kryo registrator class and create one file of that
Geometry geometry = ShapeSerde.deserialize(input, geometryFactory);
geometry.setUserData(readUserData(kryo, input));
return geometry;
diff --git a/core/src/main/java/org/apache/sedona/core/serde/spatialindex/SpatialIndexSerde.java b/core/src/main/java/org/apache/sedona/core/serde/spatialindex/SpatialIndexSerde.java
index 084c892..72ca10e 100644
--- a/core/src/main/java/org/apache/sedona/core/serde/spatialindex/SpatialIndexSerde.java
+++ b/core/src/main/java/org/apache/sedona/core/serde/spatialindex/SpatialIndexSerde.java
@@ -24,8 +24,6 @@ import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import org.apache.log4j.Logger;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
-import org.apache.sedona.core.geometryObjects.WKBGeometrySerde;
import org.locationtech.jts.index.quadtree.IndexSerde;
import org.locationtech.jts.index.quadtree.Quadtree;
import org.locationtech.jts.index.strtree.STRtree;
diff --git a/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java b/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
index 5dd6556..6d796af 100644
--- a/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
+++ b/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
@@ -16,7 +16,7 @@ package org.locationtech.jts.index.quadtree;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
+import org.apache.sedona.core.serde.shape.ShapeGeometrySerde;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
@@ -29,9 +29,9 @@ import java.util.List;
*/
public class IndexSerde
{
- GeometrySerde geometrySerde;
+ ShapeGeometrySerde geometrySerde;
public IndexSerde() {
- geometrySerde = new GeometrySerde();
+ geometrySerde = new ShapeGeometrySerde();
}
public Object read(Kryo kryo, Input input){
diff --git a/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java b/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
index bf3d308..6904b98 100644
--- a/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
+++ b/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
@@ -16,7 +16,7 @@ package org.locationtech.jts.index.strtree;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
+import org.apache.sedona.core.serde.shape.ShapeGeometrySerde;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
@@ -29,9 +29,9 @@ import java.util.List;
*/
public class IndexSerde
{
- GeometrySerde geometrySerde;
+ ShapeGeometrySerde geometrySerde;
public IndexSerde() {
- geometrySerde = new GeometrySerde();
+ geometrySerde = new ShapeGeometrySerde();
}
public Object read(Kryo kryo, Input input){
diff --git a/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java b/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/ShapeGeometrySerdeTest.java
similarity index 95%
rename from core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java
rename to core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/ShapeGeometrySerdeTest.java
index 3bb9af4..ab0e0d4 100644
--- a/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java
+++ b/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/ShapeGeometrySerdeTest.java
@@ -23,7 +23,7 @@ import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import org.apache.sedona.core.geometryObjects.Circle;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
+import org.apache.sedona.core.serde.shape.ShapeGeometrySerde;
import org.apache.sedona.core.utils.GeomUtils;
import org.junit.Test;
import org.locationtech.jts.geom.Geometry;
@@ -35,7 +35,7 @@ import java.io.ByteArrayOutputStream;
import static org.junit.Assert.assertTrue;
-public class GeometrySerdeTest
+public class ShapeGeometrySerdeTest
{
private final Kryo kryo = new Kryo();
private final WKTReader wktReader = new WKTReader();
@@ -86,7 +86,7 @@ public class GeometrySerdeTest
private byte[] serialize(Geometry input)
{
- kryo.register(input.getClass(), new GeometrySerde());
+ kryo.register(input.getClass(), new ShapeGeometrySerde());
ByteArrayOutputStream bos = new ByteArrayOutputStream();
Output output = new Output(bos);