You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2021/09/21 07:37:15 UTC
[incubator-sedona] branch revert-516-Sedona-17_Shape&WKBSerDe
created (now 2a6cc5e)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a change to branch revert-516-Sedona-17_Shape&WKBSerDe
in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git.
at 2a6cc5e Revert "[SEDONA-28] shape&wkb ser de (#516)"
This branch includes the following new commits:
new 2a6cc5e Revert "[SEDONA-28] shape&wkb ser de (#516)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-sedona] 01/01: Revert "[SEDONA-28] shape&wkb ser de
(#516)"
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch revert-516-Sedona-17_Shape&WKBSerDe
in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git
commit 2a6cc5ec2b1c6d82abfe6e8eda8ea358f874fc6e
Author: Jia Yu <ji...@apache.org>
AuthorDate: Tue Sep 21 00:37:10 2021 -0700
Revert "[SEDONA-28] shape&wkb ser de (#516)"
This reverts commit a0ed49da0b62ce591d858e5dccd5cebb69a8e695.
---
.github/workflows/python.yml | 11 +-
.../apache/sedona/core/enums/SerializerType.java | 63 ----
.../{serde => geometryObjects}/GeometrySerde.java | 76 ++---
.../SpatialIndexSerde.java | 15 +-
.../sedona/core/serde/SedonaKryoRegistrator.java | 47 ++-
.../core/serde/SedonaKryoRegistratorHelper.java | 57 ----
.../core/serde/SedonaWKBKryoRegistrator.java | 40 ---
.../sedona/core/serde/WKB/WKBGeometrySerde.java | 75 -----
.../core/serde/shape/ShapeGeometrySerde.java | 58 ----
.../org/apache/sedona/core/utils/SedonaConf.java | 11 +-
.../jts/index/quadtree/IndexSerde.java | 6 +-
.../locationtech/jts/index/strtree/IndexSerde.java | 6 +-
.../shapefileParser/shapes/GeometrySerdeTest.java | 97 ++++++
.../geometryObjects/SpatialIndexSerdeTest.java | 3 +-
.../sedona/core/serde/GeometrySerdeTest.java | 154 ----------
.../core/serde/SedonaKryoRegistratorTest.java | 54 ----
.../core/serde/SedonaWKBKryoRegistratorTest.java | 55 ----
docs/download/databricks.md | 18 --
docs/tutorial/core-python.md | 148 +++++----
docs/tutorial/rdd.md | 6 +-
docs/tutorial/sql-python.md | 61 ++--
docs/tutorial/sql.md | 6 +-
.../adapters/GeomSerializer.scala | 2 +-
.../adapters/PythonConverter.scala | 5 +-
.../translation/CircleSerializer.scala | 9 +-
.../translation/GeomSerdeUtil.scala | 38 ---
.../translation/GeometrySerializer.scala | 12 +-
.../translation/PythonGeometrySerializer.scala | 23 +-
.../translation/PythonRDDToJavaConverter.scala | 66 +++++
.../translation/serde/PythonGeometrySerde.scala | 29 --
.../serde/PythonGeometrySerialization.scala | 42 ---
.../serde/ShapePythonGeometrySerde.scala | 33 ---
.../python/wrapper/TestToPythonSerialization.scala | 6 +-
python/sedona/core/geom/envelope.py | 2 +-
python/sedona/core/jvm/translate.py | 3 +
python/sedona/core/serde/__init__.py | 0
python/sedona/core/serde/binary/__init__.py | 16 -
python/sedona/core/serde/binary/buffer.py | 67 -----
python/sedona/core/serde/binary/order.py | 23 --
python/sedona/core/serde/binary/parser.py | 111 -------
python/sedona/core/serde/binary/size.py | 30 --
python/sedona/core/serde/geom_factory.py | 52 ----
python/sedona/core/serde/serializer.py | 34 ---
python/sedona/core/serde/shape/__init__.py | 16 -
python/sedona/core/serde/shape/enums.py | 39 ---
python/sedona/core/serde/shape/linestring.py | 55 ----
python/sedona/core/serde/shape/multilinestring.py | 62 ----
python/sedona/core/serde/shape/multipoint.py | 55 ----
python/sedona/core/serde/shape/multipolygon.py | 67 -----
python/sedona/core/serde/shape/offset.py | 32 --
python/sedona/core/serde/shape/point.py | 48 ---
python/sedona/core/serde/shape/polygon.py | 104 -------
python/sedona/core/serde/shape/polyline.py | 62 ----
python/sedona/core/serde/shape/shape.py | 74 -----
python/sedona/core/serde/shape/undefined.py | 36 ---
python/sedona/core/serde/shape/util.py | 71 -----
python/sedona/core/serde/spark_config.py | 49 ---
python/sedona/core/serde/wkb/__init__.py | 16 -
python/sedona/core/serde/wkb/wkb.py | 42 ---
python/sedona/core/spatialOperator/knn_query.py | 2 +-
python/sedona/core/spatialOperator/rdd.py | 1 +
python/sedona/register/geo_registrator.py | 11 -
python/sedona/sql/types.py | 22 +-
python/sedona/utils/adapter.py | 4 +-
python/sedona/utils/binary_parser.py | 143 ++++++++-
python/sedona/utils/geometry_adapter.py | 8 +-
python/sedona/utils/spatial_rdd_parser.py | 93 +++---
python/tests/conftest.py | 10 -
.../core/test_avoiding_python_jvm_serde_df.py | 2 -
python/tests/core/test_geo_data.py | 37 ---
python/tests/serialization/test_deserializers.py | 14 +-
.../serialization/test_direct_serialization.py | 24 +-
.../tests/serialization/test_rdd_serialization.py | 25 +-
python/tests/serialization/test_serializers.py | 2 -
.../tests/serialization/test_with_sc_parellize.py | 17 +-
python/tests/serializer_type.txt | 1 -
.../test_join_query_correctness.py | 5 +-
.../tests/spatial_operator/test_linestring_join.py | 1 -
python/tests/spatial_rdd/test_circle_rdd.py | 5 +-
python/tests/spatial_rdd/test_polygon_rdd.py | 5 +-
python/tests/sql/test_adapter.py | 15 -
python/tests/sql/test_function.py | 330 +++++----------------
.../sql/test_spatial_rdd_to_spatial_dataframe.py | 2 +
...ircle_rdd.py => test_assign_raw_spatial_rdd.py} | 27 +-
python/tests/test_base.py | 20 +-
python/tests/tools.py | 4 -
python/tests/utils/test_crs_transformation.py | 2 +-
.../apache/sedona/sql/serde/SedonaSerializer.scala | 74 -----
.../sedona/sql/utils/GeometrySerializer.scala | 39 ++-
.../spark/sql/sedona_sql/UDT/GeometryUDT.scala | 6 +-
.../sql/sedona_sql/expressions/Constructors.scala | 23 +-
.../sql/sedona_sql/expressions/Functions.scala | 90 +++---
.../sql/sedona_sql/expressions/Predicates.scala | 44 +--
.../sql/sedona_sql/expressions/implicits.scala | 8 +-
.../org/apache/spark/sql/sedona_sql/package.scala | 37 ---
.../strategy/join/DistanceJoinExec.scala | 4 +-
.../strategy/join/TraitJoinQueryBase.scala | 6 +-
.../apache/sedona/sql/TestBaseWKBSerdeJava.java | 71 -----
.../sedona/sql/WKBGeometrySerializerTestJava.java | 51 ----
.../org/apache/sedona/sql/adapterTestJava.java | 49 ++-
.../org/apache/sedona/sql/adapterTestScala.scala | 5 -
.../apache/sedona/sql/constructorTestScala.scala | 14 +
.../org/apache/sedona/sql/functionTestScala.scala | 6 +-
.../sql/sedona_viz/expressions/Pixelize.scala | 6 +-
104 files changed, 915 insertions(+), 2948 deletions(-)
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 6678964..f20c61a 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -18,27 +18,18 @@ jobs:
- spark: 3.1.1
scala: 2.12.8
python: 3.7
- serializer: wkb
- spark: 3.1.1
scala: 2.12.8
python: 3.8
- serializer: wkb
- spark: 3.1.1
scala: 2.12.8
python: 3.9
- serializer: wkb
- spark: 3.0.2
scala: 2.12.8
python: 3.7
- serializer: wkb
- spark: 2.4.7
scala: 2.11.8
python: 3.7
- serializer: wkb
- - spark: 3.1.1
- scala: 2.12.8
- python: 3.8
- serializer: shape
steps:
- uses: actions/checkout@v2
@@ -79,4 +70,4 @@ jobs:
run: find python-adapter/target -name sedona-* -exec cp {} spark-${SPARK_VERSION}-bin-hadoop2.7/jars/ \;
- env:
SPARK_VERSION: ${{ matrix.spark }}
- run: (export SPARK_HOME=$PWD/spark-${SPARK_VERSION}-bin-hadoop2.7;export PYTHONPATH=$SPARK_HOME/python;cd python;echo ${{ matrix.serializer }} > tests/serializer_type.txt;pipenv run pytest tests)
\ No newline at end of file
+ run: (export SPARK_HOME=$PWD/spark-${SPARK_VERSION}-bin-hadoop2.7;export PYTHONPATH=$SPARK_HOME/python;cd python;pipenv run pytest tests)
\ No newline at end of file
diff --git a/core/src/main/java/org/apache/sedona/core/enums/SerializerType.java b/core/src/main/java/org/apache/sedona/core/enums/SerializerType.java
deleted file mode 100644
index f13a851..0000000
--- a/core/src/main/java/org/apache/sedona/core/enums/SerializerType.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sedona.core.enums;
-
-/**
- * Spark Kryo Serializer type
- */
-public enum SerializerType {
-
- SHAPE(0),
- WKB(1);
-
- final int id;
-
- public int getId() {
- return id;
- }
-
- SerializerType(int id)
- {
- this.id = id;
- }
-
- /**
- * Gets the serializer type.
- *
- * @param str the str
- * @return the index type
- */
- public static SerializerType getSerializerType(String str)
- {
- for (SerializerType me : SerializerType.values()) {
- if (me.name().equalsIgnoreCase(str)) { return me; }
- }
- return null;
- }
-
- public static SerializerType fromId(int id)
- {
- for (SerializerType type : values()) {
- if (type.id == id) {
- return type;
- }
- }
- return null;
- }
-}
diff --git a/core/src/main/java/org/apache/sedona/core/serde/GeometrySerde.java b/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
old mode 100644
new mode 100755
similarity index 75%
rename from core/src/main/java/org/apache/sedona/core/serde/GeometrySerde.java
rename to core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
index 65086c0..213e8f2
--- a/core/src/main/java/org/apache/sedona/core/serde/GeometrySerde.java
+++ b/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
@@ -17,52 +17,48 @@
* under the License.
*/
-package org.apache.sedona.core.serde;
+package org.apache.sedona.core.geometryObjects;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Registration;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
-import org.apache.sedona.core.enums.SerializerType;
-import org.apache.sedona.core.geometryObjects.Circle;
-import org.locationtech.jts.geom.*;
+import org.apache.sedona.core.formatMapper.shapefileParser.parseUtils.shp.ShapeSerde;
+import org.locationtech.jts.geom.Envelope;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryCollection;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.geom.LineString;
+import org.locationtech.jts.geom.MultiLineString;
+import org.locationtech.jts.geom.MultiPoint;
+import org.locationtech.jts.geom.MultiPolygon;
+import org.locationtech.jts.geom.Point;
+import org.locationtech.jts.geom.Polygon;
/**
* Provides methods to efficiently serialize and deserialize geometry types.
* <p>
- * Supports Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPoly
+ * Supports Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon,
* GeometryCollection, Circle and Envelope types.
* <p>
* First byte contains {@link Type#id}. Then go type-specific bytes, followed
* by user-data attached to the geometry.
- *
- * User need to implement readGeometry and writeGeometry using a specific SerDe format
- * for example: {@link org.apache.sedona.core.serde.shape.ShapeGeometrySerde}
*/
-abstract public class GeometrySerde extends Serializer {
-
- protected static final Logger log = Logger.getLogger(GeometrySerde.class.getName());
- protected static final GeometryFactory geometryFactory = new GeometryFactory();
-
- /**
- * Write the serialized type as Int from {@link org.apache.sedona.core.enums.SerializerType}
- * Used mainly by the python binding to know which serde to use
- * @param out kryo output stream
- * @param type SerializerType
- */
- protected void writeSerializedType(Output out, SerializerType type){
- out.writeByte(type.getId());
- }
+public class GeometrySerde
+ extends Serializer
+{
+
+ private static final Logger log = Logger.getLogger(GeometrySerde.class);
+ private static final GeometryFactory geometryFactory = new GeometryFactory();
@Override
public void write(Kryo kryo, Output out, Object object)
{
if (object instanceof Circle) {
Circle circle = (Circle) object;
- writeType(out, GeometrySerde.Type.CIRCLE);
+ writeType(out, Type.CIRCLE);
out.writeDouble(circle.getRadius());
writeGeometry(kryo, out, circle.getCenterGeometry());
writeUserData(kryo, out, circle);
@@ -70,12 +66,12 @@ abstract public class GeometrySerde extends Serializer {
else if (object instanceof Point || object instanceof LineString
|| object instanceof Polygon || object instanceof MultiPoint
|| object instanceof MultiLineString || object instanceof MultiPolygon) {
- writeType(out, GeometrySerde.Type.SHAPE);
+ writeType(out, Type.SHAPE);
writeGeometry(kryo, out, (Geometry) object);
}
else if (object instanceof GeometryCollection) {
GeometryCollection collection = (GeometryCollection) object;
- writeType(out, GeometrySerde.Type.GEOMETRYCOLLECTION);
+ writeType(out, Type.GEOMETRYCOLLECTION);
out.writeInt(collection.getNumGeometries());
for (int i = 0; i < collection.getNumGeometries(); i++) {
writeGeometry(kryo, out, collection.getGeometryN(i));
@@ -84,7 +80,7 @@ abstract public class GeometrySerde extends Serializer {
}
else if (object instanceof Envelope) {
Envelope envelope = (Envelope) object;
- writeType(out, GeometrySerde.Type.ENVELOPE);
+ writeType(out, Type.ENVELOPE);
out.writeDouble(envelope.getMinX());
out.writeDouble(envelope.getMaxX());
out.writeDouble(envelope.getMinY());
@@ -96,14 +92,19 @@ abstract public class GeometrySerde extends Serializer {
}
}
- private void writeType(Output out, GeometrySerde.Type type)
+ private void writeType(Output out, Type type)
{
out.writeByte((byte) type.id);
}
- protected abstract void writeGeometry(Kryo kryo, Output out, Geometry geometry);
+ private void writeGeometry(Kryo kryo, Output out, Geometry geometry)
+ {
+ byte[] data = ShapeSerde.serialize(geometry);
+ out.write(data, 0, data.length);
+ writeUserData(kryo, out, geometry);
+ }
- protected void writeUserData(Kryo kryo, Output out, Geometry geometry)
+ private void writeUserData(Kryo kryo, Output out, Geometry geometry)
{
out.writeBoolean(geometry.getUserData() != null);
if (geometry.getUserData() != null) {
@@ -116,7 +117,7 @@ abstract public class GeometrySerde extends Serializer {
public Object read(Kryo kryo, Input input, Class aClass)
{
byte typeId = input.readByte();
- GeometrySerde.Type geometryType = GeometrySerde.Type.fromId(typeId);
+ Type geometryType = Type.fromId(typeId);
switch (geometryType) {
case SHAPE:
return readGeometry(kryo, input);
@@ -152,7 +153,7 @@ abstract public class GeometrySerde extends Serializer {
}
}
- protected Object readUserData(Kryo kryo, Input input)
+ private Object readUserData(Kryo kryo, Input input)
{
Object userData = null;
if (input.readBoolean()) {
@@ -162,7 +163,12 @@ abstract public class GeometrySerde extends Serializer {
return userData;
}
- protected abstract Geometry readGeometry(Kryo kryo, Input input);
+ private Geometry readGeometry(Kryo kryo, Input input)
+ {
+ Geometry geometry = ShapeSerde.deserialize(input, geometryFactory);
+ geometry.setUserData(readUserData(kryo, input));
+ return geometry;
+ }
private enum Type
{
@@ -178,9 +184,9 @@ abstract public class GeometrySerde extends Serializer {
this.id = id;
}
- public static GeometrySerde.Type fromId(int id)
+ public static Type fromId(int id)
{
- for (GeometrySerde.Type type : values()) {
+ for (Type type : values()) {
if (type.id == id) {
return type;
}
diff --git a/core/src/main/java/org/apache/sedona/core/serde/spatialindex/SpatialIndexSerde.java b/core/src/main/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerde.java
similarity index 92%
rename from core/src/main/java/org/apache/sedona/core/serde/spatialindex/SpatialIndexSerde.java
rename to core/src/main/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerde.java
index 72ca10e..a8e5cbe 100644
--- a/core/src/main/java/org/apache/sedona/core/serde/spatialindex/SpatialIndexSerde.java
+++ b/core/src/main/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerde.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.sedona.core.serde.spatialindex;
+package org.apache.sedona.core.geometryObjects;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
@@ -28,8 +28,6 @@ import org.locationtech.jts.index.quadtree.IndexSerde;
import org.locationtech.jts.index.quadtree.Quadtree;
import org.locationtech.jts.index.strtree.STRtree;
-import java.util.Objects;
-
/**
* Provides methods to efficiently serialize and deserialize spatialIndex types.
* <p>
@@ -43,9 +41,18 @@ public class SpatialIndexSerde
private static final Logger log = Logger.getLogger(SpatialIndexSerde.class);
+ private final GeometrySerde geometrySerde;
+
public SpatialIndexSerde()
{
super();
+ geometrySerde = new GeometrySerde();
+ }
+
+ public SpatialIndexSerde(GeometrySerde geometrySerde)
+ {
+ super();
+ this.geometrySerde = geometrySerde;
}
@Override
@@ -76,7 +83,7 @@ public class SpatialIndexSerde
{
byte typeID = input.readByte();
Type indexType = Type.fromId(typeID);
- switch (Objects.requireNonNull(indexType, "indexType cannot be null")) {
+ switch (indexType) {
case QUADTREE: {
IndexSerde indexSerde = new IndexSerde();
return indexSerde.read(kryo, input);
diff --git a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java
index dfb5962..57894ea 100644
--- a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java
+++ b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java
@@ -20,22 +20,47 @@
package org.apache.sedona.core.serde;
import com.esotericsoftware.kryo.Kryo;
-import org.apache.sedona.core.serde.shape.ShapeGeometrySerde;
-import org.apache.sedona.core.serde.spatialindex.SpatialIndexSerde;
+import org.apache.log4j.Logger;
+import org.apache.sedona.core.geometryObjects.Circle;
+import org.apache.sedona.core.geometryObjects.GeometrySerde;
+import org.apache.sedona.core.geometryObjects.SpatialIndexSerde;
import org.apache.spark.serializer.KryoRegistrator;
+import org.locationtech.jts.geom.Envelope;
+import org.locationtech.jts.geom.GeometryCollection;
+import org.locationtech.jts.geom.LineString;
+import org.locationtech.jts.geom.MultiLineString;
+import org.locationtech.jts.geom.MultiPoint;
+import org.locationtech.jts.geom.MultiPolygon;
+import org.locationtech.jts.geom.Point;
+import org.locationtech.jts.geom.Polygon;
+import org.locationtech.jts.index.quadtree.Quadtree;
+import org.locationtech.jts.index.strtree.STRtree;
-/**
- * Register Kryo classes using the Geometry Serde(using the ShapeFile serialization)
- * and SpatialIndexSerde for index objects
- */
public class SedonaKryoRegistrator
- implements KryoRegistrator {
+ implements KryoRegistrator
+{
+
+ final static Logger log = Logger.getLogger(SedonaKryoRegistrator.class);
@Override
- public void registerClasses(Kryo kryo) {
- ShapeGeometrySerde serializer = new ShapeGeometrySerde();
- SpatialIndexSerde indexSerializer = new SpatialIndexSerde();
+ public void registerClasses(Kryo kryo)
+ {
+ GeometrySerde serializer = new GeometrySerde();
+ SpatialIndexSerde indexSerializer = new SpatialIndexSerde(serializer);
+
+ log.info("Registering custom serializers for geometry types");
- SedonaKryoRegistratorHelper.registerClasses(kryo, serializer, indexSerializer);
+ kryo.register(Point.class, serializer);
+ kryo.register(LineString.class, serializer);
+ kryo.register(Polygon.class, serializer);
+ kryo.register(MultiPoint.class, serializer);
+ kryo.register(MultiLineString.class, serializer);
+ kryo.register(MultiPolygon.class, serializer);
+ kryo.register(GeometryCollection.class, serializer);
+ kryo.register(Circle.class, serializer);
+ kryo.register(Envelope.class, serializer);
+ // TODO: Replace the default serializer with default spatial index serializer
+ kryo.register(Quadtree.class, indexSerializer);
+ kryo.register(STRtree.class, indexSerializer);
}
}
diff --git a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistratorHelper.java b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistratorHelper.java
deleted file mode 100644
index 42ab294..0000000
--- a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistratorHelper.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.core.serde;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.Serializer;
-import org.apache.log4j.Logger;
-import org.apache.sedona.core.geometryObjects.Circle;
-import org.apache.sedona.core.serde.shape.ShapeGeometrySerde;
-import org.apache.sedona.core.serde.WKB.WKBGeometrySerde;
-import org.apache.sedona.core.serde.spatialindex.SpatialIndexSerde;
-import org.locationtech.jts.geom.*;
-import org.locationtech.jts.index.quadtree.Quadtree;
-import org.locationtech.jts.index.strtree.STRtree;
-
-/**
- * Register Kryo classes by the type of geometry serde mechanism
- */
-public class SedonaKryoRegistratorHelper {
-
- final static Logger log = Logger.getLogger(SedonaWKBKryoRegistrator.class);
-
- public static void registerClasses(Kryo kryo, Serializer geometrySerdeType, SpatialIndexSerde indexSerializer) {
- log.info("Registering custom serializers for geometry types");
-
- kryo.register(Point.class, geometrySerdeType);
- kryo.register(LineString.class, geometrySerdeType);
- kryo.register(Polygon.class, geometrySerdeType);
- kryo.register(MultiPoint.class, geometrySerdeType);
- kryo.register(MultiLineString.class, geometrySerdeType);
- kryo.register(MultiPolygon.class, geometrySerdeType);
- kryo.register(GeometryCollection.class, geometrySerdeType);
- kryo.register(Circle.class, geometrySerdeType);
- kryo.register(Envelope.class, geometrySerdeType);
-
- kryo.register(Quadtree.class, indexSerializer);
- kryo.register(STRtree.class, indexSerializer);
- }
-
-}
diff --git a/core/src/main/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistrator.java b/core/src/main/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistrator.java
deleted file mode 100644
index adc5047..0000000
--- a/core/src/main/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistrator.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.core.serde;
-
-import com.esotericsoftware.kryo.Kryo;
-import org.apache.sedona.core.serde.spatialindex.SpatialIndexSerde;
-import org.apache.sedona.core.serde.WKB.WKBGeometrySerde;
-import org.apache.spark.serializer.KryoRegistrator;
-
-/**
- * Register Kryo classes using the WKB Geometry Serde and SpatialIndexSerde for index objects
- */
-public class SedonaWKBKryoRegistrator
- implements KryoRegistrator {
-
- @Override
- public void registerClasses(Kryo kryo) {
- WKBGeometrySerde serializer = new WKBGeometrySerde();
- SpatialIndexSerde indexSerializer = new SpatialIndexSerde();
-
- SedonaKryoRegistratorHelper.registerClasses(kryo, serializer, indexSerializer);
- }
-}
diff --git a/core/src/main/java/org/apache/sedona/core/serde/WKB/WKBGeometrySerde.java b/core/src/main/java/org/apache/sedona/core/serde/WKB/WKBGeometrySerde.java
deleted file mode 100755
index ca323cb..0000000
--- a/core/src/main/java/org/apache/sedona/core/serde/WKB/WKBGeometrySerde.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.core.serde.WKB;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-import org.apache.sedona.core.enums.SerializerType;
-import org.apache.sedona.core.serde.GeometrySerde;
-import org.locationtech.jts.geom.Geometry;
-import org.locationtech.jts.io.ParseException;
-import org.locationtech.jts.io.WKBReader;
-import org.locationtech.jts.io.WKBWriter;
-
-/**
- * Specific implementation of GeometrySerde using the WKB format
- */
-public class WKBGeometrySerde
- extends GeometrySerde
-{
-
- @Override
- protected void writeGeometry(Kryo kryo, Output out, Geometry geometry)
- {
- WKBWriter writer = new WKBWriter(2, 2, true);
- byte[] data = writer.write(geometry);
-
- // Mainly used by the python binding to know in which serde to use
- writeSerializedType(out, SerializerType.WKB);
-
- // write geometry length size to read bytes until userData
- out.writeInt(data.length);
-
- out.write(data, 0, data.length);
- writeUserData(kryo, out, geometry);
- }
-
- @Override
- protected Geometry readGeometry(Kryo kryo, Input input) {
- WKBReader reader = new WKBReader();
- Geometry geometry;
-
- // Skip the unneeded Serialized Type
- input.readByte();
- int geometryBytesLength = input.readInt();
- byte[] bytes = input.readBytes(geometryBytesLength);
-
- try {
- geometry = reader.read(bytes);
- geometry.setUserData(readUserData(kryo, input));
- } catch (ParseException e) {
- log.error("Cannot parse geometry bytes", e);
- return null;
- }
-
- return geometry;
- }
-}
diff --git a/core/src/main/java/org/apache/sedona/core/serde/shape/ShapeGeometrySerde.java b/core/src/main/java/org/apache/sedona/core/serde/shape/ShapeGeometrySerde.java
deleted file mode 100755
index 60d9e80..0000000
--- a/core/src/main/java/org/apache/sedona/core/serde/shape/ShapeGeometrySerde.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.core.serde.shape;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-import org.apache.log4j.Logger;
-import org.apache.sedona.core.enums.SerializerType;
-import org.apache.sedona.core.formatMapper.shapefileParser.parseUtils.shp.ShapeSerde;
-import org.apache.sedona.core.serde.GeometrySerde;
-import org.locationtech.jts.geom.Geometry;
-import org.locationtech.jts.geom.GeometryFactory;
-
-/**
- * Specific implementation of GeometrySerde using the Shape format
- */
-public class ShapeGeometrySerde
- extends GeometrySerde
-{
- @Override
- protected void writeGeometry(Kryo kryo, Output out, Geometry geometry)
- {
- // Mainly used by the python binding to know in which serde to use
- writeSerializedType(out, SerializerType.SHAPE);
-
- byte[] data = ShapeSerde.serialize(geometry);
- out.write(data, 0, data.length);
- writeUserData(kryo, out, geometry);
- }
-
- @Override
- protected Geometry readGeometry(Kryo kryo, Input input)
- {
- // Skip the unneeded Serialized Type
- input.readByte();
- Geometry geometry = ShapeSerde.deserialize(input, geometryFactory);
- geometry.setUserData(readUserData(kryo, input));
- return geometry;
- }
-}
diff --git a/core/src/main/java/org/apache/sedona/core/utils/SedonaConf.java b/core/src/main/java/org/apache/sedona/core/utils/SedonaConf.java
index aca42cb..1b8b2b7 100644
--- a/core/src/main/java/org/apache/sedona/core/utils/SedonaConf.java
+++ b/core/src/main/java/org/apache/sedona/core/utils/SedonaConf.java
@@ -19,7 +19,10 @@
package org.apache.sedona.core.utils;
-import org.apache.sedona.core.enums.*;
+import org.apache.sedona.core.enums.GridType;
+import org.apache.sedona.core.enums.IndexType;
+import org.apache.sedona.core.enums.JoinBuildSide;
+import org.apache.sedona.core.enums.JoinSparitionDominantSide;
import org.apache.spark.SparkConf;
import org.locationtech.jts.geom.Envelope;
@@ -36,8 +39,6 @@ public class SedonaConf
private IndexType indexType = IndexType.QUADTREE;
- private SerializerType serializerType = SerializerType.SHAPE;
-
// Parameters for JoinQuery including RangeJoin and DistanceJoin
private JoinSparitionDominantSide joinSparitionDominantSide = JoinSparitionDominantSide.LEFT;
@@ -56,8 +57,6 @@ public class SedonaConf
{
this.useIndex = sparkConf.getBoolean("sedona.global.index", true);
this.indexType = IndexType.getIndexType(sparkConf.get("sedona.global.indextype", "quadtree"));
- this.serializerType = SerializerType.getSerializerType(sparkConf.get("sedona.serializer.type", "shape"));
-
this.joinApproximateTotalCount = sparkConf.getLong("sedona.join.approxcount", -1);
String[] boundaryString = sparkConf.get("sedona.join.boundary", "0,0,0,0").split(",");
this.datasetBoundary = new Envelope(Double.parseDouble(boundaryString[0]), Double.parseDouble(boundaryString[0]),
@@ -83,8 +82,6 @@ public class SedonaConf
return indexType;
}
- public SerializerType getSerializerType() { return serializerType; }
-
public void setIndexType(IndexType indexType)
{
this.indexType = indexType;
diff --git a/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java b/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
index 6d796af..5dd6556 100644
--- a/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
+++ b/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
@@ -16,7 +16,7 @@ package org.locationtech.jts.index.quadtree;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.sedona.core.serde.shape.ShapeGeometrySerde;
+import org.apache.sedona.core.geometryObjects.GeometrySerde;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
@@ -29,9 +29,9 @@ import java.util.List;
*/
public class IndexSerde
{
- ShapeGeometrySerde geometrySerde;
+ GeometrySerde geometrySerde;
public IndexSerde() {
- geometrySerde = new ShapeGeometrySerde();
+ geometrySerde = new GeometrySerde();
}
public Object read(Kryo kryo, Input input){
diff --git a/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java b/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
index 6904b98..bf3d308 100644
--- a/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
+++ b/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
@@ -16,7 +16,7 @@ package org.locationtech.jts.index.strtree;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.sedona.core.serde.shape.ShapeGeometrySerde;
+import org.apache.sedona.core.geometryObjects.GeometrySerde;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
@@ -29,9 +29,9 @@ import java.util.List;
*/
public class IndexSerde
{
- ShapeGeometrySerde geometrySerde;
+ GeometrySerde geometrySerde;
public IndexSerde() {
- geometrySerde = new ShapeGeometrySerde();
+ geometrySerde = new GeometrySerde();
}
public Object read(Kryo kryo, Input input){
diff --git a/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java b/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java
new file mode 100644
index 0000000..3bb9af4
--- /dev/null
+++ b/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.core.formatMapper.shapefileParser.shapes;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import org.apache.sedona.core.geometryObjects.Circle;
+import org.apache.sedona.core.geometryObjects.GeometrySerde;
+import org.apache.sedona.core.utils.GeomUtils;
+import org.junit.Test;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryCollection;
+import org.locationtech.jts.io.ParseException;
+import org.locationtech.jts.io.WKTReader;
+
+import java.io.ByteArrayOutputStream;
+
+import static org.junit.Assert.assertTrue;
+
+public class GeometrySerdeTest
+{
+ private final Kryo kryo = new Kryo();
+ private final WKTReader wktReader = new WKTReader();
+
+ @Test
+ public void test()
+ throws Exception
+ {
+ test("POINT (1.3 4.5)");
+ test("MULTIPOINT ((1 1), (1.3 4.5), (5.2 999))");
+ test("LINESTRING (1 1, 1.3 4.5, 5.2 999)");
+ test("MULTILINESTRING ((1 1, 1.3 4.5, 5.2 999), (0 0, 0 1))");
+ test("POLYGON ((0 0, 0 1, 1 1, 1 0.4, 0 0))");
+ test("POLYGON ((0 0, 0 1, 1 1, 1 0.4, 0 0), (0.2 0.2, 0.5 0.2, 0.5 0.5, 0.2 0.5, 0.2 0.2))");
+ test("MULTIPOLYGON (((0 0, 0 1, 1 1, 1 0.4, 0 0)), " +
+ "((0 0, 0 1, 1 1, 1 0.4, 0 0), (0.2 0.2, 0.5 0.2, 0.5 0.5, 0.2 0.5, 0.2 0.2)))");
+ test("GEOMETRYCOLLECTION (POINT(4 6), LINESTRING(4 6,7 10))");
+ }
+
+ private void test(String wkt)
+ throws Exception
+ {
+ Geometry geometry = parseWkt(wkt);
+ assertTrue(GeomUtils.equalsExactGeom(geometry, serde(geometry)));
+
+ geometry.setUserData("This is a test");
+ assertTrue(GeomUtils.equalsExactGeom(geometry, serde(geometry)));
+
+ if (geometry instanceof GeometryCollection) {
+ return;
+ }
+
+ Circle circle = new Circle(geometry, 1.2);
+ assertTrue(GeomUtils.equalsExactGeom(circle, serde(circle)));
+ }
+
+ private Geometry parseWkt(String wkt)
+ throws ParseException
+ {
+ return wktReader.read(wkt);
+ }
+
+ private Geometry serde(Geometry input)
+ {
+ byte[] ser = serialize(input);
+ return kryo.readObject(new Input(ser), input.getClass());
+ }
+
+ private byte[] serialize(Geometry input)
+ {
+ kryo.register(input.getClass(), new GeometrySerde());
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ Output output = new Output(bos);
+ kryo.writeObject(output, input);
+ output.close();
+ return bos.toByteArray();
+ }
+}
diff --git a/core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java b/core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java
index 99bb46c..b44ee21 100644
--- a/core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java
+++ b/core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java
@@ -22,7 +22,6 @@ package org.apache.sedona.core.geometryObjects;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.sedona.core.serde.spatialindex.SpatialIndexSerde;
import org.junit.Test;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.Envelope;
@@ -70,7 +69,7 @@ public class SpatialIndexSerdeTest
compareSize(Quadtree.class);
compareSize(STRtree.class);
- // workbench of compare time
+// // workbench of compare time
compareTime(Quadtree.class);
compareTime(STRtree.class);
}
diff --git a/core/src/test/java/org/apache/sedona/core/serde/GeometrySerdeTest.java b/core/src/test/java/org/apache/sedona/core/serde/GeometrySerdeTest.java
deleted file mode 100644
index da153f2..0000000
--- a/core/src/test/java/org/apache/sedona/core/serde/GeometrySerdeTest.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.core.serde;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-import org.apache.sedona.core.geometryObjects.Circle;
-import org.apache.sedona.core.serde.WKB.WKBGeometrySerde;
-import org.apache.sedona.core.serde.shape.ShapeGeometrySerde;
-import org.apache.sedona.core.utils.GeomUtils;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.locationtech.jts.geom.Geometry;
-import org.locationtech.jts.geom.GeometryCollection;
-import org.locationtech.jts.io.ParseException;
-import org.locationtech.jts.io.WKTReader;
-
-import java.io.ByteArrayOutputStream;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * To add serde test case, you need to add the following functions: (WKB for instance)
- * 1. @Test testUsingWKBSerde
- * 2. private testUsingWKBSerde
- * 3. wkbSerde
- */
-public class GeometrySerdeTest
-{
- private final Kryo kryo = new Kryo();
- private final WKTReader wktReader = new WKTReader();
-
- private final String POINT = "POINT (1.3 4.5)";
- private final String MULTIPOINT = "MULTIPOINT ((1 1), (1.3 4.5), (5.2 999))";
- private final String LINESTRING = "LINESTRING (1 1, 1.3 4.5, 5.2 999)";
- private final String MULTILINESTRING = "MULTILINESTRING ((1 1, 1.3 4.5, 5.2 999), (0 0, 0 1))";
- private final String POLYGON = "POLYGON ((0 0, 0 1, 1 1, 1 0.4, 0 0))";
- private final String POLYGON_WITH_HOLE = "POLYGON ((0 0, 0 1, 1 1, 1 0.4, 0 0), " +
- "(0.2 0.2, 0.5 0.2, 0.5 0.5, 0.2 0.5, 0.2 0.2))";
- private final String MULTIPOLYGON = "MULTIPOLYGON (((0 0, 0 1, 1 1, 1 0.4, 0 0)), " +
- "((0 0, 0 1, 1 1, 1 0.4, 0 0), (0.2 0.2, 0.5 0.2, 0.5 0.5, 0.2 0.5, 0.2 0.2)))";
- private final String GEOMETRYCOLLECTION = "GEOMETRYCOLLECTION (POINT(4 6), LINESTRING(4 6,7 10))";
-
- @Test
- public void testUsingShapeSerde()
- throws Exception
- {
- testUsingShapeSerde(POINT);
- testUsingShapeSerde(MULTIPOINT);
- testUsingShapeSerde(LINESTRING);
- testUsingShapeSerde(MULTILINESTRING);
- testUsingShapeSerde(POLYGON);
- testUsingShapeSerde(POLYGON_WITH_HOLE);
- testUsingShapeSerde(MULTIPOLYGON);
- testUsingShapeSerde(GEOMETRYCOLLECTION);
- }
-
- @Test
- public void testUsingWKBSerde()
- throws Exception
- {
- testUsingWKBSerde(POINT);
- testUsingWKBSerde(MULTIPOINT);
- testUsingWKBSerde(LINESTRING);
- testUsingWKBSerde(MULTILINESTRING);
- testUsingWKBSerde(POLYGON);
- testUsingWKBSerde(POLYGON_WITH_HOLE);
- testUsingWKBSerde(MULTIPOLYGON);
- testUsingWKBSerde(GEOMETRYCOLLECTION);
- }
-
- private void testUsingShapeSerde(String wkt)
- throws Exception
- {
- Geometry geometry = parseWkt(wkt);
- assertTrue(GeomUtils.equalsExactGeom(geometry, shapeSerde(geometry)));
-
- geometry.setUserData("This is a test");
- assertTrue(GeomUtils.equalsExactGeom(geometry, shapeSerde(geometry)));
-
- if (geometry instanceof GeometryCollection) {
- return;
- }
-
- Circle circle = new Circle(geometry, 1.2);
- assertTrue(GeomUtils.equalsExactGeom(circle, shapeSerde(circle)));
- }
-
- private void testUsingWKBSerde(String wkt)
- throws Exception
- {
- Geometry geometry = parseWkt(wkt);
- assertTrue(GeomUtils.equalsExactGeom(geometry, wkbSerde(geometry)));
-
- geometry.setUserData("This is a test");
- assertTrue(GeomUtils.equalsExactGeom(geometry, wkbSerde(geometry)));
-
- if (geometry instanceof GeometryCollection) {
- return;
- }
-
- Circle circle = new Circle(geometry, 1.2);
- assertTrue(GeomUtils.equalsExactGeom(circle, wkbSerde(circle)));
- }
-
- private Geometry parseWkt(String wkt)
- throws ParseException
- {
- return wktReader.read(wkt);
- }
-
- private Geometry shapeSerde(Geometry input)
- {
- ShapeGeometrySerde serde = new ShapeGeometrySerde();
- byte[] ser = serialize(input, serde);
- return kryo.readObject(new Input(ser), input.getClass());
- }
-
- private Geometry wkbSerde(Geometry input)
- {
- WKBGeometrySerde serde = new WKBGeometrySerde();
- byte[] ser = serialize(input, serde);
- return kryo.readObject(new Input(ser), input.getClass());
- }
-
- private byte[] serialize(Geometry input, GeometrySerde serde)
- {
- kryo.register(input.getClass(), serde);
-
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- Output output = new Output(bos);
- kryo.writeObject(output, input);
- output.close();
- return bos.toByteArray();
- }
-}
diff --git a/core/src/test/java/org/apache/sedona/core/serde/SedonaKryoRegistratorTest.java b/core/src/test/java/org/apache/sedona/core/serde/SedonaKryoRegistratorTest.java
deleted file mode 100644
index c828126..0000000
--- a/core/src/test/java/org/apache/sedona/core/serde/SedonaKryoRegistratorTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package org.apache.sedona.core.serde;
-
-import com.esotericsoftware.kryo.Kryo;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.sedona.core.TestBase;
-import org.apache.sedona.core.serde.shape.ShapeGeometrySerde;
-import org.apache.sedona.core.serde.spatialindex.SpatialIndexSerde;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.serializer.KryoSerializer;
-import org.junit.*;
-import org.locationtech.jts.geom.Point;
-
-import static org.junit.Assert.assertEquals;
-
-public class SedonaKryoRegistratorTest extends TestBase {
-
- public static JavaSparkContext sc;
- public static SparkConf conf;
-
- @BeforeClass
- public static void onceExecutedBeforeAll()
- {
- conf = new SparkConf()
- .setAppName(SedonaKryoRegistratorTest.class.getName())
- .setMaster("local[2]")
- .set("spark.serializer", KryoSerializer.class.getName());
-
- Logger.getLogger("org").setLevel(Level.WARN);
- Logger.getLogger("akka").setLevel(Level.WARN);
- }
-
- @After
- public void tearDown()
- throws Exception
- {
- sc.stop();
- }
-
- @Test
- public void testRegistration() {
- conf.set("spark.kryo.registrator", SedonaKryoRegistrator.class.getName());
- sc = new JavaSparkContext(conf);
-
- Kryo kryo = new Kryo();
- ShapeGeometrySerde shapeGeometrySerde = new ShapeGeometrySerde();
- SpatialIndexSerde indexSerializer = new SpatialIndexSerde();
-
- SedonaKryoRegistratorHelper.registerClasses(kryo, shapeGeometrySerde, indexSerializer);
-
- assertEquals(ShapeGeometrySerde.class.getName(), kryo.getSerializer(Point.class).getClass().getName());
- }
-}
diff --git a/core/src/test/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistratorTest.java b/core/src/test/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistratorTest.java
deleted file mode 100644
index d0ac671..0000000
--- a/core/src/test/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistratorTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-package org.apache.sedona.core.serde;
-
-import com.esotericsoftware.kryo.Kryo;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.sedona.core.TestBase;
-import org.apache.sedona.core.serde.WKB.WKBGeometrySerde;
-import org.apache.sedona.core.serde.spatialindex.SpatialIndexSerde;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.serializer.KryoSerializer;
-import org.junit.*;
-import org.locationtech.jts.geom.Point;
-
-import static org.junit.Assert.assertEquals;
-
-public class SedonaWKBKryoRegistratorTest extends TestBase {
-
- public static JavaSparkContext sc;
- public static SparkConf conf;
-
- @BeforeClass
- public static void onceExecutedBeforeAll()
- {
- conf = new SparkConf()
- .setAppName(SedonaWKBKryoRegistratorTest.class.getName())
- .setMaster("local[2]")
- .set("spark.serializer", KryoSerializer.class.getName());
-
- Logger.getLogger("org").setLevel(Level.WARN);
- Logger.getLogger("akka").setLevel(Level.WARN);
- }
-
- @After
- public void tearDown()
- throws Exception
- {
- sc.stop();
- }
-
- @Test
- public void testRegistration()
- {
- conf.set("spark.kryo.registrator", SedonaWKBKryoRegistrator.class.getName());
- sc = new JavaSparkContext(conf);
-
- Kryo kryo = new Kryo();
- WKBGeometrySerde wkbGeometrySerde = new WKBGeometrySerde();
- SpatialIndexSerde indexSerializer = new SpatialIndexSerde();
-
- SedonaKryoRegistratorHelper.registerClasses(kryo, wkbGeometrySerde, indexSerializer);
-
- assertEquals(WKBGeometrySerde.class.getName(), kryo.getSerializer(Point.class).getClass().getName());
- }
-}
diff --git a/docs/download/databricks.md b/docs/download/databricks.md
deleted file mode 100644
index e68d9b4..0000000
--- a/docs/download/databricks.md
+++ /dev/null
@@ -1,18 +0,0 @@
-# Installation on Databricks
-
-## Community edition (free-tier)
-
-You just need to install the Sedona jars and Sedona Python on Databricks using Databricks default web UI. Then everything will work.
-
-## Advanced editions
-
-If you are not using the free version of Databricks, there is an issue with the path where Sedona Python looks for the jar. Thanks to the report from Sedona user @amoyrand.
-
-Two steps to fix this:
-
-1. Upload the jars in `/dbfs/FileStore/jars/`
-2. Add the jar path to Spark session config. For example, `.config("spark.jars", "/dbfs/FileStore/jars/sedona-python-adapter-3.0_2.12-1.0.1-incubating.jar") \`.
-
-## Pure SQL environment
-
-Sedona cannot be used in [a pure SQL environment](../../tutorial/sql-pure-sql) (e.g., an SQL notebook) on Databricks. You have to mix it with Scala or Python in order to call `SedonaSQLRegistrator.registerAll(sparkSession)`. Please see a similar report on [Stackoverflow](https://stackoverflow.com/questions/66721168/sparksessionextensions-injectfunction-in-databricks-environment).
\ No newline at end of file
diff --git a/docs/tutorial/core-python.md b/docs/tutorial/core-python.md
index cbf91c4..a4d7f1d 100644
--- a/docs/tutorial/core-python.md
+++ b/docs/tutorial/core-python.md
@@ -1,9 +1,36 @@
# Spatial RDD Applications in Python
## Introduction
-
+<div style="text-align: justify">
Sedona provides a Python wrapper on Sedona core Java/Scala library.
-Sedona SpatialRDDs (and other classes when it was necessary) have implemented meta classes which allow to use overloaded functions, methods and constructors to be the most similar to Java/Scala API as possible.
+Sedona SpatialRDDs (and other classes when it was necessary) have implemented meta classes which allow
+to use overloaded functions, methods and constructors to be the most similar to Java/Scala API as possible.
+</div>
+
+Apache Sedona core provides five special SpatialRDDs:
+
+<li> PointRDD </li>
+<li> PolygonRDD </li>
+<li> LineStringRDD </li>
+<li> CircleRDD </li>
+<li> RectangleRDD </li>
+<div style="text-align: justify">
+<p>
+All of them can be imported from <b> sedona.core.SpatialRDD </b> module
+
+<b> sedona </b> has written serializers which convert Sedona SpatialRDD to Python objects.
+Converting will produce GeoData objects which have 2 attributes:
+</p>
+</div>
+<li> geom: shapely.geometry.BaseGeometry </li>
+<li> userData: str </li>
+
+geom attribute holds geometry representation as shapely objects.
+userData is string representation of other attributes separated by "\t"
+</br>
+
+GeoData has one method to get user data.
+<li> getUserData() -> str </li>
!!!note
This tutorial is based on [Sedona Core Jupyter Notebook example](../jupyter-notebook). You can interact with Sedona Python Jupyter notebook immediately on Binder. Click [](https://mybinder.org/v2/gh/apache/incubator-sedona/HEAD?filepath=binder) and wait for a few minutes. Then select a notebook and enjoy!
@@ -18,43 +45,25 @@ Sedona has a suite of well-written geometry and index serializers. Forgetting to
```python
conf.set("spark.serializer", KryoSerializer.getName)
conf.set("spark.kryo.registrator", SedonaKryoRegistrator.getName)
-conf.set("sedona.serializer.type", "shape") # Optional: default value = shape
sc = SparkContext(conf=conf)
```
-
-!!!note
- You can use multiple serialization type by changing spark conf option ```sedona.serializer.type```. Currently two options are available `shape`, `wkb`. The former is more performant while the latter solved a bug that you can find here: [ST_GeomFromWkt and ST_GeomFromText converts multipolygon to polygon](https://docs.google.com/spreadsheets/d/e/2PACX-1vQ4G8zwdfVhvoJAMWmBAGWNDTtE32Oh0dk95BF5HL996k0YCRDyU0cXry1xwuTGcQ4sixB0NTc4K4-0/pubhtml). The performance comparison can be found here: [ [...]
-
-## Geometry object in Sedona Python
-
-JVM Geometry in Sedona is binded to a Python GeoData object. It has the following attribtues:
-
-* geom: shapely.geometry.BaseGeometry. It holds geometry representation as shapely objects.
-* userData: string. It is a string representation of other attributes separated by "\t". You can call GeoData.getUserData() to obtain the string.
-* serde: int. It has two possible values takes one of the two values: 0 (Shapefile serializer) or 1 (WKB serialzer)
-
-
-You can create a Python GeoData object by using
-
-```python
-GeoData.with_current_serialization(geom=Point(21, 52), userData="user_data")
-```
-
-
+
## Create a SpatialRDD
### Create a typed SpatialRDD
Apache Sedona core provides three special SpatialRDDs:
-
-* PointRDD
-* PolygonRDD
-* LineStringRDD
+<li> PointRDD </li>
+<li> PolygonRDD </li>
+<li> LineStringRDD </li>
+<li> CircleRDD </li>
+<li> RectangleRDD </li>
+<br>
They can be loaded from CSV, TSV, WKT, WKB, Shapefiles, GeoJSON formats.
-To pass the format to SpatialRDD constructor please use <b> FileDataSplitter </b> enumeration.
+To pass the format to SpatialRDD constructor please use <b> FileDataSplitter </b> enumeration.
-Sedona SpatialRDDs (and other classes when it was necessary) have implemented meta classes which allow
-to use overloaded functions how Scala/Java Apache Sedona API allows. For example:
+sedona SpatialRDDs (and other classes when it was necessary) have implemented meta classes which allow
+to use overloaded functions how Scala/Java Apache Sedona API allows. ex.
```python
@@ -85,7 +94,7 @@ point_rdd = PointRDD(
#### From SparkSQL DataFrame
-To create spatialRDD from other formats, you can use adapter between Spark DataFrame and SpatialRDD
+To create spatialRDD from other formats you can use adapter between Spark DataFrame and SpatialRDD
<li> Load data in SedonaSQL. </li>
@@ -143,7 +152,7 @@ spatial_rdd = Adapter.toSpatialRdd(spatial_df, "geom")
```
For WKT/WKB/GeoJSON data, please use ==ST_GeomFromWKT / ST_GeomFromWKB / ST_GeomFromGeoJSON== instead.
-
+
## Read other attributes in an SpatialRDD
Each SpatialRDD can carry non-spatial attributes such as price, age and name as long as the user sets ==carryOtherAttributes== as [TRUE](#create-a-spatialrdd).
@@ -168,8 +177,11 @@ query_result = RangeQuery.SpatialRangeQuery(spatial_rdd, range_query_window, con
```
!!!note
- Please use RangeQueryRaw from the same module if you want to avoid jvm python serde while converting to Spatial DataFrame. It takes the same parameters as RangeQuery but returns reference to jvm rdd which can be converted to dataframe without python - jvm serde using Adapter.
-
+ Please use RangeQueryRaw from the same module
+ if you want to avoid jvm python serde while converting to Spatial DataFrame
+ It takes the same parameters as RangeQuery but returns reference to jvm rdd which
+ can be converted to dataframe without python - jvm serde using Adapter.
+
Example:
```python
from sedona.core.geom.envelope import Envelope
@@ -186,7 +198,7 @@ query_result = RangeQuery.SpatialRangeQuery(spatial_rdd, range_query_window, con
### Range query window
-Besides the rectangle (Envelope) type range query window, Apache Sedona range query window can be
+Besides the rectangle (Envelope) type range query window, Apache Sedona range query window can be
<li> Point </li>
<li> Polygon </li>
<li> LineString </li>
@@ -230,7 +242,7 @@ query_result = RangeQuery.SpatialRangeQuery(
The output format of the spatial range query is another RDD which consists of GeoData objects.
-SpatialRangeQuery result can be used as RDD with map or other spark RDD funtions. Also it can be used as
+SpatialRangeQuery result can be used as RDD with map or other spark RDD funtions. Also it can be used as
Python objects when using collect method.
Example:
@@ -282,7 +294,7 @@ result = KNNQuery.SpatialKnnQuery(object_rdd, point, k, using_index)
### Query center geometry
-Besides the Point type, Apache Sedona KNN query center can be
+Besides the Point type, Apache Sedona KNN query center can be
<li> Polygon </li>
<li> LineString </li>
@@ -307,11 +319,11 @@ result = KNNQuery.SpatialKnnQuery(spatial_rdd, point, k, using_index)
```
!!!warning
- Only R-Tree index supports Spatial KNN query
+ Only R-Tree index supports Spatial KNN query
### Output format
-The output format of the spatial KNN query is a list of GeoData objects.
+The output format of the spatial KNN query is a list of GeoData objects.
The list has K GeoData objects.
Example:
@@ -343,8 +355,8 @@ query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
result = JoinQuery.SpatialJoinQuery(object_rdd, query_window_rdd, using_index, consider_boundary_intersection)
```
-Result of SpatialJoinQuery is RDD which consists of GeoData instance and list of GeoData instances which spatially intersects or
-are covered by GeoData.
+Result of SpatialJoinQuery is RDD which consists of GeoData instance and list of GeoData instances which spatially intersects or
+are covered by GeoData.
```python
result.collect())
@@ -373,7 +385,7 @@ object_rdd.spatialPartitioning(GridType.KDBTREE)
query_window_rdd.spatialPartitioning(object_rdd.getPartitioner())
```
-Or
+Or
```python
query_window_rdd.spatialPartitioning(GridType.KDBTREE)
@@ -417,7 +429,7 @@ Polygon,LineString
...
```
-example
+example
```python
result.collect()
```
@@ -444,7 +456,7 @@ A distance join query takes two spatial RDD assuming that we have two SpatialRDD
And finds the geometries (from spatial_rdd) are within given distance to it. spatial_rdd and object_rdd
can be any geometry type (point, line, polygon) and are not necessary to have the same geometry type
-
+
You can use the following code to issue an Distance Join Query on them.
```python
@@ -467,7 +479,39 @@ result = JoinQuery.DistanceJoinQueryFlat(spatial_rdd, circle_rdd, using_index, c
```
!!!note
- Please use JoinQueryRaw from the same module for methods if you want to convert the results to DataFrame using Adapter.
+ Please use JoinQueryRaw from the same module for methods
+
+ - spatialJoin
+
+ - DistanceJoinQueryFlat
+
+ - SpatialJoinQueryFlat
+
+ For better performance while converting to dataframe with adapter.
+ That approach allows to avoid costly serialization between Python
+ and jvm and in result operating on python object instead of native geometries.
+
+ Example:
+ ```python
+ from sedona.core.SpatialRDD import CircleRDD
+ from sedona.core.enums import GridType
+ from sedona.core.spatialOperator import JoinQueryRaw
+
+ object_rdd.analyze()
+
+ circle_rdd = CircleRDD(object_rdd, 0.1) ## Create a CircleRDD using the given distance
+ circle_rdd.analyze()
+
+ circle_rdd.spatialPartitioning(GridType.KDBTREE)
+ spatial_rdd.spatialPartitioning(circle_rdd.getPartitioner())
+
+ consider_boundary_intersection = False ## Only return gemeotries fully covered by each query window in queryWindowRDD
+ using_index = False
+
+ result = JoinQueryRaw.DistanceJoinQueryFlat(spatial_rdd, circle_rdd, using_index, consider_boundary_intersection)
+
+ gdf = Adapter.toDf(result, ["left_col1", ..., "lefcoln"], ["rightcol1", ..., "rightcol2"], spark)
+ ```
### Output format
@@ -495,13 +539,13 @@ result.map(lambda x: x[0].geom.centroid).collect()
...
]
```
-
+
## Save to permanent storage
You can always save an SpatialRDD back to some permanent storage such as HDFS and Amazon S3. You can save distributed SpatialRDD to WKT, GeoJSON and object files.
!!!note
- Non-spatial attributes such as price, age and name will also be stored to permanent storage.
+ Non-spatial attributes such as price, age and name will also be stored to permanent storage.
### Save an SpatialRDD (not indexed)
@@ -542,7 +586,7 @@ object_rdd.rawJvmSpatialRDD.saveAsObjectFile("hdfs://PATH")
```
!!!note
- Each object in a distributed object file is a byte array (not human-readable). This byte array is the serialized format of a Geometry or a SpatialIndex.
+ Each object in a distributed object file is a byte array (not human-readable). This byte array is the serialized format of a Geometry or a SpatialIndex.
### Save an SpatialRDD (indexed)
@@ -600,7 +644,6 @@ from sedona.core.formatMapper import WktReader
WktReader.readToGeometryRDD(sc, wkt_geometries_location, 0, True, False)
```
-
```
<sedona.core.SpatialRDD.spatial_rdd.SpatialRDD at 0x7f8fd2fbf250>
```
@@ -611,7 +654,6 @@ from sedona.core.formatMapper import WkbReader
WkbReader.readToGeometryRDD(sc, wkb_geometries_location, 0, True, False)
```
-
```
<sedona.core.SpatialRDD.spatial_rdd.SpatialRDD at 0x7f8fd2eece50>
```
@@ -622,7 +664,6 @@ from sedona.core.formatMapper import GeoJsonReader
GeoJsonReader.readToGeometryRDD(sc, geo_json_file_location)
```
-
```
<sedona.core.SpatialRDD.spatial_rdd.SpatialRDD at 0x7f8fd2eecb90>
```
@@ -633,7 +674,6 @@ from sedona.core.formatMapper.shapefileParser import ShapefileReader
ShapefileReader.readToGeometryRDD(sc, shape_file_location)
```
-
```
<sedona.core.SpatialRDD.spatial_rdd.SpatialRDD at 0x7f8fd2ee0710>
```
@@ -650,8 +690,8 @@ When you use Sedona functions such as
- RangeQuery.SpatialRangeQuery
For better performance when converting to dataframe you can use
-JoinQueryRaw and RangeQueryRaw from the same module and adapter to convert
-to Spatial DataFrame.
+JoinQueryRaw and RangeQueryRaw from the same module and adapter to convert
+to Spatial DataFrame.
Example, JoinQueryRaw:
diff --git a/docs/tutorial/rdd.md b/docs/tutorial/rdd.md
index 338e651..f5924ec 100644
--- a/docs/tutorial/rdd.md
+++ b/docs/tutorial/rdd.md
@@ -4,11 +4,11 @@ The page outlines the steps to create Spatial RDDs and run spatial queries using
## Set up dependencies
1. Read [Sedona Maven Central coordinates](../download/maven-coordinates.md)
-2. Select ==the minimum dependencies==: Add Apache Spark (only the Spark core) and `sedona-python-adapter`.
+2. Select ==the minimum dependencies==: Add Apache Spark (only the Spark core) and Sedona (core).
3. Add the dependencies in build.sbt or pom.xml.
!!!note
- To enjoy the full functions of Sedona, we suggest you include ==the full dependencies==: [Apache Spark core](https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11), [Apache SparkSQL](https://mvnrepository.com/artifact/org.apache.spark/spark-sql), `sedona-python-adapter`, `sedona-viz`. Please see [RDD example project](/tutorial/demo/)
+ To enjoy the full functions of Sedona, we suggest you include ==the full dependencies==: [Apache Spark core](https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11), [Apache SparkSQL](https://mvnrepository.com/artifact/org.apache.spark/spark-sql), Sedona-core, Sedona-SQL, Sedona-Viz. Please see [RDD example project](/tutorial/demo/)
## Initiate SparkContext
@@ -23,7 +23,7 @@ val sc = new SparkContext(conf)
```
!!!warning
- Sedona equips customized geometry and index serializers to reduce memory consumption. Two sets of serializers are available: (1) **SedonaKryoRegistrator** - uses the Shapefile format (default serde) (2) **SedonaWKBKryoRegistrator** - uses the WKB format. The former is more performant while the latter solved a bug that you can find here: [ST_GeomFromWkt and ST_GeomFromText converts multipolygon to polygon](https://docs.google.com/spreadsheets/d/e/2PACX-1vQ4G8zwdfVhvoJAMWmBAGWNDTtE32Oh0dk [...]
+ Sedona has a suite of well-written geometry and index serializers. Forgetting to enable these serializers will lead to high memory consumption.
If you add ==the Sedona full dependencies== as suggested above, please use the following two lines to enable Sedona Kryo serializer instead:
```Scala
diff --git a/docs/tutorial/sql-python.md b/docs/tutorial/sql-python.md
index 7e1b009..1620b98 100644
--- a/docs/tutorial/sql-python.md
+++ b/docs/tutorial/sql-python.md
@@ -2,10 +2,10 @@
## Introduction
-This package is an extension to Apache Spark SQL package. It allow to use
+This package is an extension to Apache Spark SQL package. It allow to use
spatial functions on dataframes.
-SedonaSQL supports SQL/MM Part3 Spatial SQL Standard.
+SedonaSQL supports SQL/MM Part3 Spatial SQL Standard.
It includes four kinds of SQL operators as follows.
All these operators can be directly called through:
@@ -15,7 +15,7 @@ spark.sql("YOUR_SQL")
!!!note
This tutorial is based on [Sedona SQL Jupyter Notebook example](../jupyter-notebook). You can interact with Sedona Python Jupyter notebook immediately on Binder. Click [](https://mybinder.org/v2/gh/apache/incubator-sedona/HEAD?filepath=binder) and wait for a few minutes. Then select a notebook and enjoy!
-
+
## Installation
Please read [Quick start](/download/overview/#install-sedona-python) to install Sedona Python.
@@ -36,27 +36,40 @@ You can also register functions by passing `--conf spark.sql.extensions=org.apac
Use KryoSerializer.getName and SedonaKryoRegistrator.getName class properties to reduce memory impact.
```python
-spark = SparkSession.
- builder.
- master("local[*]").
- appName("Sedona App").
- config("spark.serializer", KryoSerializer.getName).
- config("spark.kryo.registrator", SedonaKryoRegistrator.getName) .
- config("sedona.serializer.type", "shape") . # Optional: default value = shape
- getOrCreate()
+spark = SparkSession.\
+ builder.\
+ master("local[*]").\
+ appName("Sedona App").\
+ config("spark.serializer", KryoSerializer.getName).\
+ config("spark.kryo.registrator", SedonaKryoRegistrator.getName) .\
+ getOrCreate()
```
-!!!note
- You can use multiple serialization type by changing spark conf option ```sedona.serializer.type```. Currently two options are available `shape`, `wkb`. The former is more performant while the latter solved a bug that you can find here: [ST_GeomFromWkt and ST_GeomFromText converts multipolygon to polygon](https://docs.google.com/spreadsheets/d/e/2PACX-1vQ4G8zwdfVhvoJAMWmBAGWNDTtE32Oh0dk95BF5HL996k0YCRDyU0cXry1xwuTGcQ4sixB0NTc4K4-0/pubhtml). The performance comparison can be found here: [ [...]
+To turn on SedonaSQL function inside pyspark code use SedonaRegistrator.registerAll method on existing pyspark.sql.SparkSession instance ex.
+
+`SedonaRegistrator.registerAll(spark)`
+After that all the functions from SedonaSQL are available,
+moreover using collect or toPandas methods on Spark DataFrame
+returns Shapely BaseGeometry objects.
-### Create Spatial DataFrame
+Based on GeoPandas DataFrame,
+Pandas DataFrame with shapely objects or Sequence with
+shapely objects, Spark DataFrame can be created using
+spark.createDataFrame method. To specify Schema with
+geometry inside please use `GeometryType()` instance
+(look at examples section to see that in practice).
+
+
+### Examples
+
+### SedonaSQL
All SedonaSQL functions (list depends on SedonaSQL version) are available in Python API.
For details please refer to API/SedonaSQL page.
-#### Read CSV/TSV
+For example use SedonaSQL for Spatial Join.
```python3
@@ -75,7 +88,6 @@ counties_geom = spark.sql(
counties_geom.show(5)
```
-
```
+-----------+--------------------+
|county_code| geometry|
@@ -87,9 +99,6 @@ counties_geom.show(5)
| 1427|POLYGON ((19.5087...|
+-----------+--------------------+
```
-
-#### Read Shapefile
-
```python3
import geopandas as gpd
@@ -113,8 +122,6 @@ points_geom.show(5, False)
+---------+-----------------------------+
```
-### Run spatial Join
-
```python3
points_geom.createOrReplaceTempView("pois")
@@ -131,7 +138,6 @@ spatial_join_result = spark.sql(
spatial_join_result.explain()
```
-
```
== Physical Plan ==
*(2) Project [county_code#230, fclass#239]
@@ -140,9 +146,6 @@ spatial_join_result.explain()
+- Project [county_code#230, st_geomfromwkt(geom#232) AS geometry#236]
+- *(1) FileScan csv [county_code#230,geom#232] Batched: false, Format: CSV, Location: InMemoryFileIndex[file:/projects/sedona/counties.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<county_code:string,geom:string>
```
-
-### Aggregate results
-
Calculating Number of Pois within counties per fclass.
```python3
@@ -253,9 +256,9 @@ gdf.plot(
<br>
<br>
-## Interact shapely objects in Spatial DataFrame
+## Creating Spark DataFrame based on shapely objects
-### Create DataFrame using Shapely objects
+### Supported Shapely objects
| shapely object | Available |
|-----------------|--------------------|
@@ -287,6 +290,8 @@ schema = StructType(
Also Spark DataFrame with geometry type can be converted to list of shapely objects with <b> collect </b> method.
+## Example usage for Shapely objects
+
### Point
```python
@@ -496,4 +501,4 @@ gdf.show(1, False)
|1 |MULTIPOLYGON (((0 0, 0 2, 2 2, 2 0, 0 0), (1 1, 1.5 1, 1.5 1.5, 1 1.5, 1 1)), ((0 0, 0 1, 1 1, 1 0, 0 0)))|
+---+----------------------------------------------------------------------------------------------------------+
-```
\ No newline at end of file
+```
diff --git a/docs/tutorial/sql.md b/docs/tutorial/sql.md
index e6d1279..80431ff 100644
--- a/docs/tutorial/sql.md
+++ b/docs/tutorial/sql.md
@@ -10,11 +10,11 @@ Detailed SedonaSQL APIs are available here: [SedonaSQL API](../api/sql/Overview.
## Set up dependencies
1. Read [Sedona Maven Central coordinates](../download/maven-coordinates.md)
-2. Select ==the minimum dependencies==: Add [Apache Spark core](https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11), [Apache SparkSQL](https://mvnrepository.com/artifact/org.apache.spark/spark-sql), `sedona-python-adapter`
+2. Select ==the minimum dependencies==: Add [Apache Spark core](https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11), [Apache SparkSQL](https://mvnrepository.com/artifact/org.apache.spark/spark-sql), Sedona-core and Sedona-SQL
3. Add the dependencies in build.sbt or pom.xml.
!!!note
- To enjoy the full functions of Sedona, we suggest you include ==the full dependencies==: [Apache Spark core](https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11), [Apache SparkSQL](https://mvnrepository.com/artifact/org.apache.spark/spark-sql), `sedona-python-adapter`, `sedona-viz`. Please see [SQL example project](/tutorial/demo/)
+ To enjoy the full functions of Sedona, we suggest you include ==the full dependencies==: [Apache Spark core](https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11), [Apache SparkSQL](https://mvnrepository.com/artifact/org.apache.spark/spark-sql), Sedona-core, Sedona-SQL, Sedona-Viz. Please see [SQL example project](/tutorial/demo/)
## Initiate SparkSession
@@ -30,7 +30,7 @@ var sparkSession = SparkSession.builder()
```
!!!warning
- Sedona equips customized geometry and index serializers to reduce memory consumption. Two sets of serializers are available: (1) **SedonaKryoRegistrator** - uses the Shapefile format (default serde) (2) **SedonaWKBKryoRegistrator** - uses the WKB format. The former is more performant while the latter solved a bug that you can find here: [ST_GeomFromWkt and ST_GeomFromText converts multipolygon to polygon](https://docs.google.com/spreadsheets/d/e/2PACX-1vQ4G8zwdfVhvoJAMWmBAGWNDTtE32Oh0dk [...]
+ Sedona has a suite of well-written geometry and index serializers. Forgetting to enable these serializers will lead to high memory consumption.
If you add ==the Sedona full dependencies== as suggested above, please use the following two lines to enable Sedona Kryo serializer instead:
```Scala
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/GeomSerializer.scala b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/GeomSerializer.scala
index 7d2fbcf..4432550 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/GeomSerializer.scala
+++ b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/GeomSerializer.scala
@@ -22,5 +22,5 @@ package org.apache.sedona.python.wrapper.adapters
import org.apache.sedona.python.wrapper.translation.PythonGeometrySerializer
trait GeomSerializer {
- val geometrySerializer = PythonGeometrySerializer()
+ val geometrySerializer = new PythonGeometrySerializer()
}
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/PythonConverter.scala b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/PythonConverter.scala
index 06ceb89..8065a55 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/PythonConverter.scala
+++ b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/PythonConverter.scala
@@ -19,7 +19,7 @@
package org.apache.sedona.python.wrapper.adapters
-import org.apache.sedona.python.wrapper.translation.{FlatPairRddConverter, GeometryRddConverter, GeometrySeqToPythonConverter, ListPairRddConverter}
+import org.apache.sedona.python.wrapper.translation.{FlatPairRddConverter, GeometryRddConverter, GeometrySeqToPythonConverter, ListPairRddConverter, PythonRDDToJavaConverter}
import org.apache.spark.api.java.{JavaPairRDD, JavaRDD}
import org.locationtech.jts.geom.Geometry
@@ -36,6 +36,9 @@ object PythonConverter extends GeomSerializer {
def translateSpatialPairRDDWithListToPython(spatialRDD: JavaPairRDD[Geometry, java.util.List[Geometry]]): JavaRDD[Array[Byte]] =
ListPairRddConverter(spatialRDD, geometrySerializer).translateToPython
+ def translatePythonRDDToJava(pythonRDD: JavaRDD[Array[Byte]]): JavaRDD[Geometry] =
+ PythonRDDToJavaConverter(pythonRDD, geometrySerializer).translateToJava
+
def translateGeometrySeqToPython(spatialData: SeqWrapper[Geometry]): Array[Array[Byte]] =
GeometrySeqToPythonConverter(spatialData, geometrySerializer).translateToPython
}
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/CircleSerializer.scala b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/CircleSerializer.scala
index 0c3861a..19db081 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/CircleSerializer.scala
+++ b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/CircleSerializer.scala
@@ -20,20 +20,21 @@
package org.apache.sedona.python.wrapper.translation
import org.apache.sedona.core.geometryObjects.Circle
-import org.apache.sedona.python.wrapper.translation.serde.PythonGeometrySerialization
import org.apache.sedona.python.wrapper.utils.implicits._
-import org.apache.spark.sql.sedona_sql.{sedonaSerializer, userSerializerType}
+import org.locationtech.jts.io.WKBWriter
case class CircleSerializer(geometry: Circle) {
private val isCircle = Array(1.toByte)
def serialize: Array[Byte] = {
- val serializedGeometry = PythonGeometrySerialization.serialize(geometry.getCenterGeometry)
+ val wkbWriter = new WKBWriter(2, 2)
+ val serializedGeom = wkbWriter.write(geometry.getCenterGeometry)
val userDataBinary = geometry.userDataToUtf8ByteArray
val userDataLengthArray = userDataBinary.length.toByteArray()
+ val serializedGeomLength = serializedGeom.length.toByteArray()
val radius = geometry.getRadius.toDouble
- isCircle ++ userDataLengthArray ++ serializedGeometry ++ userDataBinary ++ radius.toByteArray()
+ isCircle ++ serializedGeomLength ++ userDataLengthArray ++ serializedGeom ++ userDataBinary ++ radius.toByteArray()
}
}
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeomSerdeUtil.scala b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeomSerdeUtil.scala
deleted file mode 100644
index 7cc85f5..0000000
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeomSerdeUtil.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sedona.python.wrapper.translation
-
-import org.apache.sedona.core.enums.SerializerType
-import org.apache.sedona.core.formatMapper.shapefileParser.parseUtils.shp.ShapeSerde
-import org.locationtech.jts.geom.{Geometry, GeometryFactory}
-import org.locationtech.jts.io.{WKBReader, WKBWriter}
-
-
-abstract class PythonGeometrySerde extends Serializable {
- def serialize(geom: Geometry): Array[Byte]
- def deserialize: Array[Byte] => Geometry
-}
-
-case class PythonShapeSerde() extends PythonGeometrySerde with Serializable {
- override def serialize(geom: Geometry): Array[Byte] =
- ShapeSerde.serialize(geom)
-
- override def deserialize: Array[Byte] => Geometry = (array: Array[Byte]) =>
- ShapeSerde.deserialize(array, new GeometryFactory())
-}
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometrySerializer.scala b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometrySerializer.scala
index 0d43f9d..662a4b8 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometrySerializer.scala
+++ b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometrySerializer.scala
@@ -19,21 +19,21 @@
package org.apache.sedona.python.wrapper.translation
-import org.apache.sedona.python.wrapper.translation.serde.PythonGeometrySerialization
import org.apache.sedona.python.wrapper.utils.implicits._
-import org.apache.spark.sql.sedona_sql.{sedonaSerializer, userSerializerType}
import org.locationtech.jts.geom.Geometry
+import org.locationtech.jts.io.WKBWriter
+
case class GeometrySerializer(geometry: Geometry) {
private val notCircle = Array(0.toByte)
def serialize: Array[Byte] = {
- val serializedGeometry = PythonGeometrySerialization.serialize(geometry)
- val serializedGeom = serializedGeometry
+ val wkbWriter = new WKBWriter(2, 2)
+ val serializedGeom = wkbWriter.write(geometry)
val userDataBinary = geometry.userDataToUtf8ByteArray
val userDataLengthArray = userDataBinary.length.toByteArray()
- notCircle ++ userDataLengthArray ++ serializedGeom ++ userDataBinary
+ val serializedGeomLength = serializedGeom.length.toByteArray()
+ notCircle ++ serializedGeomLength ++ userDataLengthArray ++ serializedGeom ++ userDataBinary
}
}
-
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonGeometrySerializer.scala b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonGeometrySerializer.scala
index 48f1949..c0c69c7 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonGeometrySerializer.scala
+++ b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonGeometrySerializer.scala
@@ -21,44 +21,39 @@ package org.apache.sedona.python.wrapper.translation
import java.nio.ByteBuffer
-import com.esotericsoftware.kryo.io.Output
import org.apache.sedona.core.geometryObjects.Circle
import org.apache.sedona.python.wrapper.SerializationException
-import org.apache.sedona.python.wrapper.translation.serde.PythonGeometrySerialization
-import org.apache.spark.sql.sedona_sql.{sedonaSerializer, userSerializerType}
-import org.locationtech.jts.geom.{Geometry, GeometryCollection, LineString, Point, Polygon}
+import org.locationtech.jts.geom.Geometry
import org.locationtech.jts.io.WKBReader
-private[python] case class PythonGeometrySerializer() extends Serializable {
+private[python] class PythonGeometrySerializer extends Serializable {
/*
Translates JTS geometry to byte array which then will be decoded to Python shapely geometry objects
Process needs two steps:
+ - Translate JTS Geometry to Byte array using WKBWriter
- Translate user attributes using UTF-8 encoding
*/
-
- def serialize(geom: Geometry): Array[Byte] = {
- geom match {
- case geometry: Circle => CircleSerializer(geometry).serialize
- case geometry: Geometry => GeometrySerializer(geometry).serialize
- }
-
+ def serialize: (Geometry => Array[Byte]) = {
+ case geometry: Circle => CircleSerializer(geometry).serialize
+ case geometry: Geometry => GeometrySerializer(geometry).serialize
}
def deserialize: Array[Byte] => Geometry = (values: Array[Byte]) => {
+ val reader = new WKBReader()
val isCircle = values.head.toInt
val valuesLength = values.length
if (isCircle == 1) {
- val geom = PythonGeometrySerialization.deserialize(values.slice(9, valuesLength))
+ val geom = reader.read(values.slice(9, valuesLength))
val radius = ByteBuffer.wrap(values.slice(1, 9)).getDouble()
new Circle(geom, radius)
}
else if (isCircle == 0) {
- PythonGeometrySerialization.deserialize(values.slice(1, valuesLength))
+ reader.read(values.slice(1, valuesLength))
}
else throw SerializationException("Can not deserialize object")
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonRDDToJavaConverter.scala b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonRDDToJavaConverter.scala
new file mode 100644
index 0000000..470d07f
--- /dev/null
+++ b/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonRDDToJavaConverter.scala
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.python.wrapper.translation
+
+import java.io.{ByteArrayInputStream, DataInputStream}
+
+import org.apache.sedona.core.geometryObjects.Circle
+import org.apache.sedona.python.wrapper.SerializationException
+import org.apache.spark.api.java.JavaRDD
+import org.locationtech.jts.geom.Geometry
+
+case class PythonRDDToJavaConverter(javaRDD: JavaRDD[Array[Byte]], geometrySerializer: PythonGeometrySerializer) {
+ def translateToJava: JavaRDD[Geometry] = {
+ javaRDD.rdd.map[Geometry](serializedGeoData => {
+ val geoDataBytes = new ByteArrayInputStream(serializedGeoData)
+ val geoDataInputBytes = new DataInputStream(geoDataBytes)
+ val rddType = geoDataInputBytes.readInt()
+ val isCircle = geoDataInputBytes.readByte().toInt
+ if (isCircle == 1) {
+ val radius = geoDataInputBytes.readDouble()
+ val geometry = readGeometry(serializedGeoData, geoDataInputBytes, 20)
+ new Circle(geometry, radius)
+ }
+ if (isCircle == 0) {
+ readGeometry(serializedGeoData, geoDataInputBytes, 12)
+ }
+ else throw SerializationException()
+
+
+ }
+
+ )
+ }
+
+ private def readGeometry(serializedGeoData: Array[Byte], inputStream: DataInputStream, skipBytes: Int): Geometry = {
+ val geomDataLength = java.lang.Integer.reverseBytes(inputStream.readInt())
+ val userDataLength = java.lang.Integer.reverseBytes(inputStream.readInt())
+
+ val toReadGeometry = serializedGeoData.slice(skipBytes, skipBytes + geomDataLength + 1)
+ val geom = geometrySerializer.deserialize(toReadGeometry)
+
+ val userData = serializedGeoData.slice(skipBytes + geomDataLength + 1, skipBytes + geomDataLength + 1 + userDataLength)
+ .map(_.toChar).mkString
+
+ geom.setUserData(userData)
+ geom
+ }
+}
+
diff --git a/python-adapter/src/main/scala/org/apache/sedona/python/wrapper/translation/serde/PythonGeometrySerde.scala b/python-adapter/src/main/scala/org/apache/sedona/python/wrapper/translation/serde/PythonGeometrySerde.scala
deleted file mode 100644
index 430f639..0000000
--- a/python-adapter/src/main/scala/org/apache/sedona/python/wrapper/translation/serde/PythonGeometrySerde.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sedona.python.wrapper.translation.serde
-
-import com.esotericsoftware.kryo.io.Output
-import org.locationtech.jts.geom.Geometry
-
-trait PythonGeometrySerde {
-
- def serialize(geometry: Geometry): Array[Byte]
-}
diff --git a/python-adapter/src/main/scala/org/apache/sedona/python/wrapper/translation/serde/PythonGeometrySerialization.scala b/python-adapter/src/main/scala/org/apache/sedona/python/wrapper/translation/serde/PythonGeometrySerialization.scala
deleted file mode 100644
index 5898a6e..0000000
--- a/python-adapter/src/main/scala/org/apache/sedona/python/wrapper/translation/serde/PythonGeometrySerialization.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.python.wrapper.translation.serde
-
-import org.apache.sedona.core.enums.SerializerType
-import org.apache.spark.sql.sedona_sql.userSerializerType
-import org.locationtech.jts.geom.Geometry
-
-object PythonGeometrySerialization {
- def serialize(geometry: Geometry): Array[Byte] = {
- userSerializerType match {
- case SerializerType.SHAPE => ShapePythonGeometrySerde.serialize(geometry)
- case SerializerType.WKB => WkbPythonGeometrySerde.serialize(geometry)
- case _ => ShapePythonGeometrySerde.serialize(geometry)
- }
- }
-
- def deserialize(array: Array[Byte]): Geometry = {
- userSerializerType match {
- case SerializerType.SHAPE => ShapePythonGeometrySerde.deserialize(array)
- case SerializerType.WKB => WkbPythonGeometrySerde.deserialize(array)
- case _ => ShapePythonGeometrySerde.deserialize(array)
- }
- }
-}
diff --git a/python-adapter/src/main/scala/org/apache/sedona/python/wrapper/translation/serde/ShapePythonGeometrySerde.scala b/python-adapter/src/main/scala/org/apache/sedona/python/wrapper/translation/serde/ShapePythonGeometrySerde.scala
deleted file mode 100644
index e04ccc5..0000000
--- a/python-adapter/src/main/scala/org/apache/sedona/python/wrapper/translation/serde/ShapePythonGeometrySerde.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.python.wrapper.translation.serde
-
-import com.esotericsoftware.kryo.Kryo
-import com.esotericsoftware.kryo.io.Output
-import org.apache.sedona.core.formatMapper.shapefileParser.parseUtils.shp.ShapeSerde
-import org.locationtech.jts.geom.{Geometry, GeometryFactory}
-
-object ShapePythonGeometrySerde extends PythonGeometrySerde {
- def serialize(geometry: Geometry): Array[Byte] =
- Array(0.toByte) ++ ShapeSerde.serialize(geometry)
-
- def deserialize(array: Array[Byte]): Geometry =
- ShapeSerde.deserialize(array.slice(1, array.length), new GeometryFactory())
-}
diff --git a/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala b/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala
index b6ca25c..96503c8 100644
--- a/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala
+++ b/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala
@@ -19,14 +19,10 @@
package org.apache.sedona.python.wrapper
-import org.apache.sedona.python.wrapper.translation.{FlatPairRddConverter, GeometryRddConverter, GeometrySerializer, ListPairRddConverter}
+import org.apache.sedona.python.wrapper.translation.{FlatPairRddConverter, GeometryRddConverter, ListPairRddConverter}
import org.apache.spark.api.java.JavaPairRDD
import org.scalatest.Matchers
import org.apache.sedona.python.wrapper.utils.implicits._
-import org.apache.sedona.sql.serde.SedonaSerializer
-import org.apache.spark.sql.catalyst.util.GenericArrayData
-import org.apache.spark.sql.sedona_sql.userSerializerType
-
import scala.collection.JavaConverters._
diff --git a/python/sedona/core/geom/envelope.py b/python/sedona/core/geom/envelope.py
index 1f47b8b..c0a90eb 100644
--- a/python/sedona/core/geom/envelope.py
+++ b/python/sedona/core/geom/envelope.py
@@ -51,7 +51,7 @@ class Envelope(Polygon):
)
def to_bytes(self):
- from sedona.core.serde.binary.buffer import BinaryBuffer
+ from sedona.utils.binary_parser import BinaryBuffer
bin_buffer = BinaryBuffer()
bin_buffer.put_double(self.minx)
bin_buffer.put_double(self.maxx)
diff --git a/python/sedona/core/jvm/translate.py b/python/sedona/core/jvm/translate.py
index a82abff..bddc596 100644
--- a/python/sedona/core/jvm/translate.py
+++ b/python/sedona/core/jvm/translate.py
@@ -32,6 +32,9 @@ class JvmSedonaPythonConverter:
def translate_spatial_pair_rdd_with_list_to_python(self, spatial_rdd):
return self._jvm.PythonConverter.translateSpatialPairRDDWithListToPython(spatial_rdd)
+ def translate_python_rdd_to_java(self, java_rdd):
+ return self._jvm.PythonConverter.translatePythonRDDToJava(java_rdd)
+
def translate_geometry_seq_to_python(self, geometry_seq) -> List:
return self._jvm.PythonConverter.translateGeometrySeqToPython(geometry_seq)
diff --git a/python/sedona/core/serde/__init__.py b/python/sedona/core/serde/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/python/sedona/core/serde/binary/__init__.py b/python/sedona/core/serde/binary/__init__.py
deleted file mode 100644
index 38cc50c..0000000
--- a/python/sedona/core/serde/binary/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
\ No newline at end of file
diff --git a/python/sedona/core/serde/binary/buffer.py b/python/sedona/core/serde/binary/buffer.py
deleted file mode 100644
index 8945c31..0000000
--- a/python/sedona/core/serde/binary/buffer.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import struct
-
-from sedona.core.serde.binary.order import ByteOrderType
-
-
-class BinaryBuffer:
-
- def __init__(self):
- self.array = []
-
- def put_double(self, value, byte_order=ByteOrderType.BIG_ENDIAN):
- bytes = self.__pack("d", value, byte_order)
- self.__extend_buffer(bytes)
-
- def put_int(self, value, byte_order=ByteOrderType.BIG_ENDIAN):
- bytes = self.__pack("i", value, byte_order)
- self.__extend_buffer(bytes)
-
- def put_byte(self, value, byte_order=ByteOrderType.BIG_ENDIAN):
- bytes = self.__pack("b", value, byte_order)
- self.__extend_buffer(bytes)
-
- def put(self, value):
- self.__extend_buffer(value)
-
- def __pack(self, type, value, byte_order=ByteOrderType.BIG_ENDIAN):
- return struct.pack(byte_order.value + type, value)
-
- def __extend_buffer(self, bytes):
- self.array.extend(list(bytes))
-
- def __translate_values(self, values):
- return [el if el < 128 else el - 256 for el in values]
-
- def add_empty_bytes(self, tp: str, number_of_empty: int):
- if tp == "double":
- for _ in range(number_of_empty):
- self.put_double(0.0)
- elif tp == "int":
- for _ in range(number_of_empty):
- self.put_int(0)
- elif tp == "double":
- for _ in range(number_of_empty):
- self.put_byte(0)
- else:
- raise TypeError(f"Passed {tp} is not available")
-
- @property
- def byte_array(self):
- return self.__translate_values(self.array)
\ No newline at end of file
diff --git a/python/sedona/core/serde/binary/order.py b/python/sedona/core/serde/binary/order.py
deleted file mode 100644
index 531146b..0000000
--- a/python/sedona/core/serde/binary/order.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from enum import Enum
-
-
-class ByteOrderType(Enum):
- LITTLE_ENDIAN = "<"
- BIG_ENDIAN = ">"
diff --git a/python/sedona/core/serde/binary/parser.py b/python/sedona/core/serde/binary/parser.py
deleted file mode 100644
index 4995252..0000000
--- a/python/sedona/core/serde/binary/parser.py
+++ /dev/null
@@ -1,111 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import struct
-from typing import List, Union
-
-import attr
-from pyspark import SparkContext
-from shapely.wkb import loads
-
-from sedona.core.serde.binary.order import ByteOrderType
-from sedona.core.serde.binary.size import BYTE_SIZE, INT_SIZE, size_dict, DOUBLE_SIZE, CHAR_SIZE, BOOLEAN_SIZE
-
-
-@attr.s
-class BinaryParser:
- bytes = attr.ib(type=Union[bytearray, List[int]])
- current_index = attr.ib(default=0)
-
- def __attrs_post_init__(self):
- no_negatives = self.remove_negatives(self.bytes)
- self.bytes = self._convert_to_binary_array(no_negatives)
-
- def read_geometry(self, length: int):
- geom_bytes = b"".join([struct.pack("b", el) if el < 128 else struct.pack("b", el - 256) for el in
- self.bytes[self.current_index: self.current_index + length]])
-
- geom = loads(geom_bytes)
- self.current_index += length
- return geom
-
- def read_double(self, byte_order=ByteOrderType.LITTLE_ENDIAN):
- data = self.unpack("d", self.bytes, byte_order)
- self.current_index = self.current_index + DOUBLE_SIZE
- return data
-
- def read_int(self, byte_order=ByteOrderType.LITTLE_ENDIAN):
- data = self.unpack('i', self.bytes, byte_order)
- self.current_index = self.current_index + INT_SIZE
- return data
-
- def read_byte(self, byte_order=ByteOrderType.LITTLE_ENDIAN):
- data = self.unpack("b", self.bytes, byte_order)
- self.current_index = self.current_index + BYTE_SIZE
- return data
-
- def read_char(self, byte_order=ByteOrderType.LITTLE_ENDIAN):
- data = self.unpack("c", self.bytes, byte_order)
- self.current_index = self.current_index + CHAR_SIZE
- return data
-
- def read_boolean(self, byte_order=ByteOrderType.BIG_ENDIAN):
- data = self.unpack("?", self.bytes, byte_order)
- self.current_index = self.current_index + BOOLEAN_SIZE
- return data
-
- def read_string(self, length: int, encoding: str = "utf8"):
- string = self.bytes[self.current_index: self.current_index + length]
- self.current_index += length
-
- try:
- encoded_string = string.decode(encoding, "ignore")
- except UnicodeEncodeError:
- raise UnicodeEncodeError
- return encoded_string
-
- def read_kryo_string(self, length: int, sc: SparkContext) -> str:
- array_length = length - self.current_index
- byte_array = sc._gateway.new_array(sc._jvm.Byte, array_length)
-
- for index, bt in enumerate(self.bytes[self.current_index: length]):
- byte_array[index] = self.bytes[self.current_index + index]
- decoded_string = sc._jvm.org.imbruced.geo_pyspark.serializers.GeoSerializerData.deserializeUserData(
- byte_array
- )
- self.current_index = length
- return decoded_string
-
- def unpack(self, tp: str, bytes: bytearray, byte_order=ByteOrderType.LITTLE_ENDIAN):
- max_index = self.current_index + size_dict[tp]
- bytes = self._convert_to_binary_array(bytes[self.current_index: max_index])
- return struct.unpack(byte_order.value + tp, bytes)[0]
-
- @classmethod
- def remove_negatives(cls, bytes):
- return [cls.remove_negative(bt) for bt in bytes]
-
- @classmethod
- def remove_negative(cls, byte):
- bt_pos = byte if byte >= 0 else byte + 256
- return bt_pos
-
- @staticmethod
- def _convert_to_binary_array(bytes):
- if type(bytes) == list:
- bytes = bytearray(bytes)
- return bytes
diff --git a/python/sedona/core/serde/binary/size.py b/python/sedona/core/serde/binary/size.py
deleted file mode 100644
index bc3e97d..0000000
--- a/python/sedona/core/serde/binary/size.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-DOUBLE_SIZE = 8
-INT_SIZE = 4
-BYTE_SIZE = 1
-CHAR_SIZE = 1
-BOOLEAN_SIZE = 1
-
-size_dict = {
- "d": DOUBLE_SIZE,
- "i": INT_SIZE,
- "b": BYTE_SIZE,
- "s": CHAR_SIZE,
- "?": BOOLEAN_SIZE
-}
\ No newline at end of file
diff --git a/python/sedona/core/serde/geom_factory.py b/python/sedona/core/serde/geom_factory.py
deleted file mode 100644
index a173b1f..0000000
--- a/python/sedona/core/serde/geom_factory.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from typing import List
-
-from shapely.geometry.base import BaseGeometry
-
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.core.serde.shape.shape import ShapeSerde
-from sedona.core.serde.wkb.wkb import WkbSerde
-
-geometry_serializers = {
- 0: ShapeSerde(),
- 1: WkbSerde()
-}
-
-geometry_serializers_instances = {
- "shape": ShapeSerde(),
- "wkb": WkbSerde()
-}
-
-serializers = {
- "shape": 0,
- "wkb": 1
-}
-
-
-class SerializationFactory:
-
- def __init__(self, geometry_serde: 'GeometrySerde'):
- self.geometry_serde = geometry_serde
-
- def serialize(self, geom: BaseGeometry, binary_buffer: BinaryBuffer) -> List[int]:
- return self.geometry_serde.to_bytes(geom, binary_buffer)
-
- def deserialize(self, bytes: BinaryParser) -> BaseGeometry:
- return self.geometry_serde.geometry_from_bytes(bytes)
diff --git a/python/sedona/core/serde/serializer.py b/python/sedona/core/serde/serializer.py
deleted file mode 100644
index c53821d..0000000
--- a/python/sedona/core/serde/serializer.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from typing import List
-
-from shapely.geometry.base import BaseGeometry
-
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.parser import BinaryParser
-
-
-class GeometrySerde:
-
- byte_number = None
-
- def deserialize(self, bin_parser: BinaryParser) -> BaseGeometry:
- raise NotImplementedError("child class should implement method geometry from bytes")
-
- def serialize(self, geom: BaseGeometry, buffer: BinaryBuffer) -> List[int]:
- raise NotImplementedError("child class should implement method to bytes")
diff --git a/python/sedona/core/serde/shape/__init__.py b/python/sedona/core/serde/shape/__init__.py
deleted file mode 100644
index 38cc50c..0000000
--- a/python/sedona/core/serde/shape/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
\ No newline at end of file
diff --git a/python/sedona/core/serde/shape/enums.py b/python/sedona/core/serde/shape/enums.py
deleted file mode 100644
index cd85bda..0000000
--- a/python/sedona/core/serde/shape/enums.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from enum import Enum
-
-
-class GeomEnum(Enum):
- undefined = 0
- point = 1
- polyline = 3
- polygon = 5
- multipoint = 8
-
- @classmethod
- def has_value(cls, value):
- return value in cls._value2member_map_
-
- @classmethod
- def get_name(cls, value):
- return cls._value2member_map_[value].name
-
-
-class ShapeEnum(Enum):
-
- shape = 0
diff --git a/python/sedona/core/serde/shape/linestring.py b/python/sedona/core/serde/shape/linestring.py
deleted file mode 100644
index 9025f4a..0000000
--- a/python/sedona/core/serde/shape/linestring.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from typing import Union
-
-import attr
-from shapely.geometry import LineString, MultiLineString
-
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.order import ByteOrderType
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.core.serde.shape.enums import GeomEnum
-from sedona.core.serde.shape.util import add_shape_geometry_metadata, put_coordinates
-from sedona.utils.abstract_parser import GeometryParser
-
-
-@attr.s
-class LineStringParser(GeometryParser):
- name = "LineString"
-
- @classmethod
- def serialize(cls, obj: LineString, binary_buffer: BinaryBuffer):
- if isinstance(obj, LineString):
- add_shape_geometry_metadata(GeomEnum.polyline.value, binary_buffer)
- binary_buffer.add_empty_bytes("double", 4)
- num_points = obj.coords.__len__()
- binary_buffer.put_int(1, ByteOrderType.LITTLE_ENDIAN)
- binary_buffer.put_int(num_points, ByteOrderType.LITTLE_ENDIAN)
- binary_buffer.put_int(0, ByteOrderType.LITTLE_ENDIAN)
-
- put_coordinates(obj.coords, binary_buffer)
-
- binary_buffer.put_byte(-127)
- else:
- raise TypeError(f"Need a {cls.name} instance")
- return binary_buffer.byte_array
-
- @classmethod
- def deserialize(cls, parser: BinaryParser) -> Union[LineString, MultiLineString]:
- raise NotImplemented()
-
diff --git a/python/sedona/core/serde/shape/multilinestring.py b/python/sedona/core/serde/shape/multilinestring.py
deleted file mode 100644
index 518031b..0000000
--- a/python/sedona/core/serde/shape/multilinestring.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from typing import Union
-
-import attr
-from shapely.geometry import MultiLineString, LineString
-
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.core.serde.shape.enums import GeomEnum
-from sedona.core.serde.shape.util import add_shape_geometry_metadata, put_coordinates
-from sedona.utils.abstract_parser import GeometryParser
-from sedona.utils.binary_parser import ByteOrderType
-
-
-@attr.s
-class MultiLineStringParser(GeometryParser):
- name = "MultiLineString"
-
- @classmethod
- def serialize(cls, obj: MultiLineString, binary_buffer: BinaryBuffer):
- if isinstance(obj, MultiLineString):
- add_shape_geometry_metadata(GeomEnum.polyline.value, binary_buffer)
- binary_buffer.add_empty_bytes("double", 4)
-
- num_parts = len(obj.geoms)
- num_points = sum([len(el.coords) for el in obj.geoms])
-
- binary_buffer.put_int(num_parts, ByteOrderType.LITTLE_ENDIAN)
- binary_buffer.put_int(num_points, ByteOrderType.LITTLE_ENDIAN)
-
- offset = 0
- for _ in range(num_parts):
- binary_buffer.put_int(offset, ByteOrderType.LITTLE_ENDIAN)
- offset = offset + obj.geoms[_].coords.__len__()
-
- for geom in obj.geoms:
- put_coordinates(geom.coords, binary_buffer)
-
- binary_buffer.put_byte(-127)
- else:
- raise TypeError(f"Need a {cls.name} instance")
- return binary_buffer.byte_array
-
- @classmethod
- def deserialize(cls, parser: BinaryParser) -> Union[LineString, MultiLineString]:
- raise NotImplemented()
\ No newline at end of file
diff --git a/python/sedona/core/serde/shape/multipoint.py b/python/sedona/core/serde/shape/multipoint.py
deleted file mode 100644
index da77986..0000000
--- a/python/sedona/core/serde/shape/multipoint.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import attr
-from shapely.geometry import MultiPoint
-
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.order import ByteOrderType
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.core.serde.shape.enums import GeomEnum
-from sedona.core.serde.shape.util import add_shape_geometry_metadata, read_coordinates
-from sedona.utils.abstract_parser import GeometryParser
-
-
-@attr.s
-class MultiPointParser(GeometryParser):
- name = "MultiPoint"
-
- @classmethod
- def serialize(cls, obj: MultiPoint, binary_buffer: BinaryBuffer):
- if isinstance(obj, MultiPoint):
- add_shape_geometry_metadata(GeomEnum.multipoint.value, binary_buffer)
- binary_buffer.add_empty_bytes("double", 4)
- binary_buffer.put_int(len(obj.geoms), ByteOrderType.LITTLE_ENDIAN)
- for point in obj.geoms:
- binary_buffer.put_double(point.x, ByteOrderType.LITTLE_ENDIAN)
- binary_buffer.put_double(point.y, ByteOrderType.LITTLE_ENDIAN)
- binary_buffer.put_byte(-127)
- else:
- raise TypeError(f"Need a {cls.name} instance")
- return binary_buffer.byte_array
-
- @classmethod
- def deserialize(cls, parser: BinaryParser) -> MultiPoint:
- for _ in range(4):
- parser.read_double(ByteOrderType.LITTLE_ENDIAN)
- number_of_points = parser.read_int(ByteOrderType.LITTLE_ENDIAN)
-
- coordinates = read_coordinates(parser, number_of_points)
-
- return MultiPoint(coordinates)
\ No newline at end of file
diff --git a/python/sedona/core/serde/shape/multipolygon.py b/python/sedona/core/serde/shape/multipolygon.py
deleted file mode 100644
index 1d177af..0000000
--- a/python/sedona/core/serde/shape/multipolygon.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import attr
-from shapely.geometry import MultiPolygon
-
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.order import ByteOrderType
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.core.serde.shape.enums import GeomEnum
-from sedona.core.serde.shape.util import get_number_of_polygon_points, get_number_of_rings, add_shape_geometry_metadata, \
- add_offsets_to_polygon, reverse_linear_ring, put_coordinates
-from sedona.utils.abstract_parser import GeometryParser
-
-
-@attr.s
-class MultiPolygonParser(GeometryParser):
- name = "MultiPolygon"
-
- @classmethod
- def serialize(cls, obj: MultiPolygon, binary_buffer: BinaryBuffer):
- if isinstance(obj, MultiPolygon):
- num_polygons = len(obj.geoms)
- num_points = sum([get_number_of_polygon_points(polygon) for polygon in obj.geoms])
- num_rings = sum([get_number_of_rings(polygon) for polygon in obj.geoms])
- add_shape_geometry_metadata(GeomEnum.polygon.value, binary_buffer)
- binary_buffer.add_empty_bytes("double", 4)
- binary_buffer.put_int(num_rings, ByteOrderType.LITTLE_ENDIAN)
- binary_buffer.put_int(num_points, ByteOrderType.LITTLE_ENDIAN)
-
- offset = 0
- for geom in obj.geoms:
- offset = add_offsets_to_polygon(geom, binary_buffer, offset)
-
- for geom in obj.geoms:
- coordinates_exterior = reverse_linear_ring(geom.exterior, False)
- put_coordinates(coordinates_exterior, binary_buffer)
-
- for ring in geom.interiors:
- coordinates = reverse_linear_ring(ring)
- put_coordinates(coordinates, binary_buffer)
-
- binary_buffer.put_byte(1)
- binary_buffer.put_byte(3)
- binary_buffer.put_byte(1)
- binary_buffer.put_byte(-127)
- else:
- raise TypeError(f"Need a {cls.name} instance")
- return binary_buffer.byte_array
-
- @classmethod
- def deserialize(cls, parser: BinaryParser) -> MultiPolygon:
- raise NotImplementedError("For multipolygon, PolygonParser class is used.")
diff --git a/python/sedona/core/serde/shape/offset.py b/python/sedona/core/serde/shape/offset.py
deleted file mode 100644
index bd0fdee..0000000
--- a/python/sedona/core/serde/shape/offset.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import attr
-
-from sedona.core.serde.binary.order import ByteOrderType
-
-
-@attr.s
-class OffsetsReader:
-
- @staticmethod
- def read_offsets(parser, num_parts, max_offset):
- offsets = []
- for i in range(num_parts):
- offsets.append(parser.read_int(ByteOrderType.LITTLE_ENDIAN))
- offsets.append(max_offset)
- return offsets
diff --git a/python/sedona/core/serde/shape/point.py b/python/sedona/core/serde/shape/point.py
deleted file mode 100644
index deabb15..0000000
--- a/python/sedona/core/serde/shape/point.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import attr
-from shapely.geometry import Point
-
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.order import ByteOrderType
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.core.serde.shape.enums import GeomEnum
-from sedona.core.serde.shape.util import add_shape_geometry_metadata
-from sedona.utils.abstract_parser import GeometryParser
-
-
-@attr.s
-class PointParser(GeometryParser):
- name = "Point"
-
- @classmethod
- def serialize(cls, obj: Point, binary_buffer: BinaryBuffer):
- if isinstance(obj, Point):
- add_shape_geometry_metadata(GeomEnum.point.value, binary_buffer)
- binary_buffer.put_double(obj.x, ByteOrderType.LITTLE_ENDIAN)
- binary_buffer.put_double(obj.y, ByteOrderType.LITTLE_ENDIAN)
- binary_buffer.put_byte(0)
- else:
- raise TypeError(f"Need a {cls.name} instance")
- return binary_buffer.byte_array
-
- @classmethod
- def deserialize(cls, parser: BinaryParser) -> Point:
- x = parser.read_double(ByteOrderType.LITTLE_ENDIAN)
- y = parser.read_double(ByteOrderType.LITTLE_ENDIAN)
- return Point(x, y)
diff --git a/python/sedona/core/serde/shape/polygon.py b/python/sedona/core/serde/shape/polygon.py
deleted file mode 100644
index 02ae3af..0000000
--- a/python/sedona/core/serde/shape/polygon.py
+++ /dev/null
@@ -1,104 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from typing import Union
-
-import attr
-from shapely.geometry import Polygon, MultiPolygon, LinearRing
-
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.order import ByteOrderType
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.core.serde.shape.enums import GeomEnum
-from sedona.core.serde.shape.offset import OffsetsReader
-from sedona.core.serde.shape.util import add_shape_geometry_metadata, get_number_of_rings, get_number_of_polygon_points, \
- add_offsets_to_polygon, reverse_linear_ring, put_coordinates, read_coordinates
-from sedona.utils.abstract_parser import GeometryParser
-
-
-@attr.s
-class PolygonParser(GeometryParser):
- name = "Polygon"
-
- @classmethod
- def serialize(cls, obj: Polygon, binary_buffer: BinaryBuffer):
- if isinstance(obj, Polygon):
- add_shape_geometry_metadata(GeomEnum.polygon.value, binary_buffer)
-
- num_rings = get_number_of_rings(obj)
-
- num_points = get_number_of_polygon_points(obj)
-
- binary_buffer.add_empty_bytes("double", 4)
-
- binary_buffer.put_int(num_rings, ByteOrderType.LITTLE_ENDIAN)
- binary_buffer.put_int(num_points, ByteOrderType.LITTLE_ENDIAN)
-
- add_offsets_to_polygon(obj, binary_buffer, 0)
-
- coordinates_exterior = reverse_linear_ring(obj.exterior, False)
- put_coordinates(coordinates_exterior, binary_buffer)
-
- for ring in obj.interiors:
- coordinates = reverse_linear_ring(ring)
- put_coordinates(coordinates, binary_buffer)
-
- binary_buffer.put_byte(-127)
-
- else:
- raise TypeError(f"Need a {cls.name} instance")
- return binary_buffer.byte_array
-
- @classmethod
- def deserialize(cls, parser: BinaryParser) -> Union[Polygon, MultiPolygon]:
- for _ in range(4):
- parser.read_double(ByteOrderType.LITTLE_ENDIAN)
- num_rings = parser.read_int(ByteOrderType.LITTLE_ENDIAN)
- num_points = parser.read_int(ByteOrderType.LITTLE_ENDIAN)
- offsets = OffsetsReader.read_offsets(parser, num_parts=num_rings, max_offset=num_points)
- polygons = []
- holes = []
- shells_ccw = False
- shell = None
- for i in range(num_rings):
- read_scale = offsets[i + 1] - offsets[i]
- cs_ring = read_coordinates(parser, read_scale)
- if (len(cs_ring)) < 3:
- continue
-
- ring = LinearRing(cs_ring)
-
- if shell is None:
- shell = ring
- shells_ccw = LinearRing(cs_ring).is_ccw
- elif LinearRing(cs_ring).is_ccw != shells_ccw:
- holes.append(ring)
- else:
- if shell is not None:
- polygon = Polygon(shell, holes)
- polygons.append(polygon)
- shell = ring
- holes = []
-
- if shell is not None:
- geometry = Polygon(shell, holes)
- polygons.append(geometry)
-
- if polygons.__len__() == 1:
- return polygons[0]
-
- return MultiPolygon(polygons)
diff --git a/python/sedona/core/serde/shape/polyline.py b/python/sedona/core/serde/shape/polyline.py
deleted file mode 100644
index 43ef247..0000000
--- a/python/sedona/core/serde/shape/polyline.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from typing import Union
-
-import attr
-from shapely.geometry import MultiLineString, LineString
-
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.order import ByteOrderType
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.core.serde.shape.offset import OffsetsReader
-from sedona.core.serde.shape.util import read_coordinates
-from sedona.sql.exceptions import InvalidGeometryException
-from sedona.utils.abstract_parser import GeometryParser
-
-
-@attr.s
-class PolyLineParser(GeometryParser):
- name = "Polyline"
-
- @classmethod
- def serialize(cls, obj: Union[MultiLineString, LineString], binary_buffer: BinaryBuffer):
- raise NotImplementedError("")
-
- @classmethod
- def deserialize(cls, parser: BinaryParser) -> Union[LineString, MultiLineString]:
- for _ in range(4):
- parser.read_double(ByteOrderType.LITTLE_ENDIAN)
-
- num_parts = parser.read_int(ByteOrderType.LITTLE_ENDIAN)
- num_points = parser.read_int(ByteOrderType.LITTLE_ENDIAN)
-
- offsets = OffsetsReader.read_offsets(parser, num_parts, num_points)
- lines = []
- for i in range(num_parts):
- read_scale = offsets[i + 1] - offsets[i]
- coordinate_sequence = read_coordinates(parser, read_scale)
- lines.append(LineString(coordinate_sequence))
-
- if num_parts == 1:
- line = lines[0]
- elif num_parts > 1:
- line = MultiLineString(lines)
- else:
- raise InvalidGeometryException("Invalid geometry")
-
- return line
diff --git a/python/sedona/core/serde/shape/shape.py b/python/sedona/core/serde/shape/shape.py
deleted file mode 100644
index 37b3a5b..0000000
--- a/python/sedona/core/serde/shape/shape.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from typing import List
-
-from shapely.geometry.base import BaseGeometry
-
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.core.serde.serializer import GeometrySerde
-from sedona.core.serde.shape.enums import GeomEnum
-from sedona.core.serde.shape.linestring import LineStringParser
-from sedona.core.serde.shape.multilinestring import MultiLineStringParser
-from sedona.core.serde.shape.multipoint import MultiPointParser
-from sedona.core.serde.shape.multipolygon import MultiPolygonParser
-from sedona.core.serde.shape.point import PointParser
-from sedona.core.serde.shape.polygon import PolygonParser
-from sedona.core.serde.shape.polyline import PolyLineParser
-from sedona.core.serde.shape.undefined import UndefinedParser
-from sedona.sql.exceptions import GeometryUnavailableException
-from sedona.utils.abstract_parser import GeometryParser
-from sedona.utils.decorators import classproperty
-
-
-class ShapeSerde(GeometrySerde):
-
- byte_number = 0
-
- def deserialize(self, bin_parser: BinaryParser) -> BaseGeometry:
- gm_type = bin_parser.read_byte()
- if GeomEnum.has_value(gm_type):
- name = GeomEnum.get_name(gm_type)
- parser: GeometryParser = self.parsers[name]
- geom = parser.deserialize(bin_parser)
- return geom
- else:
- raise GeometryUnavailableException(f"Can not deserialize object")
-
- def serialize(self, geom: BaseGeometry, buffer: BinaryBuffer) -> List[int]:
- buffer.put_byte(self.byte_number)
- geom_name = str(geom.__class__.__name__).lower()
- try:
- appr_parser = self.parsers[geom_name]
- except KeyError:
- raise KeyError(f"Parser for geometry {geom_name}")
- return appr_parser.serialize(geom, buffer)
-
- @classproperty
- def parsers(self):
- geom_cls = dict(
- undefined=UndefinedParser,
- point=PointParser,
- polyline=PolyLineParser,
- multilinestring=MultiLineStringParser,
- linestring=LineStringParser,
- polygon=PolygonParser,
- multipoint=MultiPointParser,
- multipolygon=MultiPolygonParser
- )
- return geom_cls
diff --git a/python/sedona/core/serde/shape/undefined.py b/python/sedona/core/serde/shape/undefined.py
deleted file mode 100644
index 1ef70d7..0000000
--- a/python/sedona/core/serde/shape/undefined.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import attr
-from shapely.geometry.base import BaseGeometry
-
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.utils.abstract_parser import GeometryParser
-
-
-@attr.s
-class UndefinedParser(GeometryParser):
- name = "Undefined"
-
- @classmethod
- def serialize(cls, obj: BaseGeometry, binary_buffer: BinaryBuffer):
- raise NotImplementedError()
-
- @classmethod
- def deserialize(cls, parser: BinaryParser) -> BaseGeometry:
- raise NotImplementedError()
diff --git a/python/sedona/core/serde/shape/util.py b/python/sedona/core/serde/shape/util.py
deleted file mode 100644
index 8b9a2cf..0000000
--- a/python/sedona/core/serde/shape/util.py
+++ /dev/null
@@ -1,71 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from typing import Iterable, Tuple, List
-
-from shapely.geometry import Point, LinearRing
-from shapely.geometry import Polygon
-
-from sedona.utils.binary_parser import ByteOrderType
-from sedona.utils.types import numeric
-
-
-def read_coordinates(parser: 'BinaryParser', read_scale: int):
- coordinates = []
- for i in range(read_scale):
- coordinates.append(
- (parser.read_double(ByteOrderType.LITTLE_ENDIAN), parser.read_double(ByteOrderType.LITTLE_ENDIAN)))
- return coordinates
-
-
-def put_coordinates(coordinates: Iterable[Iterable[numeric]], binary_buffer: 'BinaryBuffer'):
- for coordinate in coordinates:
- binary_buffer.put_double(Point(coordinate).x, ByteOrderType.LITTLE_ENDIAN)
- binary_buffer.put_double(Point(coordinate).y, ByteOrderType.LITTLE_ENDIAN)
-
-
-def add_shape_geometry_metadata(geom_type: int, binary_buffer: 'BinaryBuffer'):
- binary_buffer.put_byte(geom_type)
-
-
-def reverse_linear_ring(linear_ring: LinearRing, ccw: bool = True) -> List[Tuple[numeric, numeric]]:
- if linear_ring.is_ccw == ccw:
- return linear_ring.coords
- else:
- return list(reversed(linear_ring.coords))
-
-
-def get_number_of_polygon_points(geom: Polygon) -> int:
- interior_point_num = sum([el.coords.__len__() for el in geom.interiors])
- exterior_num_points = geom.exterior.coords.__len__()
- return interior_point_num + exterior_num_points
-
-
-def get_number_of_rings(geom: Polygon) -> int:
- return geom.interiors.__len__() + 1
-
-
-def add_offsets_to_polygon(geom: Polygon, binary_buffer: 'BinaryBuffer', initial_offset: int) -> int:
- offset = initial_offset
- num_rings = get_number_of_rings(geom)
- binary_buffer.put_int(offset, ByteOrderType.LITTLE_ENDIAN)
- offset += geom.exterior.coords.__len__()
- for _ in range(num_rings - 1):
- binary_buffer.put_int(offset, ByteOrderType.LITTLE_ENDIAN)
- offset = offset + geom.interiors[_].coords.__len__()
-
- return offset
diff --git a/python/sedona/core/serde/spark_config.py b/python/sedona/core/serde/spark_config.py
deleted file mode 100644
index 15bb703..0000000
--- a/python/sedona/core/serde/spark_config.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-from sedona.core.serde.geom_factory import serializers
-
-
-class Singleton(type):
- _instances = {}
-
- def __call__(cls, *args, **kwargs):
- if cls not in cls._instances:
- cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
- return cls._instances[cls]
-
-
-class SparkConfGetter(metaclass=Singleton):
-
- @property
- def serialization(self):
- if not hasattr(self, "__serialization"):
- raise AttributeError("can not find current serialization please create spark session and register sedona"
- " functions by using SedonaRegistrator.registerAll(spark)")
- return getattr(self, "__serialization")
-
- @serialization.setter
- def serialization(self, value):
- setattr(self, "__serialization", value)
-
- @property
- def serialization_number(self):
- return serializers[self.serialization]
-
-
-spark_conf_getter = SparkConfGetter()
\ No newline at end of file
diff --git a/python/sedona/core/serde/wkb/__init__.py b/python/sedona/core/serde/wkb/__init__.py
deleted file mode 100644
index 38cc50c..0000000
--- a/python/sedona/core/serde/wkb/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
\ No newline at end of file
diff --git a/python/sedona/core/serde/wkb/wkb.py b/python/sedona/core/serde/wkb/wkb.py
deleted file mode 100644
index 5b07b18..0000000
--- a/python/sedona/core/serde/wkb/wkb.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from typing import List
-
-from shapely.geometry.base import BaseGeometry
-from shapely.wkb import dumps
-
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.order import ByteOrderType
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.core.serde.serializer import GeometrySerde
-
-
-class WkbSerde(GeometrySerde):
-
- byte_number = 1
-
- def deserialize(self, bin_parser: BinaryParser) -> BaseGeometry:
- geom_length = bin_parser.read_int(ByteOrderType.BIG_ENDIAN)
- geom = bin_parser.read_geometry(geom_length)
- return geom
-
- def serialize(self, geom: BaseGeometry, buffer: BinaryBuffer) -> List[int]:
- geom_bytes = dumps(geom, srid=4326)
- buffer.put_byte(self.byte_number)
- buffer.put_int(len(geom_bytes), ByteOrderType.BIG_ENDIAN)
- return [*buffer.byte_array, *geom_bytes, *[0]]
diff --git a/python/sedona/core/spatialOperator/knn_query.py b/python/sedona/core/spatialOperator/knn_query.py
index 2140e38..1d4c122 100644
--- a/python/sedona/core/spatialOperator/knn_query.py
+++ b/python/sedona/core/spatialOperator/knn_query.py
@@ -20,7 +20,7 @@ from shapely.geometry.base import BaseGeometry
from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
from sedona.core.jvm.translate import JvmSedonaPythonConverter
-from sedona.core.serde.binary.parser import BinaryParser
+from sedona.utils.binary_parser import BinaryParser
from sedona.utils.decorators import require
from sedona.utils.geometry_adapter import GeometryAdapter
from sedona.utils.spatial_rdd_parser import SpatialRDDParserData
diff --git a/python/sedona/core/spatialOperator/rdd.py b/python/sedona/core/spatialOperator/rdd.py
index b40cd61..61969a1 100644
--- a/python/sedona/core/spatialOperator/rdd.py
+++ b/python/sedona/core/spatialOperator/rdd.py
@@ -31,6 +31,7 @@ class SedonaRDD:
jvm = self.sc._jvm
serialized = JvmSedonaPythonConverter(jvm). \
translate_spatial_rdd_to_python(self.jsrdd)
+
return RDD(serialized, self.sc, SedonaPickler())
diff --git a/python/sedona/register/geo_registrator.py b/python/sedona/register/geo_registrator.py
index 30175a4..5a50a79 100644
--- a/python/sedona/register/geo_registrator.py
+++ b/python/sedona/register/geo_registrator.py
@@ -18,9 +18,7 @@
import attr
from pyspark.sql import SparkSession
from py4j.java_gateway import java_import
-from pyspark import SparkContext
-from sedona.core.serde.spark_config import spark_conf_getter
from sedona.register.java_libs import SedonaJvmLib
from sedona.utils.prep import assign_all
@@ -43,15 +41,6 @@ class SedonaRegistrator:
spark.sql("SELECT 1 as geom").count()
PackageImporter.import_jvm_lib(spark._jvm)
cls.register(spark)
- current_sc = SparkContext._active_spark_context
-
- if current_sc:
- conf = current_sc.getConf()
- serializer_type = conf.get("sedona.serializer.type")
- spark_conf_getter.serialization = serializer_type
- else:
- spark_conf_getter.serialization = "shape"
-
return True
@classmethod
diff --git a/python/sedona/sql/types.py b/python/sedona/sql/types.py
index 952fce5..f0c52ec 100644
--- a/python/sedona/sql/types.py
+++ b/python/sedona/sql/types.py
@@ -15,19 +15,14 @@
# specific language governing permissions and limitations
# under the License.
-from pyspark.sql.types import UserDefinedType, ArrayType, ByteType
+import struct
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.core.serde.geom_factory import geometry_serializers_instances, geometry_serializers
-from sedona.core.serde.spark_config import spark_conf_getter
+from pyspark.sql.types import UserDefinedType, ArrayType, ByteType
+from shapely.wkb import dumps, loads
class GeometryType(UserDefinedType):
- def __init__(self):
- self.serde = spark_conf_getter.serialization
-
@classmethod
def sqlType(cls):
return ArrayType(ByteType(), containsNull=False)
@@ -39,15 +34,12 @@ class GeometryType(UserDefinedType):
return [el - 256 if el >= 128 else el for el in self.serialize(obj)]
def serialize(self, obj):
- binary_buffer = BinaryBuffer()
- binary_buffer.put_byte(0)
- return geometry_serializers_instances[self.serde].serialize(obj, binary_buffer)
+ return dumps(obj)
def deserialize(self, datum):
- parser = BinaryParser(list(datum))
- _ = parser.read_byte()
- serde = parser.read_byte()
- return geometry_serializers[serde].deserialize(parser)
+ bytes_data = b''.join([struct.pack('b', el) for el in datum])
+ geom = loads(bytes_data)
+ return geom
@classmethod
def module(cls):
diff --git a/python/sedona/utils/adapter.py b/python/sedona/utils/adapter.py
index ab1fb60..8168efb 100644
--- a/python/sedona/utils/adapter.py
+++ b/python/sedona/utils/adapter.py
@@ -124,7 +124,7 @@ class Adapter(metaclass=MultipleMeta):
spatial_pair_rdd_mapped = spatialPairRDD.map(
lambda x: [x[0].geom, *x[0].getUserData().split("\t"), x[1].geom, *x[1].getUserData().split("\t")]
)
- df = sparkSession.createDataFrame(spatial_pair_rdd_mapped, verifySchema=False)
+ df = sparkSession.createDataFrame(spatial_pair_rdd_mapped)
return df
@classmethod
@@ -139,8 +139,6 @@ class Adapter(metaclass=MultipleMeta):
"""
df = Adapter.toDf(spatialPairRDD, sparkSession)
- df.show()
-
columns_length = df.columns.__len__()
combined_columns = ["geom_1", *leftFieldnames, "geom_2", *rightFieldNames]
if columns_length == combined_columns.__len__():
diff --git a/python/sedona/utils/binary_parser.py b/python/sedona/utils/binary_parser.py
index 005133e..d680421 100644
--- a/python/sedona/utils/binary_parser.py
+++ b/python/sedona/utils/binary_parser.py
@@ -15,7 +15,12 @@
# specific language governing permissions and limitations
# under the License.
-from enum import Enum
+import struct
+from typing import List, Union
+
+import attr
+from pyspark import SparkContext
+from shapely.wkb import loads
DOUBLE_SIZE = 8
INT_SIZE = 4
@@ -32,12 +37,142 @@ size_dict = {
}
-class ByteOrderType(Enum):
- LITTLE_ENDIAN = "<"
- BIG_ENDIAN = ">"
+@attr.s
+class BinaryParser:
+ bytes = attr.ib(type=Union[bytearray, List[int]])
+ current_index = attr.ib(default=0)
+
+ def __attrs_post_init__(self):
+ no_negatives = self.remove_negatives(self.bytes)
+ self.bytes = self._convert_to_binary_array(no_negatives)
+
+ def read_geometry(self, length: int):
+ geom_bytes = b"".join([struct.pack("b", el) if el < 128 else struct.pack("b", el - 256) for el in
+ self.bytes[self.current_index: self.current_index + length]])
+ geom = loads(geom_bytes)
+ self.current_index += length
+ return geom
+
+ def read_double(self):
+ data = self.unpack("d", self.bytes)
+ self.current_index = self.current_index + DOUBLE_SIZE
+ return data
+
+ def read_double_reverse(self):
+ data = self.unpack_reverse("d", self.bytes)
+ self.current_index = self.current_index + DOUBLE_SIZE
+ return data
+
+ def read_int(self):
+ data = self.unpack("i", self.bytes)
+ self.current_index = self.current_index + INT_SIZE
+ return data
+
+ def read_byte(self):
+ data = self.unpack("b", self.bytes)
+ self.current_index = self.current_index + BYTE_SIZE
+ return data
+
+ def read_char(self):
+ data = self.unpack("c", self.bytes)
+ self.current_index = self.current_index + CHAR_SIZE
+ return data
+
+ def read_boolean(self):
+ data = self.unpack("?", self.bytes)
+ self.current_index = self.current_index + BOOLEAN_SIZE
+ return data
+
+ def read_string(self, length: int, encoding: str = "utf8"):
+ string = self.bytes[self.current_index: self.current_index + length]
+ self.current_index += length
+
+ try:
+ encoded_string = string.decode(encoding, "ignore")
+ except UnicodeEncodeError:
+ raise UnicodeEncodeError
+ return encoded_string
+
+ def read_kryo_string(self, length: int, sc: SparkContext) -> str:
+ array_length = length - self.current_index
+ byte_array = sc._gateway.new_array(sc._jvm.Byte, array_length)
+
+ for index, bt in enumerate(self.bytes[self.current_index: length]):
+ byte_array[index] = self.bytes[self.current_index + index]
+ decoded_string = sc._jvm.org.imbruced.geo_pyspark.serializers.GeoSerializerData.deserializeUserData(
+ byte_array
+ )
+ self.current_index = length
+ return decoded_string
+
+ def unpack(self, tp: str, bytes: bytearray):
+ max_index = self.current_index + size_dict[tp]
+ bytes = self._convert_to_binary_array(bytes[self.current_index: max_index])
+ return struct.unpack(tp, bytes)[0]
+
+ def unpack_reverse(self, tp: str, bytes: bytearray):
+ max_index = self.current_index + size_dict[tp]
+ bytes = bytearray(reversed(self._convert_to_binary_array(bytes[self.current_index: max_index])))
+ return struct.unpack(tp, bytes)[0]
+
+ @classmethod
+ def remove_negatives(cls, bytes):
+ return [cls.remove_negative(bt) for bt in bytes]
+
+ @classmethod
+ def remove_negative(cls, byte):
+ bt_pos = byte if byte >= 0 else byte + 256
+ return bt_pos
+
+ @staticmethod
+ def _convert_to_binary_array(bytes):
+ if type(bytes) == list:
+ bytes = bytearray(bytes)
+ return bytes
+
+
+class BinaryBuffer:
+
+ def __init__(self):
+ self.array = []
+
+ def put_double(self, value):
+ bytes = self.__pack("d", value)
+ self.__extend_buffer(bytes)
+
+ def put_int(self, value):
+ bytes = self.__pack("i", value)
+ self.__extend_buffer(bytes)
+
+ def put_byte(self, value):
+ bytes = self.__pack("b", value)
+ self.__extend_buffer(bytes)
+ def put(self, value):
+ self.__extend_buffer(value)
+ def __pack(self, type, value):
+ return struct.pack(type, value)
+ def __extend_buffer(self, bytes):
+ self.array.extend(list(bytes))
+ def __translate_values(self, values):
+ return [el if el < 128 else el - 256 for el in values]
+ def add_empty_bytes(self, tp: str, number_of_empty):
+ if tp == "double":
+ for _ in range(number_of_empty):
+ self.put_double(0.0)
+ elif tp == "int":
+ for _ in range(number_of_empty):
+ self.put_int(0)
+ elif tp == "double":
+ for _ in range(number_of_empty):
+ self.put_byte(0)
+ else:
+ raise TypeError(f"Passed {tp} is not available")
+ @property
+ def byte_array(self):
+ return self.__translate_values(self.array)
diff --git a/python/sedona/utils/geometry_adapter.py b/python/sedona/utils/geometry_adapter.py
index ac58c49..9a81307 100644
--- a/python/sedona/utils/geometry_adapter.py
+++ b/python/sedona/utils/geometry_adapter.py
@@ -19,9 +19,6 @@ from shapely.geometry.base import BaseGeometry
from sedona.core.geom.envelope import Envelope
from sedona.core.jvm.translate import JvmGeometryAdapter
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.geom_factory import geometry_serializers_instances
-from sedona.core.serde.spark_config import spark_conf_getter
from sedona.utils.spatial_rdd_parser import GeometryFactory
@@ -37,10 +34,7 @@ class GeometryAdapter:
if isinstance(geom, Envelope):
jvm_geom = geom.create_jvm_instance(jvm)
else:
- byte_buffer = BinaryBuffer()
- decoded_geom = GeometryFactory.to_bytes(geom, geometry_serializers_instances[spark_conf_getter.serialization],
- byte_buffer)
-
+ decoded_geom = GeometryFactory.to_bytes(geom)
jvm_geom = JvmGeometryAdapter(jvm).translate_to_java(decoded_geom)
return jvm_geom
diff --git a/python/sedona/utils/spatial_rdd_parser.py b/python/sedona/utils/spatial_rdd_parser.py
index 3c3f8aa..f897ecd 100644
--- a/python/sedona/utils/spatial_rdd_parser.py
+++ b/python/sedona/utils/spatial_rdd_parser.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import struct
from abc import ABC
from copy import copy
from typing import List, Any
@@ -22,19 +23,15 @@ from typing import List, Any
import attr
from shapely.geometry.base import BaseGeometry
from pyspark import PickleSerializer
+from shapely.wkb import dumps
from sedona.core.geom.circle import Circle
-from sedona.core.serde.binary.buffer import BinaryBuffer
-from sedona.core.serde.binary.parser import BinaryParser
-from sedona.core.serde.geom_factory import geometry_serializers
-from sedona.core.serde.serializer import GeometrySerde
-from sedona.core.serde.spark_config import spark_conf_getter
-from sedona.utils.binary_parser import ByteOrderType
+from sedona.utils.binary_parser import BinaryParser
class GeoData:
- def __init__(self, geom: BaseGeometry, userData: str, serde):
+ def __init__(self, geom: BaseGeometry, userData: str):
"""
:param geom:
@@ -42,7 +39,6 @@ class GeoData:
"""
self._geom = geom
self._userData = userData
- self._serde = serde
def getUserData(self):
return self.userData
@@ -50,36 +46,30 @@ class GeoData:
def __getstate__(self):
attributes = copy(self.__slots__)
geom = getattr(self, attributes[0])
- serde = getattr(self, attributes[2])
- binary_buffer = BinaryBuffer()
if isinstance(geom, Circle):
- geom_bytes = CircleGeometryFactory.to_bytes(geom,
- geometry_serializers[serde],
- binary_buffer)
+ geom_bytes = CircleGeometryFactory.to_bytes(geom)
else:
- geom_bytes = GeometryFactory.to_bytes(geom, geometry_serializers[serde],
- binary_buffer)
+ geom_bytes = GeometryFactory.to_bytes(geom)
return dict(
geom=bytearray([el if el >= 0 else el + 256 for el in geom_bytes]),
- userData=getattr(self, attributes[1]),
- serde=bytearray([serde])
+ userData=getattr(self, attributes[1])
)
def __setstate__(self, attributes):
bin_parser = BinaryParser(attributes["geom"])
is_circle = bin_parser.read_byte()
+ geom_bytes = attributes["geom"]
+
if is_circle:
- radius = bin_parser.read_double(ByteOrderType.LITTLE_ENDIAN)
- serde = bin_parser.read_byte()
- geom = geometry_serializers[serde].deserialize(bin_parser)
+ radius = bin_parser.read_double()
+ geom = bin_parser.read_geometry(geom_bytes.__len__() - 9)
self._geom = Circle(geom, radius)
else:
- self._geom = geometry_serializers[bin_parser.read_byte()].deserialize(bin_parser)
+ self._geom = bin_parser.read_geometry(geom_bytes.__len__() - 1)
self._userData = attributes["userData"]
- self._serde = attributes["serde"]
@property
def geom(self):
@@ -89,11 +79,7 @@ class GeoData:
def userData(self):
return self._userData
- @property
- def serde(self):
- return self._serde
-
- __slots__ = ("_geom", "_userData", "_serde")
+ __slots__ = ("_geom", "_userData")
def __repr__(self):
return f"Geometry: {str(self.geom.__class__.__name__)} userData: {self.userData}"
@@ -104,10 +90,6 @@ class GeoData:
def __ne__(self, other):
return self.geom != other.geom or self.userData != other.userData
- @classmethod
- def with_current_serialization(cls, geom: BaseGeometry, userData: str):
- return GeoData(geom, userData, spark_conf_getter.serialization_number)
-
@attr.s
class AbstractSpatialRDDParser(ABC):
@@ -134,7 +116,7 @@ class SpatialPairRDDParserData(AbstractSpatialRDDParser):
def deserialize(cls, bin_parser: 'BinaryParser'):
left_geom_data = cls._deserialize_geom(bin_parser)
- _ = bin_parser.read_int(ByteOrderType.LITTLE_ENDIAN)
+ _ = bin_parser.read_int()
right_geom_data = cls._deserialize_geom(bin_parser)
@@ -154,7 +136,7 @@ class SpatialRDDParserData(AbstractSpatialRDDParser):
@classmethod
def deserialize(cls, bin_parser: 'BinaryParser'):
left_geom_data = cls._deserialize_geom(bin_parser)
- _ = bin_parser.read_int(ByteOrderType.LITTLE_ENDIAN)
+ _ = bin_parser.read_int()
return left_geom_data
@@ -171,7 +153,8 @@ class SpatialRDDParserDataMultipleRightGeom(AbstractSpatialRDDParser):
def deserialize(cls, bin_parser: 'BinaryParser'):
left_geom_data = cls._deserialize_geom(bin_parser)
- geometry_numbers = bin_parser.read_int(ByteOrderType.LITTLE_ENDIAN)
+ geometry_numbers = bin_parser.read_int()
+
right_geoms = []
for right_geometry_number in range(geometry_numbers):
@@ -201,7 +184,7 @@ class SedonaPickler(PickleSerializer):
def loads(self, obj, encoding="bytes"):
binary_parser = BinaryParser(obj)
- spatial_parser_number = binary_parser.read_int(ByteOrderType.LITTLE_ENDIAN)
+ spatial_parser_number = binary_parser.read_int()
spatial_parser = self.get_parser(spatial_parser_number)
parsed_row = spatial_parser.deserialize(binary_parser)
@@ -214,16 +197,13 @@ class SedonaPickler(PickleSerializer):
return PARSERS[number]
-class GeometryReader:
+def read_geometry_from_bytes(bin_parser: BinaryParser):
+ geom_data_length = bin_parser.read_int()
+ user_data_length = bin_parser.read_int()
+ geom = bin_parser.read_geometry(geom_data_length)
+ user_data = bin_parser.read_string(user_data_length)
- def read_geometry_from_bytes(self, bin_parser: BinaryParser):
- user_data_length = bin_parser.read_int(ByteOrderType.LITTLE_ENDIAN)
- serde = bin_parser.read_byte()
- parser = geometry_serializers[serde]
- geom = parser.deserialize(bin_parser)
- user_data = bin_parser.read_string(user_data_length)
-
- return (geom, user_data, serde)
+ return (geom, user_data)
@attr.s
@@ -231,15 +211,13 @@ class GeometryFactory:
@classmethod
def geometry_from_bytes(cls, bin_parser: BinaryParser) -> GeoData:
- geom, user_data, serde = GeometryReader().read_geometry_from_bytes(bin_parser)
- geo_data = GeoData(geom=geom, userData=user_data, serde=serde)
+ geom, user_data = read_geometry_from_bytes(bin_parser)
+ geo_data = GeoData(geom=geom, userData=user_data)
return geo_data
@classmethod
- def to_bytes(cls, geom: BaseGeometry, geom_serde: GeometrySerde, byte_buffer: BinaryBuffer) -> List[int]:
- additional_bytes = BinaryBuffer()
- additional_bytes.put_byte(0)
- return additional_bytes.byte_array + geom_serde.serialize(geom, byte_buffer)
+ def to_bytes(cls, geom: BaseGeometry) -> List[int]:
+ return struct.pack("b", 0) + dumps(geom)
@attr.s
@@ -247,20 +225,17 @@ class CircleGeometryFactory:
@classmethod
def geometry_from_bytes(cls, bin_parser: BinaryParser) -> GeoData:
- geom, user_data, serde = GeometryReader().read_geometry_from_bytes(bin_parser)
- radius = bin_parser.read_double(ByteOrderType.LITTLE_ENDIAN)
- geo_data = GeoData(geom=Circle(geom, radius), userData=user_data, serde=serde)
+ geom, user_data = read_geometry_from_bytes(bin_parser)
+ radius = bin_parser.read_double()
+ geo_data = GeoData(geom=Circle(geom, radius), userData=user_data)
return geo_data
@classmethod
- def to_bytes(cls, geom: Circle, geom_serde: GeometrySerde, byte_buffer: BinaryBuffer) -> List[int]:
- additional_bytes = BinaryBuffer()
- additional_bytes.put_byte(1)
- additional_bytes.put_double(geom.radius)
- return additional_bytes.byte_array + geom_serde.serialize(geom.centerGeometry, byte_buffer)
+ def to_bytes(cls, geom: Circle) -> List[int]:
+ return struct.pack("b", 1) + struct.pack("d", geom.radius) + dumps(geom.centerGeometry)
geom_deserializers = {
1: CircleGeometryFactory,
0: GeometryFactory
-}
\ No newline at end of file
+}
diff --git a/python/tests/conftest.py b/python/tests/conftest.py
deleted file mode 100644
index ab19a9c..0000000
--- a/python/tests/conftest.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from sedona.core.serde.spark_config import spark_conf_getter
-from tests.tools import serializer_path
-
-
-def pytest_configure(config):
- with open(serializer_path) as file:
- lines = file.readlines()
- serializer = "".join(lines).strip()
-
- spark_conf_getter.serialization = serializer
diff --git a/python/tests/core/test_avoiding_python_jvm_serde_df.py b/python/tests/core/test_avoiding_python_jvm_serde_df.py
index f2f4cd6..d86a965 100644
--- a/python/tests/core/test_avoiding_python_jvm_serde_df.py
+++ b/python/tests/core/test_avoiding_python_jvm_serde_df.py
@@ -164,8 +164,6 @@ class TestOmitPythonJvmSerdeToDf(TestBase):
df_without_column_names = Adapter.toDf(result, self.spark)
- df_without_column_names.show()
-
raw_geometries = self.__row_to_list(
df_without_column_names.collect()
)
diff --git a/python/tests/core/test_geo_data.py b/python/tests/core/test_geo_data.py
deleted file mode 100644
index 8f7c261..0000000
--- a/python/tests/core/test_geo_data.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from shapely.geometry import Point
-
-from sedona.utils.spatial_rdd_parser import GeoData
-from tests.test_base import TestBase
-
-
-class TestGeoData(TestBase):
-
- def test_geo_data_with_current_serialization(self):
- # given geo data from used serialization type
- geo_data = GeoData.with_current_serialization(Point(21, 52), "user")
-
- # when creating rdd
- rdd = self.sc.parallelize([geo_data])
-
- # then rdd should be created
- assert rdd.count() == 1
-
- # and it should be possible to collect the result
- assert rdd.collect()[0] == geo_data
\ No newline at end of file
diff --git a/python/tests/serialization/test_deserializers.py b/python/tests/serialization/test_deserializers.py
index 5b1acd7..b68af39 100644
--- a/python/tests/serialization/test_deserializers.py
+++ b/python/tests/serialization/test_deserializers.py
@@ -19,8 +19,6 @@ import os
from shapely.geometry import MultiPoint, Point, MultiLineString, LineString, Polygon, MultiPolygon
import geopandas as gpd
-from pyspark.sql.types import StructType, StructField, StringType
-from sedona.sql.types import GeometryType
from tests import tests_resource
from tests.test_base import TestBase
@@ -104,17 +102,9 @@ class TestGeometryConvert(TestBase):
def test_from_geopandas_convert(self):
gdf = gpd.read_file(os.path.join(tests_resource, "shapefiles/gis_osm_pois_free_1/"))
- schema = StructType(
- [
- StructField("osm_id", StringType()),
- StructField("code", StringType()),
- StructField("fclass", StringType()),
- StructField("name", StringType()),
- StructField("geometry", GeometryType()),
- ]
- )
+
self.spark.createDataFrame(
- gdf, schema=schema
+ gdf
).show()
def test_to_geopandas(self):
diff --git a/python/tests/serialization/test_direct_serialization.py b/python/tests/serialization/test_direct_serialization.py
index 2042a76..d657535 100644
--- a/python/tests/serialization/test_direct_serialization.py
+++ b/python/tests/serialization/test_direct_serialization.py
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import pytest
+
from shapely.geometry import Polygon
from shapely.wkt import loads
@@ -24,7 +24,6 @@ from tests.test_base import TestBase
class TestDirectSerialization(TestBase):
- @pytest.mark.skipif(TestBase.serializer_type == "shape", reason="shp parser does not handle geometry collection")
def test_polygon(self):
polygon = Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])
jvm_geom = GeometryAdapter.create_jvm_geometry_from_base_geometry(self.sc._jvm, polygon)
@@ -45,27 +44,6 @@ class TestDirectSerialization(TestBase):
assert jvm_geom.toString() == wkt
- @pytest.mark.skipif(TestBase.serializer_type == "wkb", reason="shp parser does not handle geometry collection")
- def test_polygon(self):
- polygon = Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])
- jvm_geom = GeometryAdapter.create_jvm_geometry_from_base_geometry(self.sc._jvm, polygon)
-
- assert jvm_geom.toString() == "POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))"
-
- ext = [(0, 0), (0, 2), (2, 2), (2, 0), (0, 0)]
- int = [(1, 1), (1, 1.5), (1.5, 1.5), (1.5, 1), (1, 1)]
-
- polygon = Polygon(ext, [int])
- jvm_geom = GeometryAdapter.create_jvm_geometry_from_base_geometry(self.sc._jvm, polygon)
-
- assert jvm_geom.toString() == "POLYGON ((0 0, 0 2, 2 2, 2 0, 0 0), (1 1, 1.5 1, 1.5 1.5, 1 1.5, 1 1))"
-
- wkt = "POLYGON ((-71.1776585052917 42.3902909739571, -71.1776820268866 42.3903701743239, -71.1776063012595 42.3903825660754, -71.1775826583081 42.3903033653531, -71.1776585052917 42.3902909739571))"
- polygon = loads(wkt)
- jvm_geom = GeometryAdapter.create_jvm_geometry_from_base_geometry(self.sc._jvm, polygon)
-
- assert jvm_geom.toString() == wkt
-
def test_point(self):
wkt = "POINT (-71.064544 42.28787)"
point = loads(wkt)
diff --git a/python/tests/serialization/test_rdd_serialization.py b/python/tests/serialization/test_rdd_serialization.py
index 5c2d979..137dcc0 100644
--- a/python/tests/serialization/test_rdd_serialization.py
+++ b/python/tests/serialization/test_rdd_serialization.py
@@ -17,8 +17,6 @@
import os
-import pytest
-
from sedona.core.SpatialRDD import PointRDD, PolygonRDD, CircleRDD, LineStringRDD
from sedona.core.enums import FileDataSplitter, IndexType
from tests.test_base import TestBase
@@ -79,18 +77,17 @@ class TestRDDSerialization(TestBase):
assert [geo_data.geom.wkt for geo_data in collected_polygon_rdd][:3] == input_wkt_polygons
- @pytest.mark.skipif(TestBase.serializer_type == "shape", reason="circle is not supported")
- def test_circle_rdd(self):
- object_rdd = PointRDD(
- sparkContext=self.sc,
- InputLocation=point_rdd_input_location,
- Offset=point_rdd_offset,
- splitter=point_rdd_splitter,
- carryInputData=False
- )
- circle_rdd = CircleRDD(object_rdd, 0.1)
- collected_data = circle_rdd.getRawSpatialRDD().collect()
- print([geo_data.geom.wkt for geo_data in collected_data])
+ # def test_circle_rdd(self):
+ # object_rdd = PointRDD(
+ # sparkContext=self.sc,
+ # InputLocation=point_rdd_input_location,
+ # Offset=point_rdd_offset,
+ # splitter=point_rdd_splitter,
+ # carryInputData=False
+ # )
+ # circle_rdd = CircleRDD(object_rdd, 0.1)
+ # collected_data = circle_rdd.getRawSpatialRDD().collect()
+ # print([geo_data.geom.wkt for geo_data in collected_data])
def test_linestring_rdd(self):
linestring_rdd = LineStringRDD(
diff --git a/python/tests/serialization/test_serializers.py b/python/tests/serialization/test_serializers.py
index 12895a5..2604c8e 100644
--- a/python/tests/serialization/test_serializers.py
+++ b/python/tests/serialization/test_serializers.py
@@ -43,7 +43,6 @@ class TestsSerializers(TestBase):
t.StructField("geom_to", GeometryType(), True)
]
)
-
self.spark.createDataFrame(
data,
schema
@@ -70,7 +69,6 @@ class TestsSerializers(TestBase):
t.StructField("geom", GeometryType(), True)
]
)
-
m_point_out = self.spark.createDataFrame(
data,
schema
diff --git a/python/tests/serialization/test_with_sc_parellize.py b/python/tests/serialization/test_with_sc_parellize.py
index ccc733d..3db854a 100644
--- a/python/tests/serialization/test_with_sc_parellize.py
+++ b/python/tests/serialization/test_with_sc_parellize.py
@@ -19,7 +19,6 @@ from shapely.geometry import Point, LineString, Polygon
from shapely.wkt import loads
from sedona.core.SpatialRDD import PointRDD, LineStringRDD, PolygonRDD
-from sedona.core.serde.geom_factory import serializers
from sedona.utils.spatial_rdd_parser import GeoData
from tests.test_base import TestBase
@@ -28,9 +27,9 @@ class TestWithScParallelize(TestBase):
def test_geo_data_convert_to_point_rdd(self):
points = [
- GeoData(geom=Point(52.0, -21.0), userData="a", serde=serializers[self.serializer_type]),
- GeoData(geom=Point(-152.4546, -23.1423), userData="b", serde=serializers[self.serializer_type]),
- GeoData(geom=Point(62.253456, 221.2145), userData="c", serde=serializers[self.serializer_type])
+ GeoData(geom=Point(52.0, -21.0), userData="a"),
+ GeoData(geom=Point(-152.4546, -23.1423), userData="b"),
+ GeoData(geom=Point(62.253456, 221.2145), userData="c")
]
rdd_data = self.sc.parallelize(points)
@@ -46,8 +45,8 @@ class TestWithScParallelize(TestBase):
linestring2 = loads(wkt)
linestrings = [
- GeoData(geom=linestring, userData="a", serde=serializers[self.serializer_type]),
- GeoData(geom=linestring2, userData="b", serde=serializers[self.serializer_type]),
+ GeoData(geom=linestring, userData="a"),
+ GeoData(geom=linestring2, userData="b"),
]
rdd_data = self.sc.parallelize(linestrings)
@@ -69,9 +68,9 @@ class TestWithScParallelize(TestBase):
polygon3 = loads(wkt)
polygons = [
- GeoData(geom=polygon, userData="a", serde=serializers[self.serializer_type]),
- GeoData(geom=polygon2, userData="b", serde=serializers[self.serializer_type]),
- GeoData(geom=polygon3, userData="c", serde=serializers[self.serializer_type]),
+ GeoData(geom=polygon, userData="a"),
+ GeoData(geom=polygon2, userData="b"),
+ GeoData(geom=polygon3, userData="c"),
]
rdd_data = self.sc.parallelize(polygons)
diff --git a/python/tests/serializer_type.txt b/python/tests/serializer_type.txt
deleted file mode 100644
index aa07c59..0000000
--- a/python/tests/serializer_type.txt
+++ /dev/null
@@ -1 +0,0 @@
-wkb
diff --git a/python/tests/spatial_operator/test_join_query_correctness.py b/python/tests/spatial_operator/test_join_query_correctness.py
index a15867d..79e7aeb 100644
--- a/python/tests/spatial_operator/test_join_query_correctness.py
+++ b/python/tests/spatial_operator/test_join_query_correctness.py
@@ -22,7 +22,6 @@ from shapely.geometry.base import BaseGeometry
from sedona.core.SpatialRDD import LineStringRDD, PolygonRDD, CircleRDD, PointRDD
from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
from sedona.core.enums import IndexType, GridType
-from sedona.core.serde.geom_factory import serializers
from sedona.core.spatialOperator import JoinQuery
from sedona.utils.spatial_rdd_parser import GeoData
from tests.test_base import TestBase
@@ -198,7 +197,7 @@ class TestJoinQueryCorrectness(TestBase):
@classmethod
def make_square_line(cls, minx: float, miny: float, side: float):
- coordinates = [(minx, miny), (minx + side, miny), (minx + side, miny + side)]
+ coordinates = [(minx, miny), (minx+side, miny), (minx + side, miny+side)]
return LineString(coordinates)
@classmethod
@@ -207,7 +206,7 @@ class TestJoinQueryCorrectness(TestBase):
@classmethod
def wrap(cls, geom: BaseGeometry, user_data: str):
- return GeoData(geom=geom, userData=user_data, serde=serializers[cls.serializer_type])
+ return GeoData(geom=geom, userData=user_data)
@classmethod
def once_before_all(cls):
diff --git a/python/tests/spatial_operator/test_linestring_join.py b/python/tests/spatial_operator/test_linestring_join.py
index d5e6eda..14afa15 100644
--- a/python/tests/spatial_operator/test_linestring_join.py
+++ b/python/tests/spatial_operator/test_linestring_join.py
@@ -51,7 +51,6 @@ params_dyn.extend([{**param, **{"index_type": IndexType.RTREE}} for param in par
class TestRectangleJoin(TestJoinBase):
-
params = {
"test_nested_loop": parameters,
"test_dynamic_index_int": params_dyn,
diff --git a/python/tests/spatial_rdd/test_circle_rdd.py b/python/tests/spatial_rdd/test_circle_rdd.py
index 941176e..9dc8948 100644
--- a/python/tests/spatial_rdd/test_circle_rdd.py
+++ b/python/tests/spatial_rdd/test_circle_rdd.py
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import pytest
+
from pyspark import StorageLevel
from sedona.core.SpatialRDD import PointRDD, CircleRDD
@@ -24,7 +24,6 @@ from tests.properties.point_properties import input_location, offset, splitter,
class TestCircleRDD(TestBase):
- @pytest.mark.skipif(TestBase.serializer_type == "shape", reason="circle is not supported")
def test_circle_rdd(self):
spatial_rdd = PointRDD(
self.sc,
@@ -43,4 +42,4 @@ class TestCircleRDD(TestBase):
assert circle_rdd.approximateTotalCount == 3000
assert circle_rdd.rawSpatialRDD.take(1)[0].getUserData() == "testattribute0\ttestattribute1\ttestattribute2"
- # assert circle_rdd.rawSpatialRDD.take(1)[0].geom.radius == 0.5
+ assert circle_rdd.rawSpatialRDD.take(1)[0].geom.radius == 0.5
diff --git a/python/tests/spatial_rdd/test_polygon_rdd.py b/python/tests/spatial_rdd/test_polygon_rdd.py
index a0c9a4c..61b8528 100644
--- a/python/tests/spatial_rdd/test_polygon_rdd.py
+++ b/python/tests/spatial_rdd/test_polygon_rdd.py
@@ -16,14 +16,11 @@
# under the License.
from pyspark import StorageLevel
-from pyspark.sql import SparkSession
from sedona.core.SpatialRDD import PolygonRDD
from sedona.core.SpatialRDD.spatial_rdd import SpatialRDD
-from sedona.core.enums import IndexType, FileDataSplitter
+from sedona.core.enums import IndexType, FileDataSplitter, GridType
from sedona.core.geom.envelope import Envelope
-from sedona.register import SedonaRegistrator
-from sedona.utils import KryoSerializer, SedonaKryoRegistrator
from tests.properties.polygon_properties import input_location, splitter, num_partitions, input_count, input_boundary, grid_type, \
input_location_geo_json, input_location_wkt, input_location_wkb, query_envelope, polygon_rdd_input_location, \
polygon_rdd_start_offset, polygon_rdd_end_offset, polygon_rdd_splitter
diff --git a/python/tests/sql/test_adapter.py b/python/tests/sql/test_adapter.py
index 1a82ab6..c2aab85 100644
--- a/python/tests/sql/test_adapter.py
+++ b/python/tests/sql/test_adapter.py
@@ -22,8 +22,6 @@ import pytest
from pyspark.sql import DataFrame
from pyspark.sql.functions import expr
from pyspark.sql.functions import col
-from pyspark.sql.types import StructType, StructField, IntegerType
-from shapely.geometry import Point
from sedona import version
from sedona.core.SpatialRDD import PolygonRDD, CircleRDD
@@ -32,7 +30,6 @@ from sedona.core.formatMapper.shapefileParser.shape_file_reader import Shapefile
from sedona.core.geom.envelope import Envelope
from sedona.core.jvm.config import is_greater_or_equal_version
from sedona.core.spatialOperator import JoinQuery
-from sedona.sql.types import GeometryType
from sedona.utils.adapter import Adapter
from tests import geojson_input_location, shape_file_with_missing_trailing_input_location, \
geojson_id_input_location
@@ -312,15 +309,3 @@ class TestAdapter(TestBase):
assert spatial_df.columns == ["geometry", *spatial_columns]
assert spatial_df.count() == 1001
-
- def test_serialization(self):
- data = [[1, Point(21, 52)]]
- schema = StructType(
- [
- StructField("id", IntegerType()),
- StructField("geom", GeometryType())
- ]
- )
- df = self.spark.createDataFrame(data, schema=schema)
- df.show()
- df.collect()
diff --git a/python/tests/sql/test_function.py b/python/tests/sql/test_function.py
index 340fca5..94d8038 100644
--- a/python/tests/sql/test_function.py
+++ b/python/tests/sql/test_function.py
@@ -18,8 +18,7 @@
import math
from typing import List
-import pytest
-from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql import DataFrame
from pyspark.sql.functions import explode, expr
from pyspark.sql.functions import col
from pyspark.sql.types import StructType, StructField, IntegerType
@@ -34,6 +33,7 @@ from tests.test_base import TestBase
class TestPredicateJoin(TestBase):
+
geo_schema = StructType(
[StructField("geom", GeometryType(), False)]
)
@@ -52,18 +52,6 @@ class TestPredicateJoin(TestBase):
]
)
- def create_sample_points_df(self, spark: SparkSession, num_of_points: int):
- return create_sample_points_df(spark, num_of_points)
-
- def create_sample_polygons_df(self, spark: SparkSession, num_of_points: int):
- return create_sample_polygons_df(spark, num_of_points)
-
- def create_sample_lines_df(self, spark: SparkSession, num_of_points: int):
- return create_sample_lines_df(spark, num_of_points)
-
- def create_simple_polygons_df(self, spark: SparkSession, num_of_points: int):
- return create_simple_polygons_df(spark, num_of_points)
-
def test_st_convex_hull(self):
polygon_wkt_df = self.spark.read.format("csv"). \
option("delimiter", "\t"). \
@@ -178,43 +166,29 @@ class TestPredicateJoin(TestBase):
polygon_df = self.spark.sql("select ST_GeomFromWKT(polygontable._c0) as countyshape from polygontable")
polygon_df.createOrReplaceTempView("polygondf")
polygon_df.show()
- function_df = self.spark.sql(
- "select ST_Transform(ST_FlipCoordinates(polygondf.countyshape), 'epsg:4326','epsg:3857', false) from polygondf")
+ function_df = self.spark.sql("select ST_Transform(ST_FlipCoordinates(polygondf.countyshape), 'epsg:4326','epsg:3857', false) from polygondf")
function_df.show()
def test_st_intersection_intersects_but_not_contains(self):
- test_table = self.spark.sql(
- "select ST_GeomFromWKT('POLYGON((1 1, 8 1, 8 8, 1 8, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b")
+ test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON((1 1, 8 1, 8 8, 1 8, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b")
test_table.createOrReplaceTempView("testtable")
intersect = self.spark.sql("select ST_Intersection(a,b) from testtable")
assert intersect.take(1)[0][0].wkt == "POLYGON ((2 8, 8 8, 8 2, 2 2, 2 8))"
def test_st_intersection_intersects_but_left_contains_right(self):
- test_table = self.spark.sql(
- "select ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as b")
+ test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as b")
test_table.createOrReplaceTempView("testtable")
intersects = self.spark.sql("select ST_Intersection(a,b) from testtable")
- assert intersects.take(1)[0][0].wkt == "POLYGON ((3 3, 2 2, 2 3, 3 3))"
+ assert intersects.take(1)[0][0].wkt == "POLYGON ((2 2, 2 3, 3 3, 2 2))"
def test_st_intersection_intersects_but_right_contains_left(self):
- test_table = self.spark.sql(
- "select ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as a,ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as b")
+ test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as a,ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as b")
test_table.createOrReplaceTempView("testtable")
intersects = self.spark.sql("select ST_Intersection(a,b) from testtable")
assert intersects.take(1)[0][0].wkt == "POLYGON ((2 2, 2 3, 3 3, 2 2))"
- @pytest.mark.skipif(TestBase.serializer_type == "wkb", reason="shp parser does not handle geometry collection")
def test_st_intersection_not_intersects(self):
- test_table = self.spark.sql(
- "select ST_GeomFromWKT('POLYGON((40 21, 40 25, 35 20, 40 21))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b")
- test_table.createOrReplaceTempView("testtable")
- intersects = self.spark.sql("select ST_Intersection(a,b) from testtable")
- assert intersects.take(1)[0][0].wkt == "GEOMETRYCOLLECTION EMPTY"
-
- @pytest.mark.skipif(TestBase.serializer_type == "shape", reason="shp parser does not handle geometry collection")
- def test_st_intersection_not_intersects(self):
- test_table = self.spark.sql(
- "select ST_GeomFromWKT('POLYGON((40 21, 40 25, 35 20, 40 21))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b")
+ test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON((40 21, 40 22, 40 23, 40 21))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b")
test_table.createOrReplaceTempView("testtable")
intersects = self.spark.sql("select ST_Intersection(a,b) from testtable")
assert intersects.take(1)[0][0].wkt == "POLYGON EMPTY"
@@ -261,16 +235,13 @@ class TestPredicateJoin(TestBase):
polygon_df = self.spark.sql("select ST_GeomFromWKT(polygontable._c0) as countyshape from polygontable")
polygon_df.createOrReplaceTempView("polygondf")
wkt_df = self.spark.sql("select ST_AsText(countyshape) as wkt from polygondf")
- polygon_df.show()
assert polygon_df.take(1)[0]["countyshape"].wkt == loads(wkt_df.take(1)[0]["wkt"]).wkt
def test_st_n_points(self):
- test = self.spark.sql(
- "SELECT ST_NPoints(ST_GeomFromText('LINESTRING(77.29 29.07,77.42 29.26,77.27 29.31,77.29 29.07)'))")
+ test = self.spark.sql("SELECT ST_NPoints(ST_GeomFromText('LINESTRING(77.29 29.07,77.42 29.26,77.27 29.31,77.29 29.07)'))")
def test_st_geometry_type(self):
- test = self.spark.sql(
- "SELECT ST_GeometryType(ST_GeomFromText('LINESTRING(77.29 29.07,77.42 29.26,77.27 29.31,77.29 29.07)'))")
+ test = self.spark.sql("SELECT ST_GeometryType(ST_GeomFromText('LINESTRING(77.29 29.07,77.42 29.26,77.27 29.31,77.29 29.07)'))")
def test_st_azimuth(self):
sample_points = create_sample_points(20)
@@ -302,9 +273,9 @@ class TestPredicateJoin(TestBase):
assert azimuths[0] == [42.27368900609373, 222.27368900609372]
def test_st_x(self):
- point_df = self.create_sample_points_df(self.spark, 5)
- polygon_df = self.create_sample_polygons_df(self.spark, 5)
- linestring_df = self.create_sample_lines_df(self.spark, 5)
+ point_df = create_sample_points_df(self.spark, 5)
+ polygon_df = create_sample_polygons_df(self.spark, 5)
+ linestring_df = create_sample_lines_df(self.spark, 5)
points = point_df \
.selectExpr("ST_X(geom)").collect()
@@ -313,16 +284,16 @@ class TestPredicateJoin(TestBase):
linestrings = linestring_df.selectExpr("ST_X(geom) as x").filter("x IS NOT NULL")
- assert ([point[0] for point in points] == [-71.064544, -88.331492, 88.331492, 1.0453, 32.324142])
+ assert([point[0] for point in points] == [-71.064544, -88.331492, 88.331492, 1.0453, 32.324142])
- assert (not linestrings.count())
+ assert(not linestrings.count())
- assert (not polygons.count())
+ assert(not polygons.count())
def test_st_y(self):
- point_df = self.create_sample_points_df(self.spark, 5)
- polygon_df = self.create_sample_polygons_df(self.spark, 5)
- linestring_df = self.create_sample_lines_df(self.spark, 5)
+ point_df = create_sample_points_df(self.spark, 5)
+ polygon_df = create_sample_polygons_df(self.spark, 5)
+ linestring_df = create_sample_lines_df(self.spark, 5)
points = point_df \
.selectExpr("ST_Y(geom)").collect()
@@ -331,17 +302,17 @@ class TestPredicateJoin(TestBase):
linestrings = linestring_df.selectExpr("ST_Y(geom) as y").filter("y IS NOT NULL")
- assert ([point[0] for point in points] == [42.28787, 32.324142, 32.324142, 5.3324324, -88.331492])
+ assert([point[0] for point in points] == [42.28787, 32.324142, 32.324142, 5.3324324, -88.331492])
- assert (not linestrings.count())
+ assert(not linestrings.count())
- assert (not polygons.count())
+ assert(not polygons.count())
def test_st_start_point(self):
- point_df = self.create_sample_points_df(self.spark, 5)
- polygon_df = self.create_sample_polygons_df(self.spark, 5)
- linestring_df = self.create_sample_lines_df(self.spark, 5)
+ point_df = create_sample_points_df(self.spark, 5)
+ polygon_df = create_sample_polygons_df(self.spark, 5)
+ linestring_df = create_sample_lines_df(self.spark, 5)
expected_points = [
"POINT (-112.506968 45.98186)",
@@ -357,16 +328,16 @@ class TestPredicateJoin(TestBase):
linestrings = linestring_df.selectExpr("ST_StartPoint(geom) as geom").filter("geom IS NOT NULL")
- assert ([line[0] for line in linestrings.collect()] == [wkt.loads(el) for el in expected_points])
+ assert([line[0] for line in linestrings.collect()] == [wkt.loads(el) for el in expected_points])
- assert (not points.count())
+ assert(not points.count())
- assert (not polygons.count())
+ assert(not polygons.count())
def test_st_end_point(self):
- linestring_dataframe = self.create_sample_lines_df(self.spark, 5)
- other_geometry_dataframe = self.create_sample_points_df(self.spark, 5). \
- union(self.create_sample_points_df(self.spark, 5))
+ linestring_dataframe = create_sample_lines_df(self.spark, 5)
+ other_geometry_dataframe = create_sample_points_df(self.spark, 5). \
+ union(create_sample_points_df(self.spark, 5))
point_data_frame = linestring_dataframe.selectExpr("ST_EndPoint(geom) as geom"). \
filter("geom IS NOT NULL")
@@ -381,10 +352,10 @@ class TestPredicateJoin(TestBase):
empty_dataframe = other_geometry_dataframe.selectExpr("ST_EndPoint(geom) as geom"). \
filter("geom IS NOT NULL")
- assert ([wkt_row[0]
- for wkt_row in point_data_frame.selectExpr("ST_AsText(geom)").collect()] == expected_ending_points)
+ assert([wkt_row[0]
+ for wkt_row in point_data_frame.selectExpr("ST_AsText(geom)").collect()] == expected_ending_points)
- assert (empty_dataframe.count() == 0)
+ assert(empty_dataframe.count() == 0)
def test_st_boundary(self):
wkt_list = [
@@ -407,7 +378,7 @@ class TestPredicateJoin(TestBase):
boundary_table = geometry_table.selectExpr("ST_Boundary(geom) as geom")
boundary_wkt = [wkt_row[0] for wkt_row in boundary_table.selectExpr("ST_AsText(geom)").collect()]
- assert (boundary_wkt == [
+ assert(boundary_wkt == [
"MULTIPOINT ((1 1), (-1 1))",
"MULTIPOINT ((100 150), (160 170))",
"MULTILINESTRING ((10 130, 50 190, 110 190, 140 150, 150 80, 100 10, 20 40, 10 130), (70 40, 100 50, 120 80, 80 110, 50 90, 70 40))",
@@ -415,14 +386,14 @@ class TestPredicateJoin(TestBase):
])
def test_st_exterior_ring(self):
- polygon_df = self.create_simple_polygons_df(self.spark, 5,)
+ polygon_df = create_simple_polygons_df(self.spark, 5)
additional_wkt = "POLYGON((0 0 1, 1 1 1, 1 2 1, 1 1 1, 0 0 1))"
additional_wkt_df = self.spark.createDataFrame([[wkt.loads(additional_wkt)]], self.geo_schema)
polygons_df = polygon_df.union(additional_wkt_df)
- other_geometry_df = self.create_sample_lines_df(self.spark, 5).union(
- self.create_sample_points_df(self.spark, 5))
+ other_geometry_df = create_sample_lines_df(self.spark, 5).union(
+ create_sample_points_df(self.spark, 5))
linestring_df = polygons_df.selectExpr("ST_ExteriorRing(geom) as geom").filter("geom IS NOT NULL")
@@ -430,96 +401,45 @@ class TestPredicateJoin(TestBase):
linestring_wkt = [wkt_row[0] for wkt_row in linestring_df.selectExpr("ST_AsText(geom)").collect()]
- assert (linestring_wkt == ["LINESTRING (0 0, 0 1, 1 1, 1 0, 0 0)", "LINESTRING (0 0, 1 1, 1 2, 1 1, 0 0)"])
+ assert(linestring_wkt == ["LINESTRING (0 0, 0 1, 1 1, 1 0, 0 0)", "LINESTRING (0 0, 1 1, 1 2, 1 1, 0 0)"])
- assert (not empty_df.count())
+ assert(not empty_df.count())
def test_st_geometry_n(self):
data_frame = self.__wkt_list_to_data_frame(["MULTIPOINT((1 2), (3 4), (5 6), (8 9))"])
wkts = [data_frame.selectExpr(f"ST_GeometryN(geom, {i}) as geom").selectExpr("st_asText(geom)").collect()[0][0]
for i in range(0, 4)]
- assert (wkts == ["POINT (1 2)", "POINT (3 4)", "POINT (5 6)", "POINT (8 9)"])
+ assert(wkts == ["POINT (1 2)", "POINT (3 4)", "POINT (5 6)", "POINT (8 9)"])
def test_st_interior_ring_n(self):
polygon_df = self.__wkt_list_to_data_frame(
- [
- "POLYGON((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1), (1 3, 2 3, 2 4, 1 4, 1 3), (3 3, 4 3, 4 4, 3 4, 3 3))"]
+ ["POLYGON((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1), (1 3, 2 3, 2 4, 1 4, 1 3), (3 3, 4 3, 4 4, 3 4, 3 3))"]
)
- other_geometry = self.create_sample_points_df(self.spark, 5).union(
- self.create_sample_lines_df(self.spark, 5))
+ other_geometry = create_sample_points_df(self.spark, 5).union(create_sample_lines_df(self.spark, 5))
wholes = [polygon_df.selectExpr(f"ST_InteriorRingN(geom, {i}) as geom").
selectExpr("ST_AsText(geom)").collect()[0][0]
for i in range(3)]
empty_df = other_geometry.selectExpr("ST_InteriorRingN(geom, 1) as geom").filter("geom IS NOT NULL")
- assert (not empty_df.count())
- assert (wholes == ["LINESTRING (1 1, 2 1, 2 2, 1 2, 1 1)",
- "LINESTRING (1 3, 2 3, 2 4, 1 4, 1 3)",
- "LINESTRING (3 3, 4 3, 4 4, 3 4, 3 3)"])
-
- @pytest.mark.skipif(TestBase.serializer_type == "wkb", reason="shp parser does not handle geometry collection")
- def test_st_dumps(self):
- expected_geometries = [
- 'POINT (21 52)',
- 'POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))',
- 'LINESTRING (0 0, 1 1, 1 0)',
- 'POINT (10 40)',
- 'POINT (40 30)',
- 'POINT (20 20)',
- 'POINT (30 10)',
- 'POLYGON ((30 20, 45 40, 10 40, 30 20))',
- 'POLYGON ((15 5, 40 10, 10 20, 5 10, 15 5))',
- 'LINESTRING (10 10, 20 20, 10 40)',
- 'LINESTRING (40 40, 30 30, 40 20, 30 10)',
- 'POLYGON ((40 40, 20 45, 45 30, 40 40))',
- 'POLYGON ((20 35, 10 30, 10 10, 30 5, 45 20, 20 35), (30 20, 20 15, 20 25, 30 '
- '20))',
- 'POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))'
- ]
- geometry_df = self.__wkt_list_to_data_frame(
- [
- "Point(21 52)",
- "Polygon((0 0, 0 1, 1 1, 1 0, 0 0))",
- "Linestring(0 0, 1 1, 1 0)",
- "MULTIPOINT ((10 40), (40 30), (20 20), (30 10))",
- "MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), ((15 5, 40 10, 10 20, 5 10, 15 5)))",
- "MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))",
- "MULTIPOLYGON (((40 40, 20 45, 45 30, 40 40)), ((20 35, 10 30, 10 10, 30 5, 45 20, 20 35), (30 20, 20 15, 20 25, 30 20)))",
- "POLYGON((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))"
- ]
- )
-
- dumped_geometries = geometry_df.selectExpr("ST_Dump(geom) as geom")
-
- assert (dumped_geometries.select(explode(col("geom"))).count() == 14)
+ assert(not empty_df.count())
+ assert(wholes == ["LINESTRING (1 1, 2 1, 2 2, 1 2, 1 1)",
+ "LINESTRING (1 3, 2 3, 2 4, 1 4, 1 3)",
+ "LINESTRING (3 3, 4 3, 4 4, 3 4, 3 3)"])
- collected_geometries = dumped_geometries \
- .select(explode(col("geom")).alias("geom")) \
- .selectExpr("ST_AsText(geom) as geom") \
- .collect()
-
- assert ([geom_row[0] for geom_row in collected_geometries] == expected_geometries)
-
- @pytest.mark.skipif(TestBase.serializer_type == "shape", reason="shp parser does not handle geometry collection")
def test_st_dumps(self):
expected_geometries = [
- 'POINT (21 52)',
- 'POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))',
- 'LINESTRING (0 0, 1 1, 1 0)',
- 'POINT (10 40)',
- 'POINT (40 30)',
- 'POINT (20 20)',
- 'POINT (30 10)',
- 'POLYGON ((30 20, 45 40, 10 40, 30 20))',
- 'POLYGON ((15 5, 40 10, 10 20, 5 10, 15 5))',
- 'LINESTRING (10 10, 20 20, 10 40)',
- 'LINESTRING (40 40, 30 30, 40 20, 30 10)',
- 'POLYGON ((40 40, 20 45, 45 30, 40 40))',
- 'POLYGON ((20 35, 10 30, 10 10, 30 5, 45 20, 20 35), (30 20, 20 15, 20 25, 30 20))',
- 'POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))'
+ "POINT (21 52)", "POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))",
+ "LINESTRING (0 0, 1 1, 1 0)",
+ "POINT (10 40)", "POINT (40 30)", "POINT (20 20)", "POINT (30 10)",
+ "POLYGON ((30 20, 45 40, 10 40, 30 20))",
+ "POLYGON ((15 5, 40 10, 10 20, 5 10, 15 5))", "LINESTRING (10 10, 20 20, 10 40)",
+ "LINESTRING (40 40, 30 30, 40 20, 30 10)",
+ "POLYGON ((40 40, 20 45, 45 30, 40 40))",
+ "POLYGON ((20 35, 10 30, 10 10, 30 5, 45 20, 20 35), (30 20, 20 15, 20 25, 30 20))",
+ "POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))"
]
geometry_df = self.__wkt_list_to_data_frame(
@@ -537,14 +457,14 @@ class TestPredicateJoin(TestBase):
dumped_geometries = geometry_df.selectExpr("ST_Dump(geom) as geom")
- assert (dumped_geometries.select(explode(col("geom"))).count() == 14)
+ assert(dumped_geometries.select(explode(col("geom"))).count() == 14)
collected_geometries = dumped_geometries \
.select(explode(col("geom")).alias("geom")) \
.selectExpr("ST_AsText(geom) as geom") \
.collect()
- assert ([geom_row[0] for geom_row in collected_geometries] == expected_geometries)
+ assert([geom_row[0] for geom_row in collected_geometries] == expected_geometries)
def test_st_dump_points(self):
expected_points = [
@@ -557,19 +477,18 @@ class TestPredicateJoin(TestBase):
"POINT (1 1)", "POINT (1 0)",
"POINT (0 0)"
]
- geometry_df = self.create_sample_lines_df(self.spark, 1) \
- .union(self.create_sample_points_df(self.spark, 1)) \
- .union(self.create_simple_polygons_df(self.spark, 1))
+ geometry_df = create_sample_lines_df(self.spark, 1) \
+ .union(create_sample_points_df(self.spark, 1)) \
+ .union(create_simple_polygons_df(self.spark, 1))
dumped_points = geometry_df.selectExpr("ST_DumpPoints(geom) as geom") \
.select(explode(col("geom")).alias("geom"))
- assert (dumped_points.count() == 10)
+ assert(dumped_points.count() == 10)
collected_points = [geom_row[0] for geom_row in dumped_points.selectExpr("ST_AsText(geom)").collect()]
- assert (collected_points == expected_points)
+ assert(collected_points == expected_points)
- @pytest.mark.skipif(TestBase.serializer_type == "shp", reason="shp parser does not handle geometry collection")
def test_st_is_closed(self):
expected_result = [
[1, True],
@@ -593,46 +512,14 @@ class TestPredicateJoin(TestBase):
(7, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))"),
(8, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10))"),
(9, "MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))"),
- (10,
- "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))")
- ]
-
- geometry_df = self.__wkt_pair_list_with_index_to_data_frame(geometry_list)
- is_closed = geometry_df.selectExpr("index", "ST_IsClosed(geom)").collect()
- is_closed_collected = [[*row] for row in is_closed]
- assert (is_closed_collected == expected_result)
-
- @pytest.mark.skipif(TestBase.serializer_type == "wkb", reason="shp parser test")
- def test_st_is_closed(self):
- expected_result = [
- [1, True],
- [2, True],
- [3, False],
- [4, True],
- [5, True],
- [6, True],
- [7, True],
- [8, False],
- [9, False]
- ]
- geometry_list = [
- (1, "Point(21 52)"),
- (2, "Polygon((0 0, 0 1, 1 1, 1 0, 0 0))"),
- (3, "Linestring(0 0, 1 1, 1 0)"),
- (4, "Linestring(0 0, 1 1, 1 0, 0 0)"),
- (5, "MULTIPOINT ((10 40), (40 30), (20 20), (30 10))"),
- (6, "MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), ((15 5, 40 10, 10 20, 5 10, 15 5)))"),
- (7, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))"),
- (8, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10))"),
- (9, "MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))")
+ (10, "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))")
]
geometry_df = self.__wkt_pair_list_with_index_to_data_frame(geometry_list)
is_closed = geometry_df.selectExpr("index", "ST_IsClosed(geom)").collect()
is_closed_collected = [[*row] for row in is_closed]
- assert (is_closed_collected == expected_result)
+ assert(is_closed_collected == expected_result)
- @pytest.mark.skipif(TestBase.serializer_type == "shp", reason="shp parser does not handle geometry collection")
def test_num_interior_ring(self):
geometries = [
(1, "Point(21 52)"),
@@ -644,61 +531,15 @@ class TestPredicateJoin(TestBase):
(7, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))"),
(8, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10))"),
(9, "MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))"),
- (10,
- "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))"),
+ (10, "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))"),
(11, "POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))")]
geometry_df = self.__wkt_pair_list_with_index_to_data_frame(geometries)
number_of_interior_rings = geometry_df.selectExpr("index", "ST_NumInteriorRings(geom) as num")
collected_interior_rings = [[*row] for row in number_of_interior_rings.filter("num is not null").collect()]
- assert (collected_interior_rings == [[2, 0], [11, 1]])
-
- @pytest.mark.skipif(TestBase.serializer_type == "wkb", reason="shp parser test")
- def test_num_interior_ring(self):
- geometries = [
- (1, "Point(21 52)"),
- (2, "Polygon((0 0, 0 1, 1 1, 1 0, 0 0))"),
- (3, "Linestring(0 0, 1 1, 1 0)"),
- (4, "Linestring(0 0, 1 1, 1 0, 0 0)"),
- (5, "MULTIPOINT ((10 40), (40 30), (20 20), (30 10))"),
- (6, "MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), ((15 5, 40 10, 10 20, 5 10, 15 5)))"),
- (7, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))"),
- (8, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10))"),
- (9, "MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))"),
- (11, "POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))")]
-
- geometry_df = self.__wkt_pair_list_with_index_to_data_frame(geometries)
-
- number_of_interior_rings = geometry_df.selectExpr("index", "ST_NumInteriorRings(geom) as num")
- collected_interior_rings = [[*row] for row in number_of_interior_rings.filter("num is not null").collect()]
- assert (collected_interior_rings == [[2, 0], [11, 1]])
-
- @pytest.mark.skipif(TestBase.serializer_type == "shp", reason="shp parser does not handle geometry collection")
- def test_st_add_point(self):
- geometry = [
- ("Point(21 52)", "Point(21 52)"),
- ("Point(21 52)", "Polygon((0 0, 0 1, 1 1, 1 0, 0 0))"),
- ("Linestring(0 0, 1 1, 1 0)", "Point(21 52)"),
- ("Linestring(0 0, 1 1, 1 0, 0 0)", "Linestring(0 0, 1 1, 1 0, 0 0)"),
- ("Point(21 52)", "MULTIPOINT ((10 40), (40 30), (20 20), (30 10))"),
- ("MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), ((15 5, 40 10, 10 20, 5 10, 15 5)))", "Point(21 52)"),
- ("MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))", "Point(21 52)"),
- ("MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10))", "Point(21 52)"),
- ("MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))", "Point(21 52)"),
- (
- "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))",
- "Point(21 52)"),
- ("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))", "Point(21 52)")
- ]
- geometry_df = self.__wkt_pairs_to_data_frame(geometry)
- modified_geometries = geometry_df.selectExpr("ST_AddPoint(geomA, geomB) as geom")
- collected_geometries = [
- row[0] for row in modified_geometries.filter("geom is not null").selectExpr("ST_AsText(geom)").collect()
- ]
- assert (collected_geometries[0] == "LINESTRING (0 0, 1 1, 1 0, 21 52)")
+ assert(collected_interior_rings == [[2, 0], [11, 1]])
- @pytest.mark.skipif(TestBase.serializer_type == "wkb", reason="shp parser test")
def test_st_add_point(self):
geometry = [
("Point(21 52)", "Point(21 52)"),
@@ -710,6 +551,7 @@ class TestPredicateJoin(TestBase):
("MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))", "Point(21 52)"),
("MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10))", "Point(21 52)"),
("MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))", "Point(21 52)"),
+ ("GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))", "Point(21 52)"),
("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))", "Point(21 52)")
]
geometry_df = self.__wkt_pairs_to_data_frame(geometry)
@@ -717,9 +559,8 @@ class TestPredicateJoin(TestBase):
collected_geometries = [
row[0] for row in modified_geometries.filter("geom is not null").selectExpr("ST_AsText(geom)").collect()
]
- assert (collected_geometries[0] == "LINESTRING (0 0, 1 1, 1 0, 21 52)")
+ assert(collected_geometries[0] == "LINESTRING (0 0, 1 1, 1 0, 21 52)")
- @pytest.mark.skipif(TestBase.serializer_type == "shp", reason="shp parser does not handle geometry collection")
def test_st_remove_point(self):
result_and_expected = [
[self.calculate_st_remove("Linestring(0 0, 1 1, 1 0, 0 0)", 0), "LINESTRING (1 1, 1 0, 0 0)"],
@@ -729,30 +570,11 @@ class TestPredicateJoin(TestBase):
[self.calculate_st_remove("POINT(0 1)", 3), None],
[self.calculate_st_remove("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))", 3), None],
[self.calculate_st_remove("GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40))", 0), None],
- [self.calculate_st_remove(
- "MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), ((15 5, 40 10, 10 20, 5 10, 15 5)))", 3), None],
- [self.calculate_st_remove(
- "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))", 3), None]
- ]
- for actual, expected in result_and_expected:
- assert (actual == expected)
-
- @pytest.mark.skipif(TestBase.serializer_type == "wkb", reason="shp parser test")
- def test_st_remove_point(self):
- result_and_expected = [
- [self.calculate_st_remove("Linestring(0 0, 1 1, 1 0, 0 0)", 0), "LINESTRING (1 1, 1 0, 0 0)"],
- [self.calculate_st_remove("Linestring(0 0, 1 1, 1 0, 0 0)", 1), "LINESTRING (0 0, 1 0, 0 0)"],
- [self.calculate_st_remove("Linestring(0 0, 1 1, 1 0, 0 0)", 2), "LINESTRING (0 0, 1 1, 0 0)"],
- [self.calculate_st_remove("Linestring(0 0, 1 1, 1 0, 0 0)", 3), "LINESTRING (0 0, 1 1, 1 0)"],
- [self.calculate_st_remove("POINT(0 1)", 3), None],
- [self.calculate_st_remove("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))", 3), None],
- [self.calculate_st_remove(
- "MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), ((15 5, 40 10, 10 20, 5 10, 15 5)))", 3), None],
- [self.calculate_st_remove(
- "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))", 3), None]
+ [self.calculate_st_remove("MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), ((15 5, 40 10, 10 20, 5 10, 15 5)))", 3), None],
+ [self.calculate_st_remove("MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))", 3), None]
]
for actual, expected in result_and_expected:
- assert (actual == expected)
+ assert(actual == expected)
def test_st_is_ring(self):
result_and_expected = [
@@ -763,7 +585,7 @@ class TestPredicateJoin(TestBase):
[self.calculate_st_is_ring("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))"), None],
]
for actual, expected in result_and_expected:
- assert (actual == expected)
+ assert(actual == expected)
def test_st_subdivide(self):
# Given
@@ -836,12 +658,10 @@ class TestPredicateJoin(TestBase):
return geometry_collected[0][0] if geometry_collected.__len__() != 0 else None
def __wkt_pairs_to_data_frame(self, wkt_list: List) -> DataFrame:
- return self.spark.createDataFrame([[wkt.loads(wkt_a), wkt.loads(wkt_b)] for wkt_a, wkt_b in wkt_list],
- self.geo_pair_schema)
+ return self.spark.createDataFrame([[wkt.loads(wkt_a), wkt.loads(wkt_b)] for wkt_a, wkt_b in wkt_list], self.geo_pair_schema)
def __wkt_list_to_data_frame(self, wkt_list: List) -> DataFrame:
return self.spark.createDataFrame([[wkt.loads(given_wkt)] for given_wkt in wkt_list], self.geo_schema)
def __wkt_pair_list_with_index_to_data_frame(self, wkt_list: List) -> DataFrame:
- return self.spark.createDataFrame([[index, wkt.loads(given_wkt)] for index, given_wkt in wkt_list],
- self.geo_schema_with_index)
+ return self.spark.createDataFrame([[index, wkt.loads(given_wkt)] for index, given_wkt in wkt_list], self.geo_schema_with_index)
\ No newline at end of file
diff --git a/python/tests/sql/test_spatial_rdd_to_spatial_dataframe.py b/python/tests/sql/test_spatial_rdd_to_spatial_dataframe.py
index 064f56e..1b73360 100644
--- a/python/tests/sql/test_spatial_rdd_to_spatial_dataframe.py
+++ b/python/tests/sql/test_spatial_rdd_to_spatial_dataframe.py
@@ -77,6 +77,8 @@ class TestSpatialRDDToDataFrame(TestBase):
lambda x: [x.geom, *x.getUserData().split("\t")]
)
+ self.spark.createDataFrame(raw_spatial_rdd).show()
+
schema = StructType(
[
StructField("geom", GeometryType()),
diff --git a/python/tests/spatial_rdd/test_circle_rdd.py b/python/tests/test_assign_raw_spatial_rdd.py
similarity index 63%
copy from python/tests/spatial_rdd/test_circle_rdd.py
copy to python/tests/test_assign_raw_spatial_rdd.py
index 941176e..6d5d205 100644
--- a/python/tests/spatial_rdd/test_circle_rdd.py
+++ b/python/tests/test_assign_raw_spatial_rdd.py
@@ -14,18 +14,16 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import pytest
-from pyspark import StorageLevel
-from sedona.core.SpatialRDD import PointRDD, CircleRDD
-from tests.test_base import TestBase
+from sedona.core.SpatialRDD import PointRDD
from tests.properties.point_properties import input_location, offset, splitter, num_partitions
+from tests.test_base import TestBase
+from pyspark import StorageLevel
-class TestCircleRDD(TestBase):
+class TestSpatialRddAssignment(TestBase):
- @pytest.mark.skipif(TestBase.serializer_type == "shape", reason="circle is not supported")
- def test_circle_rdd(self):
+ def test_raw_spatial_rdd_assignment(self):
spatial_rdd = PointRDD(
self.sc,
input_location,
@@ -35,12 +33,13 @@ class TestCircleRDD(TestBase):
num_partitions,
StorageLevel.MEMORY_ONLY
)
+ spatial_rdd.analyze()
- circle_rdd = CircleRDD(spatial_rdd, 0.5)
-
- circle_rdd.analyze()
-
- assert circle_rdd.approximateTotalCount == 3000
+ empty_point_rdd = PointRDD()
+ empty_point_rdd.rawSpatialRDD = spatial_rdd.rawSpatialRDD
+ empty_point_rdd.analyze()
+ assert empty_point_rdd.countWithoutDuplicates() == spatial_rdd.countWithoutDuplicates()
+ assert empty_point_rdd.boundaryEnvelope == spatial_rdd.boundaryEnvelope
- assert circle_rdd.rawSpatialRDD.take(1)[0].getUserData() == "testattribute0\ttestattribute1\ttestattribute2"
- # assert circle_rdd.rawSpatialRDD.take(1)[0].geom.radius == 0.5
+ assert empty_point_rdd.rawSpatialRDD.map(lambda x: x.geom.area).collect()[0] == 0.0
+ assert empty_point_rdd.rawSpatialRDD.take(9)[4].getUserData() == "testattribute0\ttestattribute1\ttestattribute2"
diff --git a/python/tests/test_base.py b/python/tests/test_base.py
index 95e8379..e4da6dc 100644
--- a/python/tests/test_base.py
+++ b/python/tests/test_base.py
@@ -14,44 +14,34 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
from pyspark.sql import SparkSession
from sedona.register import SedonaRegistrator
from sedona.utils import KryoSerializer, SedonaKryoRegistrator
from sedona.utils.decorators import classproperty
-from tests.tools import serializer_path
class TestBase:
@classproperty
- def serializer_type(self):
- with open(serializer_path) as file:
- lines = file.readlines()
- return "".join(lines).strip()
-
- @classproperty
def spark(self):
if not hasattr(self, "__spark"):
spark = SparkSession. \
builder. \
- config("sedona.serializer.type", self.serializer_type). \
- config("spark.serializer", KryoSerializer.getName). \
- config("spark.kryo.registrator", SedonaKryoRegistrator.getName). \
- config("spark.kryoserializer.buffer.max.mb", "512"). \
- master("local[*]"). \
+ config("spark.serializer", KryoSerializer.getName).\
+ config("spark.kryo.registrator", SedonaKryoRegistrator.getName) .\
+ master("local[*]").\
getOrCreate()
SedonaRegistrator.registerAll(spark)
setattr(self, "__spark", spark)
-
return getattr(self, "__spark")
@classproperty
def sc(self):
if not hasattr(self, "__spark"):
- setattr(self, "__spark", self.spark)
- if not hasattr(self, "__sc"):
setattr(self, "__sc", self.spark._sc)
return getattr(self, "__sc")
+
diff --git a/python/tests/tools.py b/python/tests/tools.py
index dc5bbaa..1e89f1d 100644
--- a/python/tests/tools.py
+++ b/python/tests/tools.py
@@ -23,10 +23,6 @@ from sedona.utils.spatial_rdd_parser import GeoData
tests_path = path.abspath(path.join(__file__ ,"../../../core/src/test/"))
tests_resource = path.abspath(path.join(__file__ ,"../../../core/src/test/resources/"))
-serializer_absolute_path = path.abspath(path.join(__file__, "../"))
-
-serializer_path = path.join(serializer_absolute_path, "serializer_type.txt")
-
def distance_sorting_functions(geo_data: GeoData, query_point: Point):
return geo_data.geom.distance(query_point)
diff --git a/python/tests/utils/test_crs_transformation.py b/python/tests/utils/test_crs_transformation.py
index 04a3a2b..9c7748b 100644
--- a/python/tests/utils/test_crs_transformation.py
+++ b/python/tests/utils/test_crs_transformation.py
@@ -20,7 +20,7 @@ from pyspark import StorageLevel
from sedona.core.SpatialRDD import PointRDD, PolygonRDD, CircleRDD
from sedona.core.enums import GridType
from sedona.core.geom.circle import Circle
-from sedona.core.spatialOperator import RangeQuery, KNNQuery, JoinQuery, RangeQueryRaw
+from sedona.core.spatialOperator import RangeQuery, KNNQuery, JoinQuery
from tests.properties.crs_transform import *
from tests.properties.polygon_properties import grid_type
from tests.test_base import TestBase
diff --git a/sql/src/main/scala/org/apache/sedona/sql/serde/SedonaSerializer.scala b/sql/src/main/scala/org/apache/sedona/sql/serde/SedonaSerializer.scala
deleted file mode 100644
index 6c4bf82..0000000
--- a/sql/src/main/scala/org/apache/sedona/sql/serde/SedonaSerializer.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sedona.sql.serde
-
-import com.esotericsoftware.kryo.Kryo
-import com.esotericsoftware.kryo.io.{Input, Output}
-import org.apache.sedona.core.enums.SerializerType
-import org.apache.sedona.core.serde.GeometrySerde
-import org.apache.sedona.core.serde.WKB.WKBGeometrySerde
-import org.apache.sedona.core.serde.shape.ShapeGeometrySerde
-import org.apache.spark.sql.catalyst.util.ArrayData
-import org.locationtech.jts.geom.Geometry
-import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
-
-import org.apache.spark.sql.catalyst.util.ArrayData.toArrayData
-
-class SedonaSerializer(geometrySerde: GeometrySerde) {
-
- def serialize(geometry: Geometry): Array[Byte] = {
- val out = new ByteArrayOutputStream()
- val kryo = new Kryo()
- val output = new Output(out)
- geometrySerde.write(kryo, output, geometry)
- output.close()
- out.toByteArray
- }
-
- def deserialize(values: ArrayData): Geometry = {
- deserialize(values.toByteArray())
- }
-
- def deserialize(values: Array[Byte]): Geometry = {
- val in = new ByteArrayInputStream(values)
- val kryo = new Kryo()
- val input = new Input(in)
- val geometry = geometrySerde.read(kryo, input, classOf[Geometry])
- input.close()
- geometry.asInstanceOf[Geometry]
- }
-
-}
-
-object SedonaSerializer {
-
- def apply(serializerType: SerializerType): SedonaSerializer = {
- new SedonaSerializer(getSerializer(serializerType))
- }
-
- def getSerializer(serializerType: SerializerType): GeometrySerde = {
-
- serializerType match {
- case SerializerType.SHAPE => new ShapeGeometrySerde()
- case SerializerType.WKB => new WKBGeometrySerde()
- case _ => new ShapeGeometrySerde()
- }
-
- }
-}
diff --git a/python-adapter/src/main/scala/org/apache/sedona/python/wrapper/translation/serde/WkbPythonGeometrySerde.scala b/sql/src/main/scala/org/apache/sedona/sql/utils/GeometrySerializer.scala
similarity index 60%
rename from python-adapter/src/main/scala/org/apache/sedona/python/wrapper/translation/serde/WkbPythonGeometrySerde.scala
rename to sql/src/main/scala/org/apache/sedona/sql/utils/GeometrySerializer.scala
index ec29ff5..a4f752e 100644
--- a/python-adapter/src/main/scala/org/apache/sedona/python/wrapper/translation/serde/WkbPythonGeometrySerde.scala
+++ b/sql/src/main/scala/org/apache/sedona/sql/utils/GeometrySerializer.scala
@@ -16,27 +16,36 @@
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.sedona.sql.utils
-
-package org.apache.sedona.python.wrapper.translation.serde
-
-import java.nio.ByteOrder
-
-import org.apache.sedona.python.wrapper.utils.implicits.IntImplicit
+import org.apache.spark.sql.catalyst.util.ArrayData
import org.locationtech.jts.geom.Geometry
import org.locationtech.jts.io.{WKBReader, WKBWriter}
-object WkbPythonGeometrySerde extends PythonGeometrySerde {
-
+/**
+ * SerDe using the WKB reader and writer objects
+ */
+object GeometrySerializer {
+
+ /**
+ * Given a geometry returns array of bytes
+ *
+ * @param geometry JTS geometry
+ * @return Array of bites represents this geometry
+ */
def serialize(geometry: Geometry): Array[Byte] = {
- val writer: WKBWriter = new WKBWriter(2, 2)
- val data = writer.write(geometry)
- Array(1.toByte) ++ data.length.toByteArray(ByteOrder.BIG_ENDIAN) ++ data
+ val writer = new WKBWriter(2, 2, true)
+ writer.write(geometry)
}
- def deserialize(array: Array[Byte]): Geometry = {
- val wkbReader = new WKBReader()
- wkbReader.read(array.slice(5, array.length))
+ /**
+ * Given ArrayData returns Geometry
+ *
+ * @param values ArrayData
+ * @return JTS geometry
+ */
+ def deserialize(values: ArrayData): Geometry = {
+ val reader = new WKBReader()
+ reader.read(values.toByteArray())
}
-
}
diff --git a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeometryUDT.scala b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeometryUDT.scala
index 5a38423..7e302a7 100644
--- a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeometryUDT.scala
+++ b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/UDT/GeometryUDT.scala
@@ -18,8 +18,8 @@
*/
package org.apache.spark.sql.sedona_sql.UDT
+import org.apache.sedona.sql.utils.GeometrySerializer
import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
-import org.apache.spark.sql.sedona_sql.sedonaSerializer
import org.apache.spark.sql.types._
import org.json4s.JsonDSL._
import org.json4s.JsonAST.JValue
@@ -34,12 +34,12 @@ class GeometryUDT extends UserDefinedType[Geometry] {
override def userClass: Class[Geometry] = classOf[Geometry]
override def serialize(obj: Geometry): GenericArrayData =
- new GenericArrayData(sedonaSerializer.serialize(obj))
+ new GenericArrayData(GeometrySerializer.serialize(obj))
override def deserialize(datum: Any): Geometry = {
datum match {
case values: ArrayData =>
- sedonaSerializer.deserialize(values)
+ GeometrySerializer.deserialize(values)
}
}
diff --git a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Constructors.scala b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Constructors.scala
index cdab501..ae05583 100644
--- a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Constructors.scala
+++ b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Constructors.scala
@@ -20,12 +20,12 @@ package org.apache.spark.sql.sedona_sql.expressions
import org.apache.sedona.core.enums.{FileDataSplitter, GeometryType}
import org.apache.sedona.core.formatMapper.FormatMapper
+import org.apache.sedona.sql.utils.GeometrySerializer
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.util.GenericArrayData
import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT
-import org.apache.spark.sql.sedona_sql.sedonaSerializer
import org.apache.spark.sql.types.{DataType, Decimal}
import org.apache.spark.unsafe.types.UTF8String
import org.locationtech.jts.geom.{Coordinate, GeometryFactory}
@@ -48,8 +48,7 @@ case class ST_PointFromText(inputExpressions: Seq[Expression])
var fileDataSplitter = FileDataSplitter.getFileDataSplitter(geomFormat)
var formatMapper = new FormatMapper(fileDataSplitter, false, GeometryType.POINT)
var geometry = formatMapper.readGeometry(geomString)
-
- new GenericArrayData(sedonaSerializer.serialize(geometry))
+ return new GenericArrayData(GeometrySerializer.serialize(geometry))
}
override def dataType: DataType = GeometryUDT
@@ -75,7 +74,7 @@ case class ST_PolygonFromText(inputExpressions: Seq[Expression])
var fileDataSplitter = FileDataSplitter.getFileDataSplitter(geomFormat)
var formatMapper = new FormatMapper(fileDataSplitter, false, GeometryType.POLYGON)
var geometry = formatMapper.readGeometry(geomString)
- new GenericArrayData(sedonaSerializer.serialize(geometry))
+ return new GenericArrayData(GeometrySerializer.serialize(geometry))
}
override def dataType: DataType = GeometryUDT
@@ -103,7 +102,7 @@ case class ST_LineStringFromText(inputExpressions: Seq[Expression])
var formatMapper = new FormatMapper(fileDataSplitter, false, GeometryType.LINESTRING)
var geometry = formatMapper.readGeometry(geomString)
- new GenericArrayData(sedonaSerializer.serialize(geometry))
+ return new GenericArrayData(GeometrySerializer.serialize(geometry))
}
override def dataType: DataType = GeometryUDT
@@ -128,7 +127,7 @@ case class ST_GeomFromWKT(inputExpressions: Seq[Expression])
var fileDataSplitter = FileDataSplitter.WKT
var formatMapper = new FormatMapper(fileDataSplitter, false)
var geometry = formatMapper.readGeometry(geomString)
- new GenericArrayData(sedonaSerializer.serialize(geometry))
+ return new GenericArrayData(GeometrySerializer.serialize(geometry))
}
override def dataType: DataType = GeometryUDT
@@ -153,7 +152,7 @@ case class ST_GeomFromText(inputExpressions: Seq[Expression])
var fileDataSplitter = FileDataSplitter.WKT
var formatMapper = new FormatMapper(fileDataSplitter, false)
var geometry = formatMapper.readGeometry(geomString)
- new GenericArrayData(sedonaSerializer.serialize(geometry))
+ return new GenericArrayData(GeometrySerializer.serialize(geometry))
}
override def dataType: DataType = GeometryUDT
@@ -178,7 +177,7 @@ case class ST_GeomFromWKB(inputExpressions: Seq[Expression])
var fileDataSplitter = FileDataSplitter.WKB
var formatMapper = new FormatMapper(fileDataSplitter, false)
var geometry = formatMapper.readGeometry(geomString)
- new GenericArrayData(sedonaSerializer.serialize(geometry))
+ return new GenericArrayData(GeometrySerializer.serialize(geometry))
}
override def dataType: DataType = GeometryUDT
@@ -209,7 +208,7 @@ case class ST_GeomFromGeoJSON(inputExpressions: Seq[Expression])
if (inputExpressions.length > 1) {
geometry.setUserData(generateUserData(minInputLength, inputExpressions, inputRow))
}
- new GenericArrayData(sedonaSerializer.serialize(geometry))
+ return new GenericArrayData(GeometrySerializer.serialize(geometry))
}
override def dataType: DataType = GeometryUDT
@@ -239,7 +238,7 @@ case class ST_Point(inputExpressions: Seq[Expression])
var geometryFactory = new GeometryFactory()
var geometry = geometryFactory.createPoint(new Coordinate(x, y))
- new GenericArrayData(sedonaSerializer.serialize(geometry))
+ return new GenericArrayData(GeometrySerializer.serialize(geometry))
}
override def dataType: DataType = GeometryUDT
@@ -287,7 +286,7 @@ case class ST_PolygonFromEnvelope(inputExpressions: Seq[Expression]) extends Exp
coordinates(4) = coordinates(0)
val geometryFactory = new GeometryFactory()
val polygon = geometryFactory.createPolygon(coordinates)
- new GenericArrayData(sedonaSerializer.serialize(polygon))
+ new GenericArrayData(GeometrySerializer.serialize(polygon))
}
override def dataType: DataType = GeometryUDT
@@ -302,7 +301,7 @@ trait UserDataGeneratator {
for (i <- minInputLength + 1 to inputExpressions.length - 1) {
userData = userData + "\t" + inputExpressions(i).eval(inputRow).asInstanceOf[UTF8String].toString
}
- userData
+ return userData
}
}
diff --git a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala
index 93b5173..d9e60e7 100644
--- a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala
+++ b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.sedona_sql.expressions
import org.apache.sedona.core.geometryObjects.Circle
import org.apache.sedona.core.utils.GeomUtils
+import org.apache.sedona.sql.utils.GeometrySerializer
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodegenFallback, ExprCode}
@@ -27,7 +28,6 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, Generator}
import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT
import org.apache.spark.sql.sedona_sql.expressions.implicits._
-import org.apache.spark.sql.sedona_sql.sedonaSerializer
import org.apache.spark.sql.sedona_sql.expressions.subdivide.GeometrySubDivider
import org.apache.spark.sql.types.{ArrayType, _}
import org.apache.spark.unsafe.types.UTF8String
@@ -72,11 +72,11 @@ case class ST_Distance(inputExpressions: Seq[Expression])
val leftArray = inputExpressions(0).eval(inputRow).asInstanceOf[ArrayData]
val rightArray = inputExpressions(1).eval(inputRow).asInstanceOf[ArrayData]
- val leftGeometry = sedonaSerializer.deserialize(leftArray)
+ val leftGeometry = GeometrySerializer.deserialize(leftArray)
- val rightGeometry = sedonaSerializer.deserialize(rightArray)
+ val rightGeometry = GeometrySerializer.deserialize(rightArray)
- leftGeometry.distance(rightGeometry)
+ return leftGeometry.distance(rightGeometry)
}
override def dataType = DoubleType
@@ -93,8 +93,8 @@ case class ST_ConvexHull(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 1)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
- new GenericArrayData(sedonaSerializer.serialize(geometry.convexHull()))
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ new GenericArrayData(GeometrySerializer.serialize(geometry.convexHull()))
}
override def dataType: DataType = GeometryUDT
@@ -114,7 +114,7 @@ case class ST_NPoints(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
inputExpressions.length match {
case 1 =>
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
geometry.getCoordinates.length
case _ => None
}
@@ -137,13 +137,13 @@ case class ST_Buffer(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 2)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
val buffer: Double = inputExpressions(1).eval(input) match {
case a: Decimal => a.toDouble
case a: Double => a
case a: Int => a
}
- new GenericArrayData(sedonaSerializer.serialize(geometry.buffer(buffer)))
+ new GenericArrayData(GeometrySerializer.serialize(geometry.buffer(buffer)))
}
override def dataType: DataType = GeometryUDT
@@ -163,8 +163,8 @@ case class ST_Envelope(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 1)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
- new GenericArrayData(sedonaSerializer.serialize(geometry.getEnvelope()))
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ new GenericArrayData(GeometrySerializer.serialize(geometry.getEnvelope()))
}
override def dataType: DataType = GeometryUDT
@@ -183,8 +183,8 @@ case class ST_Length(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 1)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
- geometry.getLength
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ return geometry.getLength
}
override def dataType: DataType = DoubleType
@@ -203,8 +203,8 @@ case class ST_Area(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 1)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
- geometry.getArea
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ return geometry.getArea
}
override def dataType: DataType = DoubleType
@@ -223,8 +223,8 @@ case class ST_Centroid(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 1)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
- new GenericArrayData(sedonaSerializer.serialize(geometry.getCentroid()))
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ new GenericArrayData(GeometrySerializer.serialize(geometry.getCentroid()))
}
override def dataType: DataType = GeometryUDT
@@ -244,7 +244,7 @@ case class ST_Transform(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length >= 3 && inputExpressions.length <= 4)
- val originalGeometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val originalGeometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
val sourceCRScode = CRS.decode(inputExpressions(1).eval(input).asInstanceOf[UTF8String].toString)
val targetCRScode = CRS.decode(inputExpressions(2).eval(input).asInstanceOf[UTF8String].toString)
@@ -256,7 +256,7 @@ case class ST_Transform(inputExpressions: Seq[Expression])
transform = CRS.findMathTransform(sourceCRScode, targetCRScode, false)
}
val geom = JTS.transform(originalGeometry, transform)
- new GenericArrayData(sedonaSerializer.serialize(geom))
+ new GenericArrayData(GeometrySerializer.serialize(geom))
}
override def dataType: DataType = GeometryUDT
@@ -266,7 +266,7 @@ case class ST_Transform(inputExpressions: Seq[Expression])
/**
- * Return the intersection shape of two geometries. The type is a geometry
+ * Return the intersection shape of two geometries. The return type is a geometry
*
* @param inputExpressions
*/
@@ -279,26 +279,26 @@ case class ST_Intersection(inputExpressions: Seq[Expression])
override def eval(inputRow: InternalRow): Any = {
assert(inputExpressions.length == 2)
- val leftgeometry = sedonaSerializer.deserialize(inputExpressions(0).eval(inputRow).asInstanceOf[ArrayData])
- val rightgeometry = sedonaSerializer.deserialize(inputExpressions(1).eval(inputRow).asInstanceOf[ArrayData])
+ val leftgeometry = GeometrySerializer.deserialize(inputExpressions(0).eval(inputRow).asInstanceOf[ArrayData])
+ val rightgeometry = GeometrySerializer.deserialize(inputExpressions(1).eval(inputRow).asInstanceOf[ArrayData])
val isIntersects = leftgeometry.intersects(rightgeometry)
lazy val isLeftContainsRight = leftgeometry.contains(rightgeometry)
lazy val isRightContainsLeft = rightgeometry.contains(leftgeometry)
if (!isIntersects) {
- new GenericArrayData(sedonaSerializer.serialize(emptyPolygon))
+ return new GenericArrayData(GeometrySerializer.serialize(emptyPolygon))
}
if (isIntersects && isLeftContainsRight) {
- new GenericArrayData(sedonaSerializer.serialize(rightgeometry))
+ return new GenericArrayData(GeometrySerializer.serialize(rightgeometry))
}
if (isIntersects && isRightContainsLeft) {
- new GenericArrayData(sedonaSerializer.serialize(leftgeometry))
+ return new GenericArrayData(GeometrySerializer.serialize(leftgeometry))
}
- new GenericArrayData(sedonaSerializer.serialize(leftgeometry.intersection(rightgeometry)))
+ return new GenericArrayData(GeometrySerializer.serialize(leftgeometry.intersection(rightgeometry)))
}
override def dataType: DataType = GeometryUDT
@@ -321,7 +321,7 @@ case class ST_MakeValid(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): TraversableOnce[InternalRow] = {
assert(inputExpressions.length == 2)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
val removeHoles = inputExpressions(1).eval(input).asInstanceOf[Boolean]
// in order to do flatMap on java collections(util.List[Polygon])
@@ -345,7 +345,7 @@ case class ST_MakeValid(inputExpressions: Seq[Expression])
}
val result = validGeometry.toArray.map(g => {
- val serializedGeometry = sedonaSerializer.serialize(g.asInstanceOf[Geometry])
+ val serializedGeometry = GeometrySerializer.serialize(g.asInstanceOf[Geometry])
InternalRow(new GenericArrayData(serializedGeometry))
})
@@ -369,7 +369,7 @@ case class ST_IsValid(inputExpressions: Seq[Expression])
if (inputExpressions(0).eval(input).asInstanceOf[ArrayData] == null) {
return null
}
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
val isvalidop = new IsValidOp(geometry)
isvalidop.isValid
}
@@ -391,11 +391,11 @@ case class ST_IsSimple(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 1)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
val isSimpleop = new IsSimpleOp(geometry)
- isSimpleop.isSimple
+ return isSimpleop.isSimple
}
override def dataType: DataType = BooleanType
@@ -418,7 +418,7 @@ case class ST_SimplifyPreserveTopology(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 2)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
val distanceTolerance = inputExpressions(1).eval(input) match {
case number: Decimal => number.toDouble
case number: Double => number
@@ -426,7 +426,7 @@ case class ST_SimplifyPreserveTopology(inputExpressions: Seq[Expression])
}
val simplifiedGeometry = TopologyPreservingSimplifier.simplify(geometry, distanceTolerance)
- new GenericArrayData(sedonaSerializer.serialize(simplifiedGeometry))
+ new GenericArrayData(GeometrySerializer.serialize(simplifiedGeometry))
}
override def dataType: DataType = GeometryUDT
@@ -446,10 +446,10 @@ case class ST_PrecisionReduce(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 2)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
val precisionScale = inputExpressions(1).eval(input).asInstanceOf[Int]
val precisionReduce = new GeometryPrecisionReducer(new PrecisionModel(Math.pow(10, precisionScale)))
- new GenericArrayData(sedonaSerializer.serialize(precisionReduce.reduce(geometry)))
+ new GenericArrayData(GeometrySerializer.serialize(precisionReduce.reduce(geometry)))
}
override def dataType: DataType = GeometryUDT
@@ -463,7 +463,7 @@ case class ST_AsText(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 1)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
UTF8String.fromString(geometry.toText)
}
@@ -495,7 +495,7 @@ case class ST_GeometryType(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 1)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
UTF8String.fromString("ST_" + geometry.getGeometryType)
}
@@ -506,7 +506,7 @@ case class ST_GeometryType(inputExpressions: Seq[Expression])
/**
* Returns a LineString formed by sewing together the constituent line work of a MULTILINESTRING.
- * Only works for MultiLineString. Using other geometry will GEOMETRYCOLLECTION EMPTY
+ * Only works for MultiLineString. Using other geometry will return GEOMETRYCOLLECTION EMPTY
* If the MultiLineString is can't be merged, the original multilinestring is returned
*
* @param inputExpressions Geometry
@@ -522,7 +522,7 @@ case class ST_LineMerge(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 1)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
val merger = new LineMerger()
@@ -537,13 +537,13 @@ case class ST_LineMerge(inputExpressions: Seq[Expression])
// If the merger was able to join the lines, there will be only one element
merger.getMergedLineStrings().iterator().next().asInstanceOf[Geometry]
} else {
- // if the merger couldn't join the lines, it will contain the individual lines, so the input
+ // if the merger couldn't join the lines, it will contain the individual lines, so return the input
geometry
}
}
case _ => emptyGeometry
}
- new GenericArrayData(sedonaSerializer.serialize(output))
+ new GenericArrayData(GeometrySerializer.serialize(output))
}
override def dataType: DataType = GeometryUDT
@@ -1082,8 +1082,8 @@ case class ST_IsRing(inputExpressions: Seq[Expression])
}
/**
- * Returns the number of Geometries. If geometry is a GEOMETRYCOLLECTION (or MULTI*) the number of geometries,
- * for single geometries will 1
+ * Returns the number of Geometries. If geometry is a GEOMETRYCOLLECTION (or MULTI*) return the number of geometries,
+ * for single geometries will return 1
*
* This method implements the SQL/MM specification. SQL-MM 3: 9.1.4
*
@@ -1095,7 +1095,7 @@ case class ST_NumGeometries(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 1)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
geometry.getNumGeometries()
}
@@ -1115,7 +1115,7 @@ case class ST_FlipCoordinates(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length == 1)
- val geometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val geometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
GeomUtils.flipCoordinates(geometry)
geometry.toGenericArrayData
}
diff --git a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Predicates.scala b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Predicates.scala
index d49ce65..959b8de 100644
--- a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Predicates.scala
+++ b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Predicates.scala
@@ -18,11 +18,11 @@
*/
package org.apache.spark.sql.sedona_sql.expressions
+import org.apache.sedona.sql.utils.GeometrySerializer
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.util.ArrayData
-import org.apache.spark.sql.sedona_sql.sedonaSerializer
import org.apache.spark.sql.types.BooleanType
abstract class ST_Predicate extends Expression
@@ -48,11 +48,11 @@ case class ST_Contains(inputExpressions: Seq[Expression])
val leftArray = inputExpressions(0).eval(inputRow).asInstanceOf[ArrayData]
val rightArray = inputExpressions(1).eval(inputRow).asInstanceOf[ArrayData]
- val leftGeometry = sedonaSerializer.deserialize(leftArray)
+ val leftGeometry = GeometrySerializer.deserialize(leftArray)
- val rightGeometry = sedonaSerializer.deserialize(rightArray)
+ val rightGeometry = GeometrySerializer.deserialize(rightArray)
- leftGeometry.covers(rightGeometry)
+ return leftGeometry.covers(rightGeometry)
}
override def dataType = BooleanType
@@ -78,11 +78,11 @@ case class ST_Intersects(inputExpressions: Seq[Expression])
val leftArray = inputExpressions(0).eval(inputRow).asInstanceOf[ArrayData]
val rightArray = inputExpressions(1).eval(inputRow).asInstanceOf[ArrayData]
- val leftGeometry = sedonaSerializer.deserialize(leftArray)
+ val leftGeometry = GeometrySerializer.deserialize(leftArray)
- val rightGeometry = sedonaSerializer.deserialize(rightArray)
+ val rightGeometry = GeometrySerializer.deserialize(rightArray)
- leftGeometry.intersects(rightGeometry)
+ return leftGeometry.intersects(rightGeometry)
}
override def dataType = BooleanType
@@ -108,11 +108,11 @@ case class ST_Within(inputExpressions: Seq[Expression])
val leftArray = inputExpressions(0).eval(inputRow).asInstanceOf[ArrayData]
val rightArray = inputExpressions(1).eval(inputRow).asInstanceOf[ArrayData]
- val leftGeometry = sedonaSerializer.deserialize(leftArray)
+ val leftGeometry = GeometrySerializer.deserialize(leftArray)
- val rightGeometry = sedonaSerializer.deserialize(rightArray)
+ val rightGeometry = GeometrySerializer.deserialize(rightArray)
- leftGeometry.coveredBy(rightGeometry)
+ return leftGeometry.coveredBy(rightGeometry)
}
override def dataType = BooleanType
@@ -138,11 +138,11 @@ case class ST_Crosses(inputExpressions: Seq[Expression])
val leftArray = inputExpressions(0).eval(inputRow).asInstanceOf[ArrayData]
val rightArray = inputExpressions(1).eval(inputRow).asInstanceOf[ArrayData]
- val leftGeometry = sedonaSerializer.deserialize(leftArray)
+ val leftGeometry = GeometrySerializer.deserialize(leftArray)
- val rightGeometry = sedonaSerializer.deserialize(rightArray)
+ val rightGeometry = GeometrySerializer.deserialize(rightArray)
- leftGeometry.crosses(rightGeometry)
+ return leftGeometry.crosses(rightGeometry)
}
override def dataType = BooleanType
@@ -169,11 +169,11 @@ case class ST_Overlaps(inputExpressions: Seq[Expression])
val leftArray = inputExpressions(0).eval(inputRow).asInstanceOf[ArrayData]
val rightArray = inputExpressions(1).eval(inputRow).asInstanceOf[ArrayData]
- val leftGeometry = sedonaSerializer.deserialize(leftArray)
+ val leftGeometry = GeometrySerializer.deserialize(leftArray)
- val rightGeometry = sedonaSerializer.deserialize(rightArray)
+ val rightGeometry = GeometrySerializer.deserialize(rightArray)
- leftGeometry.overlaps(rightGeometry)
+ return leftGeometry.overlaps(rightGeometry)
}
override def dataType = BooleanType
@@ -199,11 +199,11 @@ case class ST_Touches(inputExpressions: Seq[Expression])
val leftArray = inputExpressions(0).eval(inputRow).asInstanceOf[ArrayData]
val rightArray = inputExpressions(1).eval(inputRow).asInstanceOf[ArrayData]
- val leftGeometry = sedonaSerializer.deserialize(leftArray)
+ val leftGeometry = GeometrySerializer.deserialize(leftArray)
- val rightGeometry = sedonaSerializer.deserialize(rightArray)
+ val rightGeometry = GeometrySerializer.deserialize(rightArray)
- leftGeometry.touches(rightGeometry)
+ return leftGeometry.touches(rightGeometry)
}
override def dataType = BooleanType
@@ -230,16 +230,16 @@ case class ST_Equals(inputExpressions: Seq[Expression])
val rightArray = inputExpressions(1).eval(inputRow).asInstanceOf[ArrayData]
- val leftGeometry = sedonaSerializer.deserialize(leftArray)
+ val leftGeometry = GeometrySerializer.deserialize(leftArray)
- val rightGeometry = sedonaSerializer.deserialize(rightArray)
+ val rightGeometry = GeometrySerializer.deserialize(rightArray)
// Returns GeometryCollection object
val symDifference = leftGeometry.symDifference(rightGeometry)
val isEqual = symDifference.isEmpty
- isEqual
+ return isEqual
}
override def dataType = BooleanType
diff --git a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/implicits.scala b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/implicits.scala
index cb4548b..e31efe3 100644
--- a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/implicits.scala
+++ b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/implicits.scala
@@ -19,10 +19,10 @@
package org.apache.spark.sql.sedona_sql.expressions
+import org.apache.sedona.sql.utils.GeometrySerializer
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
-import org.apache.spark.sql.sedona_sql.sedonaSerializer
import org.locationtech.jts.geom.{Geometry, GeometryFactory, Point}
object implicits {
@@ -30,7 +30,7 @@ object implicits {
implicit class InputExpressionEnhancer(inputExpression: Expression) {
def toGeometry(input: InternalRow): Geometry = {
inputExpression.eval(input).asInstanceOf[ArrayData] match {
- case arrData: ArrayData => sedonaSerializer.deserialize(arrData)
+ case arrData: ArrayData => GeometrySerializer.deserialize(arrData)
case _ => null
}
}
@@ -58,7 +58,7 @@ object implicits {
implicit class ArrayDataEnhancer(arrayData: ArrayData) {
def toGeometry: Geometry = {
arrayData match {
- case arrData: ArrayData => sedonaSerializer.deserialize(arrData)
+ case arrData: ArrayData => GeometrySerializer.deserialize(arrData)
case _ => null
}
}
@@ -68,7 +68,7 @@ object implicits {
private val geometryFactory = new GeometryFactory()
def toGenericArrayData: GenericArrayData =
- new GenericArrayData(sedonaSerializer.serialize(geom))
+ new GenericArrayData(GeometrySerializer.serialize(geom))
def getPoints: Array[Point] =
geom.getCoordinates.map(coordinate => geometryFactory.createPoint(coordinate))
diff --git a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/package.scala b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/package.scala
deleted file mode 100644
index 8871eea..0000000
--- a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/package.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.spark.sql
-
-import org.apache.sedona.core.serde.GeometrySerde
-import org.apache.sedona.core.utils.SedonaConf
-import org.apache.sedona.sql.serde.SedonaSerializer
-import org.apache.spark.{SparkContext, SparkException}
-
-package object sedona_sql {
-
- private val sparkContext = SparkContext.getActive.getOrElse(None)
- private val sedonaConf = sparkContext match {
- case sc: SparkContext => new SedonaConf(sc.conf)
- case None => throw new SparkException("There is no active SparkContext. Hence, cannot create SedonaSerializer")
- }
-
- val userSerializerType = sedonaConf.getSerializerType
- val sedonaSerializer: SedonaSerializer = SedonaSerializer(userSerializerType)
-
-}
diff --git a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/DistanceJoinExec.scala b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/DistanceJoinExec.scala
index cdf737e..a921979 100644
--- a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/DistanceJoinExec.scala
+++ b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/DistanceJoinExec.scala
@@ -20,12 +20,12 @@ package org.apache.spark.sql.sedona_sql.strategy.join
import org.apache.sedona.core.geometryObjects.Circle
import org.apache.sedona.core.spatialRDD.SpatialRDD
+import org.apache.sedona.sql.utils.GeometrySerializer
import org.apache.spark.internal.Logging
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.expressions.{BindReferences, Expression, UnsafeRow}
import org.apache.spark.sql.catalyst.util.ArrayData
import org.apache.spark.sql.execution.{BinaryExecNode, SparkPlan}
-import org.apache.spark.sql.sedona_sql.sedonaSerializer
import org.locationtech.jts.geom.Geometry
// ST_Distance(left, right) <= radius
@@ -55,7 +55,7 @@ case class DistanceJoinExec(left: SparkPlan,
spatialRdd.setRawSpatialRDD(
rdd
.map { x => {
- val shape = sedonaSerializer.deserialize(shapeExpression.eval(x).asInstanceOf[ArrayData])
+ val shape = GeometrySerializer.deserialize(shapeExpression.eval(x).asInstanceOf[ArrayData])
val circle = new Circle(shape, boundRadius.eval(x).asInstanceOf[Double])
circle.setUserData(x.copy)
circle.asInstanceOf[Geometry]
diff --git a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/TraitJoinQueryBase.scala b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/TraitJoinQueryBase.scala
index 26786e3..3b7a0e1 100644
--- a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/TraitJoinQueryBase.scala
+++ b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/TraitJoinQueryBase.scala
@@ -21,11 +21,11 @@ package org.apache.spark.sql.sedona_sql.strategy.join
import org.apache.sedona.core.geometryObjects.Circle
import org.apache.sedona.core.spatialRDD.SpatialRDD
import org.apache.sedona.core.utils.SedonaConf
+import org.apache.sedona.sql.utils.GeometrySerializer
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.expressions.{Expression, UnsafeRow}
import org.apache.spark.sql.catalyst.util.ArrayData
import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.sedona_sql.sedonaSerializer
import org.locationtech.jts.geom.Geometry
trait TraitJoinQueryBase {
@@ -42,7 +42,7 @@ trait TraitJoinQueryBase {
spatialRdd.setRawSpatialRDD(
rdd
.map { x => {
- val shape = sedonaSerializer.deserialize(shapeExpression.eval(x).asInstanceOf[ArrayData])
+ val shape = GeometrySerializer.deserialize(shapeExpression.eval(x).asInstanceOf[ArrayData])
//logInfo(shape.toString)
shape.setUserData(x.copy)
shape
@@ -57,7 +57,7 @@ trait TraitJoinQueryBase {
spatialRdd.setRawSpatialRDD(
rdd
.map { x => {
- val shape = sedonaSerializer.deserialize(shapeExpression.eval(x).asInstanceOf[ArrayData])
+ val shape = GeometrySerializer.deserialize(shapeExpression.eval(x).asInstanceOf[ArrayData])
val circle = new Circle(shape, boundRadius.eval(x).asInstanceOf[Double])
circle.setUserData(x.copy)
circle.asInstanceOf[Geometry]
diff --git a/sql/src/test/java/org/apache/sedona/sql/TestBaseWKBSerdeJava.java b/sql/src/test/java/org/apache/sedona/sql/TestBaseWKBSerdeJava.java
deleted file mode 100644
index abffcde..0000000
--- a/sql/src/test/java/org/apache/sedona/sql/TestBaseWKBSerdeJava.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.sql;
-
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.sedona.sql.utils.SedonaSQLRegistrator;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SparkSession;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-/**
- * This TestBase class using the WKB Serializer
- */
-public class TestBaseWKBSerdeJava {
-
- public static String resourceFolder = System.getProperty("user.dir") + "/../core/src/test/resources/";
- public static String mixedWktGeometryInputLocation = resourceFolder + "county_small.tsv";
- public static String mixedWkbGeometryInputLocation = resourceFolder + "county_small_wkb.tsv";
- public static String csvPointInputLocation = resourceFolder + "arealm.csv";
- public static String shapefileInputLocation = resourceFolder + "shapefiles/polygon";
- protected static SparkConf conf;
- protected static JavaSparkContext sc;
- protected static SparkSession sparkSession;
-
- /**
- * Once executed before all using the WKB serde
- */
- @BeforeClass
- public static void onceExecutedBeforeAll()
- {
- conf = new SparkConf().setAppName("adapterTestJava").setMaster("local[2]");
- conf.set("spark.serializer", org.apache.spark.serializer.KryoSerializer.class.getName());
- conf.set("sedona.serializer.type", "wkb");
-
- sc = new JavaSparkContext(conf);
- sparkSession = new SparkSession(sc.sc());
- Logger.getLogger("org").setLevel(Level.WARN);
- Logger.getLogger("akka").setLevel(Level.WARN);
- SedonaSQLRegistrator.registerAll(sparkSession.sqlContext());
- }
-
- /**
- * Tear down.
- */
- @AfterClass
- public static void TearDown()
- {
- SedonaSQLRegistrator.dropAll(sparkSession);
- sparkSession.stop();
- }
-}
diff --git a/sql/src/test/java/org/apache/sedona/sql/WKBGeometrySerializerTestJava.java b/sql/src/test/java/org/apache/sedona/sql/WKBGeometrySerializerTestJava.java
deleted file mode 100644
index 9d2bf34..0000000
--- a/sql/src/test/java/org/apache/sedona/sql/WKBGeometrySerializerTestJava.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.sql;
-
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
-import org.junit.Test;
-import org.locationtech.jts.geom.Geometry;
-
-import static org.junit.Assert.assertEquals;
-
-public class WKBGeometrySerializerTestJava extends TestBaseWKBSerdeJava
-{
-
- @Test()
- public void testST_PointUsingWKBSerde()
- {
- Dataset<Row> pointDf = sparkSession.sql("SELECT ST_Point(double(1.2345), 2.3456)");
- assertEquals("simple ST_Point test using the wkb serializer", pointDf.count() , 1);
- }
-
- @Test()
- public void testMultipolygonWithHoleBugUsingWKBSerde()
- {
- String multiPolygon = "'MULTIPOLYGON (((-97.143362 27.84948, -97.14051 27.849375, -97.13742 27.849375, -97.13647 27.851056, -97.136945 27.853788, -97.138728 27.855784, -97.141223 27.853158, -97.143362 27.84948))," +
- "((-97.131954 27.894443, -97.131716 27.896018, -97.1212014 27.8937854, -97.113415 27.892132, -97.110206 27.890662, -97.110206 27.889191, -97.1104204 27.8890963, -97.112107 27.888351, -97.114247 27.886985, -97.11591 27.88583, -97.11698 27.885515, -97.1173397 27.8855439, -97.118287 27.88562, -97.1180218 27.8875761, -97.117931 27.888246, -97.118287 27.890662, -97.1187136 27.8907652, -97.125269 27.8923509, -97.129577 27.893393, -97.131954 27.894443))," +
- "((-97.150493 27.874905, -97.150849 27.875851, -97.149067 27.877531, -97.146927 27.878267, -97.14467 27.880893, -97.143362 27.881523, -97.142055 27.881103, -97.14158 27.879527, -97.142412 27.878582, -97.143719 27.878372, -97.145383 27.876586, -97.147403 27.875641, -97.149423 27.874905, -97.150493 27.874905))," +
- "((-97.277888 27.915016, -97.280673 27.916797, -97.277151 27.918885, -97.270902 27.924911, -97.268651 27.926005, -97.269181 27.927081, -97.266121 27.927604, -97.257 27.927545, -97.231476 27.927514, -97.230477 27.953057, -97.229337 27.954497, -97.227041 27.955697, -97.226871 27.956842, -97.227509 27.984419, -97.240412 27.991214, -97.249501 27.99119, -97.249808 28.00156, -97.249403 28.008394, -97.249352 28.015777, -97.246465 28.020607, -97.217976 28.006301, -97.214474 28.011763 [...]
- Dataset<Row> wkt = sparkSession.sql(String.format("SELECT ST_GeomFromWkt(%s)", multiPolygon));
- Geometry geom = (Geometry) wkt.first().get(0);
- assertEquals("ST_GeomFromWKT multipolygon read as polygon bug - using the wkb serializer solve that",
- geom.getGeometryType() , "MultiPolygon");
- }
-}
diff --git a/sql/src/test/java/org/apache/sedona/sql/adapterTestJava.java b/sql/src/test/java/org/apache/sedona/sql/adapterTestJava.java
index 7324a4d..793cda5 100644
--- a/sql/src/test/java/org/apache/sedona/sql/adapterTestJava.java
+++ b/sql/src/test/java/org/apache/sedona/sql/adapterTestJava.java
@@ -19,20 +19,67 @@
package org.apache.sedona.sql;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
import org.apache.sedona.core.enums.GridType;
import org.apache.sedona.core.enums.IndexType;
import org.apache.sedona.core.formatMapper.shapefileParser.ShapefileReader;
+import org.apache.sedona.core.serde.SedonaKryoRegistrator;
import org.apache.sedona.core.spatialOperator.JoinQuery;
import org.apache.sedona.core.spatialRDD.CircleRDD;
import org.apache.sedona.core.spatialRDD.SpatialRDD;
import org.apache.sedona.sql.utils.Adapter;
+import org.apache.sedona.sql.utils.SedonaSQLRegistrator;
+import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
-public class adapterTestJava extends TestBaseWKBSerdeJava {
+import java.io.Serializable;
+
+public class adapterTestJava
+ implements Serializable
+{
+ public static String resourceFolder = System.getProperty("user.dir") + "/../core/src/test/resources/";
+ public static String mixedWktGeometryInputLocation = resourceFolder + "county_small.tsv";
+ public static String mixedWkbGeometryInputLocation = resourceFolder + "county_small_wkb.tsv";
+ public static String csvPointInputLocation = resourceFolder + "arealm.csv";
+ public static String shapefileInputLocation = resourceFolder + "shapefiles/polygon";
+ protected static SparkConf conf;
+ protected static JavaSparkContext sc;
+ protected static SparkSession sparkSession;
+
+ /**
+ * Once executed before all.
+ */
+ @BeforeClass
+ public static void onceExecutedBeforeAll()
+ {
+ conf = new SparkConf().setAppName("adapterTestJava").setMaster("local[2]");
+ conf.set("spark.serializer", org.apache.spark.serializer.KryoSerializer.class.getName());
+ conf.set("spark.kryo.registrator", SedonaKryoRegistrator.class.getName());
+
+ sc = new JavaSparkContext(conf);
+ sparkSession = new SparkSession(sc.sc());
+ Logger.getLogger("org").setLevel(Level.WARN);
+ Logger.getLogger("akka").setLevel(Level.WARN);
+ SedonaSQLRegistrator.registerAll(sparkSession.sqlContext());
+ }
+
+ /**
+ * Tear down.
+ */
+ @AfterClass
+ public static void TearDown()
+ {
+ SedonaSQLRegistrator.dropAll(sparkSession);
+ sparkSession.stop();
+ }
@Test
public void testReadCsv()
diff --git a/sql/src/test/scala/org/apache/sedona/sql/adapterTestScala.scala b/sql/src/test/scala/org/apache/sedona/sql/adapterTestScala.scala
index 25a6572..3261715 100644
--- a/sql/src/test/scala/org/apache/sedona/sql/adapterTestScala.scala
+++ b/sql/src/test/scala/org/apache/sedona/sql/adapterTestScala.scala
@@ -22,17 +22,12 @@ package org.apache.sedona.sql
import org.apache.sedona.core.enums.{FileDataSplitter, GridType, IndexType}
import org.apache.sedona.core.formatMapper.EarthdataHDFPointMapper
import org.apache.sedona.core.formatMapper.shapefileParser.ShapefileReader
-import org.apache.sedona.core.serde.WKB.WKBGeometrySerde
-import org.apache.sedona.core.serde.shape.ShapeGeometrySerde
import org.apache.sedona.core.spatialOperator.JoinQuery
import org.apache.sedona.core.spatialRDD.{CircleRDD, PointRDD, PolygonRDD}
-import org.apache.sedona.sql.serde.SedonaSerializer
import org.apache.sedona.sql.utils.Adapter
-import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT
import org.apache.spark.storage.StorageLevel
import org.locationtech.jts.geom.Point
-import org.locationtech.jts.io.{WKBWriter, WKTReader}
import org.scalatest.GivenWhenThen
class adapterTestScala extends TestBaseScala with GivenWhenThen{
diff --git a/sql/src/test/scala/org/apache/sedona/sql/constructorTestScala.scala b/sql/src/test/scala/org/apache/sedona/sql/constructorTestScala.scala
index bf7aadb..0b6c7bd 100644
--- a/sql/src/test/scala/org/apache/sedona/sql/constructorTestScala.scala
+++ b/sql/src/test/scala/org/apache/sedona/sql/constructorTestScala.scala
@@ -22,6 +22,7 @@ package org.apache.sedona.sql
import org.apache.sedona.core.formatMapper.GeoJsonReader
import org.apache.sedona.core.formatMapper.shapefileParser.ShapefileReader
import org.apache.sedona.sql.utils.Adapter
+import org.locationtech.jts.geom.Geometry
class constructorTestScala extends TestBaseScala {
@@ -68,6 +69,19 @@ class constructorTestScala extends TestBaseScala {
assert(polygonDf.count() == 100)
}
+ it("Passed ST_GeomFromWKT multipolygon read as polygon bug") {
+ val multipolygon =
+ """'MULTIPOLYGON (((-97.143362 27.84948, -97.14051 27.849375, -97.13742 27.849375, -97.13647 27.851056, -97.136945 27.853788, -97.138728 27.855784, -97.141223 27.853158, -97.143362 27.84948)),
+ ((-97.131954 27.894443, -97.131716 27.896018, -97.1212014 27.8937854, -97.113415 27.892132, -97.110206 27.890662, -97.110206 27.889191, -97.1104204 27.8890963, -97.112107 27.888351, -97.114247 27.886985, -97.11591 27.88583, -97.11698 27.885515, -97.1173397 27.8855439, -97.118287 27.88562, -97.1180218 27.8875761, -97.117931 27.888246, -97.118287 27.890662, -97.1187136 27.8907652, -97.125269 27.8923509, -97.129577 27.893393, -97.131954 27.894443)),
+ ((-97.150493 27.874905, -97.150849 27.875851, -97.149067 27.877531, -97.146927 27.878267, -97.14467 27.880893, -97.143362 27.881523, -97.142055 27.881103, -97.14158 27.879527, -97.142412 27.878582, -97.143719 27.878372, -97.145383 27.876586, -97.147403 27.875641, -97.149423 27.874905, -97.150493 27.874905)),
+ ((-97.277888 27.915016, -97.280673 27.916797, -97.277151 27.918885, -97.270902 27.924911, -97.268651 27.926005, -97.269181 27.927081, -97.266121 27.927604, -97.257 27.927545, -97.231476 27.927514, -97.230477 27.953057, -97.229337 27.954497, -97.227041 27.955697, -97.226871 27.956842, -97.227509 27.984419, -97.240412 27.991214, -97.249501 27.99119, -97.249808 28.00156, -97.249403 28.008394, -97.249352 28.015777, -97.246465 28.020607, -97.217976 28.006301, -97.214474 28.011763, [...]
+ val wkt = sparkSession.sql(
+ s"""
+ |SELECT ST_GeomFromWkt($multipolygon)
+ |""".stripMargin)
+ assert(wkt.first().getAs[Geometry](0).getGeometryType === "MultiPolygon")
+ }
+
it("Passed ST_GeomFromWKB") {
var polygonWkbDf = sparkSession.read.format("csv").option("delimiter", "\t").option("header", "false").load(mixedWkbGeometryInputLocation)
polygonWkbDf.createOrReplaceTempView("polygontable")
diff --git a/sql/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/src/test/scala/org/apache/sedona/sql/functionTestScala.scala
index df7a6f8..3be2324 100644
--- a/sql/src/test/scala/org/apache/sedona/sql/functionTestScala.scala
+++ b/sql/src/test/scala/org/apache/sedona/sql/functionTestScala.scala
@@ -126,7 +126,7 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample
val testtable = sparkSession.sql("select ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as b")
testtable.createOrReplaceTempView("testtable")
val intersec = sparkSession.sql("select ST_Intersection(a,b) from testtable")
- assert(intersec.take(1)(0).get(0).asInstanceOf[Geometry].toText.equals("POLYGON ((3 3, 2 2, 2 3, 3 3))"))
+ assert(intersec.take(1)(0).get(0).asInstanceOf[Geometry].toText.equals("POLYGON ((2 2, 2 3, 3 3, 2 2))"))
}
it("Passed ST_Intersection - intersects but right contains left") {
@@ -139,10 +139,10 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample
it("Passed ST_Intersection - not intersects") {
- val testtable = sparkSession.sql("select ST_GeomFromWKT('POLYGON((40 21, 40 25, 35 20, 40 21))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b")
+ val testtable = sparkSession.sql("select ST_GeomFromWKT('POLYGON((40 21, 40 22, 40 23, 40 21))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b")
testtable.createOrReplaceTempView("testtable")
val intersect = sparkSession.sql("select ST_Intersection(a,b) from testtable")
- assert(intersect.take(1)(0).get(0).asInstanceOf[Geometry].toText.equals("MULTIPOLYGON EMPTY"))
+ assert(intersect.take(1)(0).get(0).asInstanceOf[Geometry].toText.equals("POLYGON EMPTY"))
}
it("Passed ST_IsValid") {
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala b/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala
index daeb550..2c486ad 100644
--- a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala
+++ b/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala
@@ -21,6 +21,7 @@ package org.apache.spark.sql.sedona_viz.expressions
import com.esotericsoftware.kryo.Kryo
import com.esotericsoftware.kryo.io.Output
import org.apache.commons.io.output.ByteArrayOutputStream
+import org.apache.sedona.sql.utils.GeometrySerializer
import org.apache.sedona.viz.core.Serde.PixelSerializer
import org.apache.sedona.viz.utils.{ColorizeOption, RasterizationUtils}
import org.apache.spark.internal.Logging
@@ -29,7 +30,6 @@ import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
import org.apache.spark.sql.sedona_viz.UDT.PixelUDT
-import org.apache.spark.sql.sedona_sql.sedonaSerializer
import org.apache.spark.sql.types.{ArrayType, DataType}
import org.locationtech.jts.geom._
@@ -40,10 +40,10 @@ case class ST_Pixelize(inputExpressions: Seq[Expression])
override def eval(input: InternalRow): Any = {
assert(inputExpressions.length <= 5)
- val inputGeometry = sedonaSerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
+ val inputGeometry = GeometrySerializer.deserialize(inputExpressions(0).eval(input).asInstanceOf[ArrayData])
val resolutionX = inputExpressions(1).eval(input).asInstanceOf[Integer]
val resolutionY = inputExpressions(2).eval(input).asInstanceOf[Integer]
- val boundary = sedonaSerializer.deserialize(inputExpressions(3).eval(input).asInstanceOf[ArrayData]).getEnvelopeInternal
+ val boundary = GeometrySerializer.deserialize(inputExpressions(3).eval(input).asInstanceOf[ArrayData]).getEnvelopeInternal
val reverseCoordinate = false
val pixels = inputGeometry match {
case geometry: LineString => {