You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2022/12/29 17:42:12 UTC
[incubator-sedona] branch master updated: [SEDONA-207] Implemented a new geometry serde for supporting Z/M dimensions and better performance (#739)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git
The following commit(s) were added to refs/heads/master by this push:
new c4e649ad [SEDONA-207] Implemented a new geometry serde for supporting Z/M dimensions and better performance (#739)
c4e649ad is described below
commit c4e649adeb7f85fe24316ff135d7e9b3c44b1f3b
Author: Kristin Cowalcijk <mo...@yeah.net>
AuthorDate: Fri Dec 30 01:42:06 2022 +0800
[SEDONA-207] Implemented a new geometry serde for supporting Z/M dimensions and better performance (#739)
---
.../geometrySerde/ByteBufferGeometryBuffer.java | 264 +++++++++
.../common/geometrySerde/CoordinateType.java | 56 ++
.../common/geometrySerde/GeometryBuffer.java | 59 ++
.../geometrySerde/GeometryBufferFactory.java | 66 +++
.../common/geometrySerde/GeometrySerializer.java | 594 +++++++++++++++++++++
.../common/geometrySerde/UnsafeGeometryBuffer.java | 300 +++++++++++
.../common/geometrySerde/GeometryBufferTest.java | 281 ++++++++++
.../geometrySerde/GeometryCollectionSerdeTest.java | 124 +++++
.../common/geometrySerde/LineStringSerdeTest.java | 149 ++++++
.../geometrySerde/MultiLineStringSerdeTest.java | 68 +++
.../common/geometrySerde/MultiPointSerdeTest.java | 113 ++++
.../geometrySerde/MultiPolygonSerdeTest.java | 89 +++
.../common/geometrySerde/PointSerdeTest.java | 112 ++++
.../common/geometrySerde/PolygonSerdeTest.java | 101 ++++
.../sedona/core/geometryObjects/GeometrySerde.java | 43 +-
python/sedona/sql/types.py | 10 +-
python/sedona/utils/geometry_serde.py | 490 +++++++++++++++++
python/tests/serialization/test_deserializers.py | 17 +-
python/tests/utils/test_geometry_serde.py | 129 +++++
.../sedona/sql/utils/GeometrySerializer.scala | 10 +-
.../datasources/parquet/GeoDataSourceUtils.scala | 46 ++
.../datasources/parquet/GeoParquetFileFormat.scala | 117 +++-
.../parquet/GeoParquetReadSupport.scala | 7 +-
.../parquet/GeoParquetRowConverter.scala | 15 +-
.../parquet/GeoParquetWriteSupport.scala | 483 +++++++++++++++++
25 files changed, 3669 insertions(+), 74 deletions(-)
diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/ByteBufferGeometryBuffer.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/ByteBufferGeometryBuffer.java
new file mode 100644
index 00000000..a0881d99
--- /dev/null
+++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/ByteBufferGeometryBuffer.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.CoordinateSequence;
+import org.locationtech.jts.geom.CoordinateXY;
+import org.locationtech.jts.geom.CoordinateXYM;
+import org.locationtech.jts.geom.CoordinateXYZM;
+import org.locationtech.jts.geom.impl.CoordinateArraySequence;
+
+class ByteBufferGeometryBuffer implements GeometryBuffer {
+ private CoordinateType coordinateType = CoordinateType.XY;
+ private final ByteBuffer byteBuffer;
+ private int markOffset = 0;
+
+ public ByteBufferGeometryBuffer(int bufferSize) {
+ byteBuffer = ByteBuffer.allocate(bufferSize);
+ byteBuffer.order(ByteOrder.nativeOrder());
+ }
+
+ public ByteBufferGeometryBuffer(byte[] bytes) {
+ byteBuffer = ByteBuffer.wrap(bytes).order(ByteOrder.nativeOrder());
+ }
+
+ public ByteBufferGeometryBuffer(ByteBuffer byteBuffer) {
+ this.byteBuffer = byteBuffer.order(ByteOrder.nativeOrder());
+ }
+
+ @Override
+ public CoordinateType getCoordinateType() {
+ return coordinateType;
+ }
+
+ @Override
+ public void setCoordinateType(CoordinateType coordinateType) {
+ this.coordinateType = coordinateType;
+ }
+
+ @Override
+ public int getLength() {
+ return byteBuffer.capacity();
+ }
+
+ @Override
+ public void mark(int offset) {
+ markOffset = offset;
+ }
+
+ @Override
+ public int getMark() {
+ return markOffset;
+ }
+
+ @Override
+ public void putByte(int offset, byte value) {
+ byteBuffer.put(offset, value);
+ }
+
+ @Override
+ public byte getByte(int offset) {
+ return byteBuffer.get(offset);
+ }
+
+ @Override
+ public void putBytes(int offset, byte[] bytes) {
+ byteBuffer.position(offset);
+ byteBuffer.put(bytes, 0, bytes.length);
+ }
+
+ @Override
+ public void getBytes(byte[] bytes, int offset, int length) {
+ byteBuffer.position(offset);
+ byteBuffer.get(bytes, 0, length);
+ }
+
+ @Override
+ public void putInt(int offset, int value) {
+ byteBuffer.putInt(offset, value);
+ }
+
+ @Override
+ public int getInt(int offset) {
+ return byteBuffer.getInt(offset);
+ }
+
+ @Override
+ public void putCoordinate(int offset, Coordinate coordinate) {
+ byteBuffer.putDouble(offset, coordinate.x);
+ byteBuffer.putDouble(offset + 8, coordinate.y);
+ offset += 16;
+ if (coordinateType.hasZ) {
+ byteBuffer.putDouble(offset, coordinate.getZ());
+ offset += 8;
+ }
+ if (coordinateType.hasM) {
+ byteBuffer.putDouble(offset, coordinate.getM());
+ }
+ }
+
+ @Override
+ public CoordinateSequence getCoordinate(int offset) {
+ double x = byteBuffer.getDouble(offset);
+ double y = byteBuffer.getDouble(offset + 8);
+ double z;
+ double m;
+ Coordinate[] coordinates = new Coordinate[1];
+ switch (coordinateType) {
+ case XY:
+ coordinates[0] = new CoordinateXY(x, y);
+ return new CoordinateArraySequence(coordinates, 2, 0);
+ case XYZ:
+ z = byteBuffer.getDouble(offset + 16);
+ coordinates[0] = new Coordinate(x, y, z);
+ return new CoordinateArraySequence(coordinates, 3, 0);
+ case XYM:
+ m = byteBuffer.getDouble(offset + 16);
+ coordinates[0] = new CoordinateXYM(x, y, m);
+ return new CoordinateArraySequence(coordinates, 3, 1);
+ case XYZM:
+ z = byteBuffer.getDouble(offset + 16);
+ m = byteBuffer.getDouble(offset + 24);
+ coordinates[0] = new CoordinateXYZM(x, y, z, m);
+ return new CoordinateArraySequence(coordinates, 4, 1);
+ default:
+ throw new IllegalStateException("coordinateType was not configured properly");
+ }
+ }
+
+ @Override
+ public void putCoordinates(int offset, CoordinateSequence coordinates) {
+ int numCoordinates = coordinates.size();
+ switch (coordinateType) {
+ case XY:
+ for (int k = 0; k < numCoordinates; k++) {
+ Coordinate coord = coordinates.getCoordinate(k);
+ byteBuffer.putDouble(offset, coord.x);
+ byteBuffer.putDouble(offset + 8, coord.y);
+ offset += 16;
+ }
+ break;
+ case XYZ:
+ for (int k = 0; k < numCoordinates; k++) {
+ Coordinate coord = coordinates.getCoordinate(k);
+ byteBuffer.putDouble(offset, coord.x);
+ byteBuffer.putDouble(offset + 8, coord.y);
+ byteBuffer.putDouble(offset + 16, coord.getZ());
+ offset += 24;
+ }
+ break;
+ case XYM:
+ for (int k = 0; k < numCoordinates; k++) {
+ Coordinate coord = coordinates.getCoordinate(k);
+ byteBuffer.putDouble(offset, coord.x);
+ byteBuffer.putDouble(offset + 8, coord.y);
+ byteBuffer.putDouble(offset + 16, coord.getM());
+ offset += 24;
+ }
+ break;
+ case XYZM:
+ for (int k = 0; k < numCoordinates; k++) {
+ Coordinate coord = coordinates.getCoordinate(k);
+ byteBuffer.putDouble(offset, coord.x);
+ byteBuffer.putDouble(offset + 8, coord.y);
+ byteBuffer.putDouble(offset + 16, coord.getZ());
+ byteBuffer.putDouble(offset + 24, coord.getM());
+ offset += 32;
+ }
+ break;
+ default:
+ throw new IllegalStateException("coordinateType was not configured properly");
+ }
+ }
+
+ @Override
+ public CoordinateSequence getCoordinates(int offset, int numCoordinates) {
+ Coordinate[] coordinates = new Coordinate[numCoordinates];
+ int dimension = 2;
+ int measures = 0;
+ switch (coordinateType) {
+ case XY:
+ for (int k = 0; k < numCoordinates; k++) {
+ double x = byteBuffer.getDouble(offset);
+ double y = byteBuffer.getDouble(offset + 8);
+ coordinates[k] = new CoordinateXY(x, y);
+ offset += 16;
+ }
+ break;
+ case XYZ:
+ dimension = 3;
+ for (int k = 0; k < numCoordinates; k++) {
+ double x = byteBuffer.getDouble(offset);
+ double y = byteBuffer.getDouble(offset + 8);
+ double z = byteBuffer.getDouble(offset + 16);
+ coordinates[k] = new Coordinate(x, y, z);
+ offset += 24;
+ }
+ break;
+ case XYM:
+ dimension = 3;
+ measures = 1;
+ for (int k = 0; k < numCoordinates; k++) {
+ double x = byteBuffer.getDouble(offset);
+ double y = byteBuffer.getDouble(offset + 8);
+ double m = byteBuffer.getDouble(offset + 16);
+ coordinates[k] = new CoordinateXYM(x, y, m);
+ offset += 24;
+ }
+ break;
+ case XYZM:
+ dimension = 4;
+ measures = 1;
+ for (int k = 0; k < numCoordinates; k++) {
+ double x = byteBuffer.getDouble(offset);
+ double y = byteBuffer.getDouble(offset + 8);
+ double z = byteBuffer.getDouble(offset + 16);
+ double m = byteBuffer.getDouble(offset + 24);
+ coordinates[k] = new CoordinateXYZM(x, y, z, m);
+ offset += 32;
+ }
+ break;
+ default:
+ throw new IllegalStateException("coordinateType was not configured properly");
+ }
+ return new CoordinateArraySequence(coordinates, dimension, measures);
+ }
+
+ @Override
+ public GeometryBuffer slice(int offset) {
+ byteBuffer.position(offset);
+ return new ByteBufferGeometryBuffer(byteBuffer.slice());
+ }
+
+ @Override
+ public byte[] toByteArray() {
+ if (byteBuffer.arrayOffset() == 0) {
+ return byteBuffer.array();
+ } else {
+ byte[] bytes = new byte[byteBuffer.capacity()];
+ byteBuffer.get(bytes);
+ return bytes;
+ }
+ }
+}
diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/CoordinateType.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/CoordinateType.java
new file mode 100644
index 00000000..b89d2e49
--- /dev/null
+++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/CoordinateType.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+enum CoordinateType {
+ XY(1, 2, false, false),
+ XYZ(2, 3, true, false),
+ XYM(3, 3, false, true),
+ XYZM(4, 4, true, true);
+
+ public final int value;
+ public final int ordinates;
+ public final boolean hasZ;
+ public final boolean hasM;
+ public final int bytes;
+
+ CoordinateType(int value, int ordinates, boolean hasZ, boolean hasM) {
+ this.value = value;
+ this.ordinates = ordinates;
+ this.hasZ = hasZ;
+ this.hasM = hasM;
+ this.bytes = 8 * ordinates;
+ }
+
+ public static CoordinateType valueOf(int value) {
+ switch (value) {
+ case 1:
+ return XY;
+ case 2:
+ return XYZ;
+ case 3:
+ return XYM;
+ case 4:
+ return XYZM;
+ default:
+ throw new IllegalArgumentException("Invalid coordinate type value: " + value);
+ }
+ }
+}
diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryBuffer.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryBuffer.java
new file mode 100644
index 00000000..d8bdc74e
--- /dev/null
+++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryBuffer.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.CoordinateSequence;
+
+interface GeometryBuffer {
+ CoordinateType getCoordinateType();
+
+ void setCoordinateType(CoordinateType coordinateType);
+
+ int getLength();
+
+ void mark(int offset);
+
+ int getMark();
+
+ void putByte(int offset, byte value);
+
+ byte getByte(int offset);
+
+ void putBytes(int offset, byte[] bytes);
+
+ void getBytes(byte[] bytes, int offset, int length);
+
+ void putInt(int offset, int value);
+
+ int getInt(int offset);
+
+ void putCoordinate(int offset, Coordinate coordinate);
+
+ CoordinateSequence getCoordinate(int offset);
+
+ void putCoordinates(int offset, CoordinateSequence coordinates);
+
+ CoordinateSequence getCoordinates(int offset, int numCoordinates);
+
+ GeometryBuffer slice(int offset);
+
+ byte[] toByteArray();
+}
diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryBufferFactory.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryBufferFactory.java
new file mode 100644
index 00000000..695e1bd5
--- /dev/null
+++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometryBufferFactory.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+public class GeometryBufferFactory {
+ private static boolean enableUnsafeGeometryBuffer = true;
+
+ public static void toggleUnsafeGeometryBuffer(boolean enabled) {
+ enableUnsafeGeometryBuffer = enabled;
+ }
+
+ public static GeometryBuffer create(int bufferSize) {
+ if (UnsafeGeometryBuffer.isUnsafeAvailable() && enableUnsafeGeometryBuffer) {
+ return new UnsafeGeometryBuffer(bufferSize);
+ } else {
+ return new ByteBufferGeometryBuffer(bufferSize);
+ }
+ }
+
+ public static GeometryBuffer create(String bufferType, int bufferSize) {
+ switch (bufferType) {
+ case "bytebuffer":
+ return new ByteBufferGeometryBuffer(bufferSize);
+ case "unsafe":
+ return new UnsafeGeometryBuffer(bufferSize);
+ default:
+ throw new IllegalArgumentException("Unknown buffer type: " + bufferType);
+ }
+ }
+
+ public static GeometryBuffer wrap(byte[] bytes) {
+ if (UnsafeGeometryBuffer.isUnsafeAvailable() && enableUnsafeGeometryBuffer) {
+ return new UnsafeGeometryBuffer(bytes);
+ } else {
+ return new ByteBufferGeometryBuffer(bytes);
+ }
+ }
+
+ public static GeometryBuffer wrap(String bufferType, byte[] bytes) {
+ switch (bufferType) {
+ case "bytebuffer":
+ return new ByteBufferGeometryBuffer(bytes);
+ case "unsafe":
+ return new UnsafeGeometryBuffer(bytes);
+ default:
+ throw new IllegalArgumentException("Unknown buffer type: " + bufferType);
+ }
+ }
+}
diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java
new file mode 100644
index 00000000..8c488a46
--- /dev/null
+++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerializer.java
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.CoordinateSequence;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryCollection;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.geom.LineString;
+import org.locationtech.jts.geom.LinearRing;
+import org.locationtech.jts.geom.MultiLineString;
+import org.locationtech.jts.geom.MultiPoint;
+import org.locationtech.jts.geom.MultiPolygon;
+import org.locationtech.jts.geom.Point;
+import org.locationtech.jts.geom.Polygon;
+import org.locationtech.jts.io.WKBConstants;
+
+public class GeometrySerializer {
+ private static final Coordinate NULL_COORDINATE = new Coordinate(Double.NaN, Double.NaN);
+ private static final GeometryFactory FACTORY = new GeometryFactory();
+
+ public static byte[] serialize(Geometry geometry) {
+ GeometryBuffer buffer;
+ if (geometry instanceof Point) {
+ buffer = serializePoint((Point) geometry);
+ } else if (geometry instanceof MultiPoint) {
+ buffer = serializeMultiPoint((MultiPoint) geometry);
+ } else if (geometry instanceof LineString) {
+ buffer = serializeLineString((LineString) geometry);
+ } else if (geometry instanceof MultiLineString) {
+ buffer = serializeMultiLineString((MultiLineString) geometry);
+ } else if (geometry instanceof Polygon) {
+ buffer = serializePolygon((Polygon) geometry);
+ } else if (geometry instanceof MultiPolygon) {
+ buffer = serializeMultiPolygon((MultiPolygon) geometry);
+ } else if (geometry instanceof GeometryCollection) {
+ buffer = serializeGeometryCollection((GeometryCollection) geometry);
+ } else {
+ throw new UnsupportedOperationException(
+ "Geometry type is not supported: " + geometry.getClass().getSimpleName());
+ }
+ return buffer.toByteArray();
+ }
+
+ public static Geometry deserialize(byte[] bytes) {
+ GeometryBuffer buffer = GeometryBufferFactory.wrap(bytes);
+ return deserialize(buffer);
+ }
+
+ public static Geometry deserialize(GeometryBuffer buffer) {
+ checkBufferSize(buffer, 8);
+ int preambleByte = buffer.getByte(0) & 0xFF;
+ int wkbType = preambleByte >> 4;
+ CoordinateType coordType = CoordinateType.valueOf((preambleByte & 0x0F) >> 1);
+ boolean hasSrid = (preambleByte & 0x01) != 0;
+ buffer.setCoordinateType(coordType);
+ int srid = 0;
+ if (hasSrid) {
+ int srid2 = (buffer.getByte(1) & 0xFF) << 16;
+ int srid1 = (buffer.getByte(2) & 0xFF) << 8;
+ int srid0 = buffer.getByte(3) & 0xFF;
+ srid = (srid2 | srid1 | srid0);
+ }
+ return deserialize(buffer, wkbType, srid);
+ }
+
+ private static Geometry deserialize(GeometryBuffer buffer, int wkbType, int srid) {
+ switch (wkbType) {
+ case WKBConstants.wkbPoint:
+ return deserializePoint(buffer, srid);
+ case WKBConstants.wkbMultiPoint:
+ return deserializeMultiPoint(buffer, srid);
+ case WKBConstants.wkbLineString:
+ return deserializeLineString(buffer, srid);
+ case WKBConstants.wkbMultiLineString:
+ return deserializeMultiLineString(buffer, srid);
+ case WKBConstants.wkbPolygon:
+ return deserializePolygon(buffer, srid);
+ case WKBConstants.wkbMultiPolygon:
+ return deserializeMultiPolygon(buffer, srid);
+ case WKBConstants.wkbGeometryCollection:
+ return deserializeGeometryCollection(buffer, srid);
+ default:
+ throw new IllegalArgumentException(
+ "Cannot deserialize buffer containing unknown geometry type ID: " + wkbType);
+ }
+ }
+
+ private static GeometryBuffer serializePoint(Point point) {
+ Coordinate coordinate = point.getCoordinate();
+ if (coordinate == null) {
+ return createGeometryBuffer(WKBConstants.wkbPoint, CoordinateType.XY, point.getSRID(), 8, 0);
+ }
+ CoordinateType coordType = getCoordinateType(coordinate);
+ int bufferSize = 8 + coordType.bytes;
+ GeometryBuffer buffer =
+ createGeometryBuffer(WKBConstants.wkbPoint, coordType, point.getSRID(), bufferSize, 1);
+ buffer.putCoordinate(8, coordinate);
+ return buffer;
+ }
+
+ private static Point deserializePoint(GeometryBuffer buffer, int srid) {
+ CoordinateType coordType = buffer.getCoordinateType();
+ int numCoordinates = getNonNegativeInt(buffer, 4);
+ Point point;
+ if (numCoordinates == 0) {
+ point = FACTORY.createPoint();
+ buffer.mark(8);
+ } else {
+ int bufferSize = 8 + coordType.bytes;
+ checkBufferSize(buffer, bufferSize);
+ CoordinateSequence coordinates = buffer.getCoordinate(8);
+ point = FACTORY.createPoint(coordinates);
+ buffer.mark(bufferSize);
+ }
+ point.setSRID(srid);
+ return point;
+ }
+
+ private static GeometryBuffer serializeMultiPoint(MultiPoint multiPoint) {
+ int numPoints = multiPoint.getNumGeometries();
+ if (numPoints == 0) {
+ return createGeometryBuffer(
+ WKBConstants.wkbMultiPoint, CoordinateType.XY, multiPoint.getSRID(), 8, 0);
+ }
+ CoordinateType coordType = getCoordinateType(multiPoint);
+ int bufferSize = 8 + numPoints * coordType.bytes;
+ GeometryBuffer buffer =
+ createGeometryBuffer(
+ WKBConstants.wkbMultiPoint, coordType, multiPoint.getSRID(), bufferSize, numPoints);
+ for (int k = 0; k < numPoints; k++) {
+ Point point = (Point) multiPoint.getGeometryN(k);
+ Coordinate coordinate = point.getCoordinate();
+ int coordinateOffset = 8 + k * coordType.bytes;
+ if (coordinate == null) {
+ buffer.putCoordinate(coordinateOffset, NULL_COORDINATE);
+ } else {
+ buffer.putCoordinate(coordinateOffset, coordinate);
+ }
+ }
+ return buffer;
+ }
+
+ private static MultiPoint deserializeMultiPoint(GeometryBuffer buffer, int srid) {
+ CoordinateType coordType = buffer.getCoordinateType();
+ int numPoints = getNonNegativeInt(buffer, 4);
+ int bufferSize = 8 + numPoints * coordType.bytes;
+ checkBufferSize(buffer, bufferSize);
+ Point[] points = new Point[numPoints];
+ for (int i = 0; i < numPoints; i++) {
+ CoordinateSequence coordinates = buffer.getCoordinate(8 + i * coordType.bytes);
+ Coordinate coordinate = coordinates.getCoordinate(0);
+ if (Double.isNaN(coordinate.x)) {
+ points[i] = FACTORY.createPoint();
+ } else {
+ points[i] = FACTORY.createPoint(coordinates);
+ }
+ points[i].setSRID(srid);
+ }
+ buffer.mark(bufferSize);
+ MultiPoint multiPoint = FACTORY.createMultiPoint(points);
+ multiPoint.setSRID(srid);
+ return multiPoint;
+ }
+
+ private static GeometryBuffer serializeLineString(LineString lineString) {
+ CoordinateSequence coordinates = lineString.getCoordinateSequence();
+ int numCoordinates = coordinates.size();
+ if (numCoordinates == 0) {
+ return createGeometryBuffer(
+ WKBConstants.wkbLineString, CoordinateType.XY, lineString.getSRID(), 8, 0);
+ }
+ CoordinateType coordType = getCoordinateType(coordinates.getCoordinate(0));
+ int bufferSize = 8 + numCoordinates * coordType.bytes;
+ GeometryBuffer buffer =
+ createGeometryBuffer(
+ WKBConstants.wkbLineString,
+ coordType,
+ lineString.getSRID(),
+ bufferSize,
+ numCoordinates);
+ buffer.putCoordinates(8, coordinates);
+ return buffer;
+ }
+
+ private static LineString deserializeLineString(GeometryBuffer buffer, int srid) {
+ CoordinateType coordType = buffer.getCoordinateType();
+ int numCoordinates = getNonNegativeInt(buffer, 4);
+ int bufferSize = 8 + numCoordinates * coordType.bytes;
+ checkBufferSize(buffer, bufferSize);
+ CoordinateSequence coordinates = buffer.getCoordinates(8, numCoordinates);
+ buffer.mark(bufferSize);
+ LineString lineString = FACTORY.createLineString(coordinates);
+ lineString.setSRID(srid);
+ return lineString;
+ }
+
+ private static GeometryBuffer serializeMultiLineString(MultiLineString multiLineString) {
+ int numLineStrings = multiLineString.getNumGeometries();
+ if (numLineStrings <= 0) {
+ return createGeometryBuffer(
+ WKBConstants.wkbMultiLineString, CoordinateType.XY, multiLineString.getSRID(), 8, 0);
+ }
+ CoordinateType coordType = getCoordinateType(multiLineString);
+ int numCoordinates = multiLineString.getNumPoints();
+ int coordsOffset = 8;
+ int numOffset = 8 + numCoordinates * coordType.bytes;
+ int bufferSize = numOffset + 4 + numLineStrings * 4;
+ GeometryBuffer buffer =
+ createGeometryBuffer(
+ WKBConstants.wkbMultiLineString,
+ coordType,
+ multiLineString.getSRID(),
+ bufferSize,
+ numCoordinates);
+ GeomPartSerializer serializer = new GeomPartSerializer(buffer, coordsOffset, numOffset);
+ serializer.writeInt(numLineStrings);
+ for (int k = 0; k < numLineStrings; k++) {
+ LineString ls = (LineString) multiLineString.getGeometryN(k);
+ serializer.write(ls);
+ }
+ assert bufferSize == serializer.intsOffset;
+ return buffer;
+ }
+
+ private static MultiLineString deserializeMultiLineString(GeometryBuffer buffer, int srid) {
+ CoordinateType coordType = buffer.getCoordinateType();
+ int numCoordinates = getNonNegativeInt(buffer, 4);
+ if (numCoordinates == 0) {
+ buffer.mark(8);
+ MultiLineString multiLineString = FACTORY.createMultiLineString();
+ multiLineString.setSRID(srid);
+ return multiLineString;
+ }
+ int coordsOffset = 8;
+ int numOffset = 8 + numCoordinates * coordType.bytes;
+ GeomPartSerializer serializer = new GeomPartSerializer(buffer, coordsOffset, numOffset);
+ int numLineStrings = serializer.checkedReadNonNegativeInt();
+ serializer.checkRemainingIntsAtLeast(numLineStrings);
+ LineString[] lineStrings = new LineString[numLineStrings];
+ for (int k = 0; k < numLineStrings; k++) {
+ LineString ls = serializer.readLineString();
+ ls.setSRID(srid);
+ lineStrings[k] = ls;
+ }
+ serializer.markEndOfBuffer();
+ MultiLineString multiLineString = FACTORY.createMultiLineString(lineStrings);
+ multiLineString.setSRID(srid);
+ return multiLineString;
+ }
+
+ private static GeometryBuffer serializePolygon(Polygon polygon) {
+ LinearRing exteriorRing = polygon.getExteriorRing();
+ if (exteriorRing == null || exteriorRing.isEmpty()) {
+ return createGeometryBuffer(
+ WKBConstants.wkbPolygon, CoordinateType.XY, polygon.getSRID(), 8, 0);
+ }
+ CoordinateSequence coordinates = exteriorRing.getCoordinateSequence();
+ CoordinateType coordType = getCoordinateType(coordinates.getCoordinate(0));
+ int numCoordinates = polygon.getNumPoints();
+ int numInteriorRings = polygon.getNumInteriorRing();
+ int coordsOffset = 8;
+ int numRingsOffset = 8 + numCoordinates * coordType.bytes;
+ int bufferSize = numRingsOffset + 4 + 4 * (numInteriorRings + 1);
+ GeometryBuffer buffer =
+ createGeometryBuffer(
+ WKBConstants.wkbPolygon, coordType, polygon.getSRID(), bufferSize, numCoordinates);
+ GeomPartSerializer serializer = new GeomPartSerializer(buffer, coordsOffset, numRingsOffset);
+ serializer.write(polygon);
+ assert bufferSize == serializer.intsOffset;
+ return buffer;
+ }
+
+ private static Polygon deserializePolygon(GeometryBuffer buffer, int srid) {
+ CoordinateType coordType = buffer.getCoordinateType();
+ int numCoordinates = getNonNegativeInt(buffer, 4);
+ if (numCoordinates == 0) {
+ buffer.mark(8);
+ Polygon polygon = FACTORY.createPolygon();
+ polygon.setSRID(srid);
+ return polygon;
+ }
+ int coordsOffset = 8;
+ int numRingsOffset = 8 + numCoordinates * coordType.bytes;
+ GeomPartSerializer serializer = new GeomPartSerializer(buffer, coordsOffset, numRingsOffset);
+ Polygon polygon = serializer.readPolygon();
+ serializer.markEndOfBuffer();
+ polygon.setSRID(srid);
+ return polygon;
+ }
+
+ private static GeometryBuffer serializeMultiPolygon(MultiPolygon multiPolygon) {
+ int numPolygons = multiPolygon.getNumGeometries();
+ if (numPolygons == 0) {
+ return createGeometryBuffer(
+ WKBConstants.wkbMultiPolygon, CoordinateType.XY, multiPolygon.getSRID(), 8, 0);
+ }
+ int numCoordinates = 0;
+ CoordinateType coordType = getCoordinateType(multiPolygon);
+ int totalRings = 0;
+ for (int k = 0; k < numPolygons; k++) {
+ Polygon polygon = (Polygon) multiPolygon.getGeometryN(k);
+ if (!polygon.isEmpty()) {
+ int numRings = polygon.getNumInteriorRing() + 1;
+ totalRings += numRings;
+ numCoordinates += polygon.getNumPoints();
+ }
+ }
+ int coordsOffset = 8;
+ int numPolygonsOffset = 8 + numCoordinates * coordType.bytes;
+ int bufferSize = numPolygonsOffset + 4 + (numPolygons * 4) + (totalRings * 4);
+ GeometryBuffer buffer =
+ createGeometryBuffer(
+ WKBConstants.wkbMultiPolygon,
+ coordType,
+ multiPolygon.getSRID(),
+ bufferSize,
+ numCoordinates);
+ GeomPartSerializer serializer = new GeomPartSerializer(buffer, coordsOffset, numPolygonsOffset);
+ serializer.writeInt(numPolygons);
+ for (int k = 0; k < numPolygons; k++) {
+ Polygon polygon = (Polygon) multiPolygon.getGeometryN(k);
+ serializer.write(polygon);
+ }
+ assert bufferSize == serializer.intsOffset;
+ return buffer;
+ }
+
+ private static MultiPolygon deserializeMultiPolygon(GeometryBuffer buffer, int srid) {
+ CoordinateType coordType = buffer.getCoordinateType();
+ int numCoordinates = getNonNegativeInt(buffer, 4);
+ if (numCoordinates == 0) {
+ buffer.mark(8);
+ MultiPolygon multiPolygon = FACTORY.createMultiPolygon();
+ multiPolygon.setSRID(srid);
+ return multiPolygon;
+ }
+ int coordsOffset = 8;
+ int numPolygonsOffset = 8 + numCoordinates * coordType.bytes;
+ GeomPartSerializer serializer = new GeomPartSerializer(buffer, coordsOffset, numPolygonsOffset);
+ int numPolygons = serializer.checkedReadNonNegativeInt();
+ Polygon[] polygons = new Polygon[numPolygons];
+ for (int k = 0; k < numPolygons; k++) {
+ Polygon polygon = serializer.readPolygon();
+ polygon.setSRID(srid);
+ polygons[k] = polygon;
+ }
+ serializer.markEndOfBuffer();
+ MultiPolygon multiPolygon = FACTORY.createMultiPolygon(polygons);
+ multiPolygon.setSRID(srid);
+ return multiPolygon;
+ }
+
+ private static GeometryBuffer serializeGeometryCollection(GeometryCollection geometryCollection) {
+ int numGeometries = geometryCollection.getNumGeometries();
+ if (numGeometries == 0) {
+ return createGeometryBuffer(
+ WKBConstants.wkbGeometryCollection,
+ CoordinateType.XY,
+ geometryCollection.getSRID(),
+ 8,
+ 0);
+ }
+ byte[][] buffers = new byte[numGeometries][];
+ int totalBytes = 0;
+ for (int k = 0; k < numGeometries; k++) {
+ byte[] buf = serialize(geometryCollection.getGeometryN(k));
+ buffers[k] = buf;
+ totalBytes += alignedOffset(buf.length);
+ }
+ int bufferSize = 8 + totalBytes;
+ GeometryBuffer buffer =
+ createGeometryBuffer(
+ WKBConstants.wkbGeometryCollection,
+ CoordinateType.XY,
+ geometryCollection.getSRID(),
+ bufferSize,
+ numGeometries);
+ int offset = 8;
+ for (int k = 0; k < numGeometries; k++) {
+ byte[] buf = buffers[k];
+ buffer.putBytes(offset, buf);
+ offset += alignedOffset(buf.length);
+ }
+ assert offset == bufferSize;
+ return buffer;
+ }
+
+ private static GeometryCollection deserializeGeometryCollection(GeometryBuffer buffer, int srid) {
+ int numGeometries = getNonNegativeInt(buffer, 4);
+ if (numGeometries == 0) {
+ buffer.mark(8);
+ GeometryCollection geometryCollection = FACTORY.createGeometryCollection();
+ geometryCollection.setSRID(srid);
+ return geometryCollection;
+ }
+ Geometry[] geometries = new Geometry[numGeometries];
+ int offset = 8;
+ for (int k = 0; k < numGeometries; k++) {
+ GeometryBuffer geomBuffer = buffer.slice(offset);
+ Geometry geometry = deserialize(geomBuffer);
+ int geomLength = alignedOffset(geomBuffer.getMark());
+ geometry.setSRID(srid);
+ geometries[k] = geometry;
+ offset += geomLength;
+ }
+ buffer.mark(offset);
+ GeometryCollection geometryCollection = FACTORY.createGeometryCollection(geometries);
+ geometryCollection.setSRID(srid);
+ return geometryCollection;
+ }
+
+ private static GeometryBuffer createGeometryBuffer(
+ int wkbType, CoordinateType coordType, int srid, int bufferSize, int numCoordinates) {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferSize);
+ buffer.setCoordinateType(coordType);
+
+ // Set header bytes [preamble][srid (3 bytes)][numCoordinates (4 bytes)]
+ int hasSridBit = (srid != 0 ? 1 : 0);
+ int preambleByte = (wkbType << 4) | (coordType.value << 1) | hasSridBit;
+ buffer.putByte(0, (byte) preambleByte);
+ if (srid != 0) {
+ // Store SRID in the next 3 bytes in big endian byte order, with the highest bit set
+ // indicating that SRID need to be decoded when deserializing it.
+ buffer.putByte(1, (byte) (srid >> 16));
+ buffer.putByte(2, (byte) (srid >> 8));
+ buffer.putByte(3, (byte) srid);
+ }
+ buffer.putInt(4, numCoordinates);
+ return buffer;
+ }
+
+ private static void checkBufferSize(GeometryBuffer buffer, int minimumSize) {
+ if (buffer.getLength() < minimumSize) {
+ throw new IllegalArgumentException("Buffer to be deserialized is incomplete");
+ }
+ }
+
+ private static int getNonNegativeInt(GeometryBuffer buffer, int offset) {
+ int value = buffer.getInt(offset);
+ if (value < 0) {
+ throw new IllegalArgumentException("Unexpected negative value encountered: " + value);
+ }
+ return value;
+ }
+
+ private static CoordinateType getCoordinateType(Coordinate coordinate) {
+ boolean hasZ = !Double.isNaN(coordinate.getZ());
+ boolean hasM = !Double.isNaN(coordinate.getM());
+ return getCoordinateType(hasZ, hasM);
+ }
+
+ private static CoordinateType getCoordinateType(Geometry geometry) {
+ Coordinate coord = geometry.getCoordinate();
+ if (coord != null) {
+ return getCoordinateType(coord);
+ } else {
+ return CoordinateType.XY;
+ }
+ }
+
+ private static CoordinateType getCoordinateType(boolean hasZ, boolean hasM) {
+ if (hasZ && hasM) {
+ return CoordinateType.XYZM;
+ } else if (hasZ) {
+ return CoordinateType.XYZ;
+ } else if (hasM) {
+ return CoordinateType.XYM;
+ } else {
+ return CoordinateType.XY;
+ }
+ }
+
+ private static int alignedOffset(int offset) {
+ return (offset + 7) & ~7;
+ }
+
+ static class GeomPartSerializer {
+ final GeometryBuffer buffer;
+ int coordsOffset;
+ final int coordsEndOffset;
+ int intsOffset;
+
+ GeomPartSerializer(GeometryBuffer buffer, int coordsOffset, int intsOffset) {
+ this.buffer = buffer;
+ this.coordsOffset = coordsOffset;
+ this.coordsEndOffset = intsOffset;
+ this.intsOffset = intsOffset;
+ }
+
+ LineString readLineString() {
+ CoordinateSequence coordinates = readCoordinates();
+ return FACTORY.createLineString(coordinates);
+ }
+
+ LinearRing readRing() {
+ CoordinateSequence coordinates = readCoordinates();
+ return FACTORY.createLinearRing(coordinates);
+ }
+
+ Polygon readPolygon() {
+ int numRings = checkedReadNonNegativeInt();
+ if (numRings == 0) {
+ return FACTORY.createPolygon();
+ }
+ checkRemainingIntsAtLeast(numRings);
+ int numInteriorRings = numRings - 1;
+ LinearRing shell = readRing();
+ LinearRing[] holes = new LinearRing[numInteriorRings];
+ for (int k = 0; k < numInteriorRings; k++) {
+ holes[k] = readRing();
+ }
+ return FACTORY.createPolygon(shell, holes);
+ }
+
+ CoordinateSequence readCoordinates() {
+ int numCoordinates = getNonNegativeInt(buffer, intsOffset);
+ int newCoordsOffset = coordsOffset + buffer.getCoordinateType().bytes * numCoordinates;
+ if (newCoordsOffset > coordsEndOffset) {
+ throw new IllegalStateException(
+ "Number of coordinates exceeds the capacity of buffer: " + numCoordinates);
+ }
+ CoordinateSequence coordinates = buffer.getCoordinates(coordsOffset, numCoordinates);
+ coordsOffset = newCoordsOffset;
+ intsOffset += 4;
+ return coordinates;
+ }
+
+ int readNonNegativeInt() {
+ int value = getNonNegativeInt(buffer, intsOffset);
+ intsOffset += 4;
+ return value;
+ }
+
+ int checkedReadNonNegativeInt() {
+ checkBufferSize(buffer, intsOffset + 4);
+ return readNonNegativeInt();
+ }
+
+ void checkRemainingIntsAtLeast(int num) {
+ checkBufferSize(buffer, intsOffset + 4 * num);
+ }
+
+ void write(LineString lineString) {
+ CoordinateSequence coordinates = lineString.getCoordinateSequence();
+ int numCoordinates = coordinates.size();
+ buffer.putCoordinates(coordsOffset, coordinates);
+ buffer.putInt(intsOffset, numCoordinates);
+ coordsOffset += numCoordinates * buffer.getCoordinateType().bytes;
+ intsOffset += 4;
+ }
+
+ void write(Polygon polygon) {
+ LinearRing exteriorRing = polygon.getExteriorRing();
+ if (exteriorRing.isEmpty()) {
+ writeInt(0);
+ return;
+ }
+ int numInteriorRings = polygon.getNumInteriorRing();
+ writeInt(numInteriorRings + 1);
+ write(exteriorRing);
+ for (int k = 0; k < numInteriorRings; k++) {
+ write(polygon.getInteriorRingN(k));
+ }
+ }
+
+ void writeInt(int value) {
+ buffer.putInt(intsOffset, value);
+ intsOffset += 4;
+ }
+
+ void markEndOfBuffer() {
+ buffer.mark(intsOffset);
+ }
+ }
+}
diff --git a/common/src/main/java/org/apache/sedona/common/geometrySerde/UnsafeGeometryBuffer.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/UnsafeGeometryBuffer.java
new file mode 100644
index 00000000..a29c3206
--- /dev/null
+++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/UnsafeGeometryBuffer.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+import org.apache.spark.unsafe.Platform;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.CoordinateSequence;
+import org.locationtech.jts.geom.CoordinateXY;
+import org.locationtech.jts.geom.CoordinateXYM;
+import org.locationtech.jts.geom.CoordinateXYZM;
+import org.locationtech.jts.geom.impl.CoordinateArraySequence;
+
+class UnsafeGeometryBuffer implements GeometryBuffer {
+ private static final long BYTE_ARRAY_BASE_OFFSET = Platform.BYTE_ARRAY_OFFSET;
+
+ public static boolean isUnsafeAvailable() {
+ return BYTE_ARRAY_BASE_OFFSET != 0;
+ }
+
+ private CoordinateType coordinateType = CoordinateType.XY;
+ private final byte[] bytes;
+ private final long baseOffset;
+ private int markOffset = 0;
+
+ public UnsafeGeometryBuffer(int bufferSize) {
+ bytes = new byte[bufferSize];
+ baseOffset = BYTE_ARRAY_BASE_OFFSET;
+ }
+
+ public UnsafeGeometryBuffer(byte[] bytes, int offset) {
+ this.bytes = bytes;
+ baseOffset = offset + BYTE_ARRAY_BASE_OFFSET;
+ }
+
+ public UnsafeGeometryBuffer(byte[] bytes) {
+ this.bytes = bytes;
+ baseOffset = BYTE_ARRAY_BASE_OFFSET;
+ }
+
+ @Override
+ public CoordinateType getCoordinateType() {
+ return coordinateType;
+ }
+
+ @Override
+ public void setCoordinateType(CoordinateType coordinateType) {
+ this.coordinateType = coordinateType;
+ }
+
+ @Override
+ public int getLength() {
+ return (int) (bytes.length - baseOffset + BYTE_ARRAY_BASE_OFFSET);
+ }
+
+ @Override
+ public void mark(int offset) {
+ markOffset = offset;
+ }
+
+ @Override
+ public int getMark() {
+ return markOffset;
+ }
+
+ @Override
+ public void putByte(int offset, byte value) {
+ Platform.putByte(bytes, baseOffset + offset, value);
+ }
+
+ @Override
+ public byte getByte(int offset) {
+ assert baseOffset + offset < bytes.length + BYTE_ARRAY_BASE_OFFSET;
+ return Platform.getByte(bytes, baseOffset + offset);
+ }
+
+ @Override
+ public void putBytes(int offset, byte[] inBytes) {
+ assert baseOffset + offset + inBytes.length <= bytes.length + BYTE_ARRAY_BASE_OFFSET;
+ Platform.copyMemory(inBytes, BYTE_ARRAY_BASE_OFFSET, bytes, baseOffset + offset, inBytes.length);
+ }
+
+ @Override
+ public void getBytes(byte[] outBytes, int offset, int length) {
+ assert baseOffset + offset + length <= bytes.length + BYTE_ARRAY_BASE_OFFSET;
+ Platform.copyMemory(bytes, baseOffset + offset, outBytes, BYTE_ARRAY_BASE_OFFSET, length);
+ }
+
+ @Override
+ public void putInt(int offset, int value) {
+ assert baseOffset + offset + 4 <= bytes.length + BYTE_ARRAY_BASE_OFFSET;
+ Platform.putInt(bytes, baseOffset + offset, value);
+ }
+
+ @Override
+ public int getInt(int offset) {
+ assert baseOffset + offset + 4 <= bytes.length + BYTE_ARRAY_BASE_OFFSET;
+ return Platform.getInt(bytes, baseOffset + offset);
+ }
+
+ @Override
+ public void putCoordinate(int offset, Coordinate coordinate) {
+ long coordOffset = baseOffset + offset;
+ assert coordOffset + coordinateType.bytes <= bytes.length + BYTE_ARRAY_BASE_OFFSET;
+ switch (coordinateType) {
+ case XY:
+ Platform.putDouble(bytes, coordOffset, coordinate.x);
+ Platform.putDouble(bytes, coordOffset + 8, coordinate.y);
+ break;
+ case XYZ:
+ Platform.putDouble(bytes, coordOffset, coordinate.x);
+ Platform.putDouble(bytes, coordOffset + 8, coordinate.y);
+ Platform.putDouble(bytes, coordOffset + 16, coordinate.getZ());
+ break;
+ case XYM:
+ Platform.putDouble(bytes, coordOffset, coordinate.x);
+ Platform.putDouble(bytes, coordOffset + 8, coordinate.y);
+ Platform.putDouble(bytes, coordOffset + 16, coordinate.getM());
+ break;
+ case XYZM:
+ Platform.putDouble(bytes, coordOffset, coordinate.x);
+ Platform.putDouble(bytes, coordOffset + 8, coordinate.y);
+ Platform.putDouble(bytes, coordOffset + 16, coordinate.getZ());
+ Platform.putDouble(bytes, coordOffset + 24, coordinate.getM());
+ break;
+ default:
+ throw new IllegalStateException("coordinateType was not configured properly");
+ }
+ }
+
+ @Override
+ public CoordinateSequence getCoordinate(int offset) {
+ long coordOffset = baseOffset + offset;
+ assert coordOffset + coordinateType.bytes <= bytes.length + BYTE_ARRAY_BASE_OFFSET;
+ double x = Platform.getDouble(bytes, coordOffset);
+ double y = Platform.getDouble(bytes, coordOffset + 8);
+ double z;
+ double m;
+ Coordinate[] coordinates = new Coordinate[1];
+ switch (coordinateType) {
+ case XY:
+ coordinates[0] = new CoordinateXY(x, y);
+ return new CoordinateArraySequence(coordinates, 2, 0);
+ case XYZ:
+ z = Platform.getDouble(bytes, coordOffset + 16);
+ coordinates[0] = new Coordinate(x, y, z);
+ return new CoordinateArraySequence(coordinates, 3, 0);
+ case XYM:
+ m = Platform.getDouble(bytes, coordOffset + 16);
+ coordinates[0] = new CoordinateXYM(x, y, m);
+ return new CoordinateArraySequence(coordinates, 3, 1);
+ case XYZM:
+ z = Platform.getDouble(bytes, coordOffset + 16);
+ m = Platform.getDouble(bytes, coordOffset + 24);
+ coordinates[0] = new CoordinateXYZM(x, y, z, m);
+ return new CoordinateArraySequence(coordinates, 4, 1);
+ default:
+ throw new IllegalStateException("coordinateType was not configured properly");
+ }
+ }
+
+ @Override
+ public void putCoordinates(int offset, CoordinateSequence coordinates) {
+ long coordOffset = baseOffset + offset;
+ int numCoordinates = coordinates.size();
+ assert coordOffset + (long) coordinateType.bytes * numCoordinates
+ <= bytes.length + BYTE_ARRAY_BASE_OFFSET;
+ switch (coordinateType) {
+ case XY:
+ for (int k = 0; k < numCoordinates; k++) {
+ Coordinate coord = coordinates.getCoordinate(k);
+ Platform.putDouble(bytes, coordOffset, coord.x);
+ Platform.putDouble(bytes, coordOffset + 8, coord.y);
+ coordOffset += 16;
+ }
+ break;
+ case XYZ:
+ for (int k = 0; k < numCoordinates; k++) {
+ Coordinate coord = coordinates.getCoordinate(k);
+ Platform.putDouble(bytes, coordOffset, coord.x);
+ Platform.putDouble(bytes, coordOffset + 8, coord.y);
+ Platform.putDouble(bytes, coordOffset + 16, coord.getZ());
+ coordOffset += 24;
+ }
+ break;
+ case XYM:
+ for (int k = 0; k < numCoordinates; k++) {
+ Coordinate coord = coordinates.getCoordinate(k);
+ Platform.putDouble(bytes, coordOffset, coord.x);
+ Platform.putDouble(bytes, coordOffset + 8, coord.y);
+ Platform.putDouble(bytes, coordOffset + 16, coord.getM());
+ coordOffset += 24;
+ }
+ break;
+ case XYZM:
+ for (int k = 0; k < numCoordinates; k++) {
+ Coordinate coord = coordinates.getCoordinate(k);
+ Platform.putDouble(bytes, coordOffset, coord.x);
+ Platform.putDouble(bytes, coordOffset + 8, coord.y);
+ Platform.putDouble(bytes, coordOffset + 16, coord.getZ());
+ Platform.putDouble(bytes, coordOffset + 24, coord.getM());
+ coordOffset += 32;
+ }
+ break;
+ default:
+ throw new IllegalStateException("coordinateType was not configured properly");
+ }
+ }
+
+ @Override
+ public CoordinateSequence getCoordinates(int offset, int numCoordinates) {
+ long coordOffset = baseOffset + offset;
+ assert coordOffset + (long) coordinateType.bytes * numCoordinates
+ <= bytes.length + BYTE_ARRAY_BASE_OFFSET;
+ Coordinate[] coordinates = new Coordinate[numCoordinates];
+ int dimension = 2;
+ int measures = 0;
+ switch (coordinateType) {
+ case XY:
+ for (int k = 0; k < numCoordinates; k++) {
+ double x = Platform.getDouble(bytes, coordOffset);
+ double y = Platform.getDouble(bytes, coordOffset + 8);
+ coordinates[k] = new CoordinateXY(x, y);
+ coordOffset += 16;
+ }
+ break;
+ case XYZ:
+ dimension = 3;
+ for (int k = 0; k < numCoordinates; k++) {
+ double x = Platform.getDouble(bytes, coordOffset);
+ double y = Platform.getDouble(bytes, coordOffset + 8);
+ double z = Platform.getDouble(bytes, coordOffset + 16);
+ coordinates[k] = new Coordinate(x, y, z);
+ coordOffset += 24;
+ }
+ break;
+ case XYM:
+ dimension = 3;
+ measures = 1;
+ for (int k = 0; k < numCoordinates; k++) {
+ double x = Platform.getDouble(bytes, coordOffset);
+ double y = Platform.getDouble(bytes, coordOffset + 8);
+ double m = Platform.getDouble(bytes, coordOffset + 16);
+ coordinates[k] = new CoordinateXYM(x, y, m);
+ coordOffset += 24;
+ }
+ break;
+ case XYZM:
+ dimension = 4;
+ measures = 1;
+ for (int k = 0; k < numCoordinates; k++) {
+ double x = Platform.getDouble(bytes, coordOffset);
+ double y = Platform.getDouble(bytes, coordOffset + 8);
+ double z = Platform.getDouble(bytes, coordOffset + 16);
+ double m = Platform.getDouble(bytes, coordOffset + 24);
+ coordinates[k] = new CoordinateXYZM(x, y, z, m);
+ coordOffset += 32;
+ }
+ break;
+ default:
+ throw new IllegalStateException("coordinateType was not configured properly");
+ }
+ return new CoordinateArraySequence(coordinates, dimension, measures);
+ }
+
+ @Override
+ public GeometryBuffer slice(int offset) {
+ assert baseOffset + offset <= bytes.length + BYTE_ARRAY_BASE_OFFSET;
+ int bytesOffset = (int) (baseOffset + offset - BYTE_ARRAY_BASE_OFFSET);
+ return new UnsafeGeometryBuffer(bytes, bytesOffset);
+ }
+
+ @Override
+ public byte[] toByteArray() {
+ if (baseOffset == BYTE_ARRAY_BASE_OFFSET) {
+ return bytes;
+ } else {
+ int length = (int) (bytes.length - baseOffset + BYTE_ARRAY_BASE_OFFSET);
+ byte[] copy = new byte[(int) length];
+ Platform.copyMemory(this.bytes, baseOffset, copy, BYTE_ARRAY_BASE_OFFSET, length);
+ return copy;
+ }
+ }
+}
diff --git a/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryBufferTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryBufferTest.java
new file mode 100644
index 00000000..3796e6b5
--- /dev/null
+++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryBufferTest.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.CoordinateSequence;
+import org.locationtech.jts.geom.CoordinateXYM;
+import org.locationtech.jts.geom.CoordinateXYZM;
+import org.locationtech.jts.geom.impl.CoordinateArraySequence;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class GeometryBufferTest {
+
+ private final String bufferType;
+
+ public GeometryBufferTest(String bufferType) {
+ this.bufferType = bufferType;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<String> testParams() {
+ return Arrays.asList("bytebuffer", "unsafe");
+ }
+
+ @Test
+ public void testPutGetByte() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 1);
+ buffer.putByte(0, (byte) 1);
+ assertEquals((byte) 1, buffer.getByte(0));
+ }
+
+ @Test
+ public void testPutGetBytes() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 4);
+ buffer.putBytes(2, new byte[]{3, 4});
+ buffer.putBytes(0, new byte[]{1, 2});
+ byte[] bytes = new byte[4];
+ buffer.getBytes(bytes, 2, 2);
+ assertEquals((byte) 3, bytes[0]);
+ assertEquals((byte) 4, bytes[1]);
+ buffer.getBytes(bytes, 0, 4);
+ assertEquals((byte) 1, bytes[0]);
+ assertEquals((byte) 2, bytes[1]);
+ assertEquals((byte) 3, bytes[2]);
+ assertEquals((byte) 4, bytes[3]);
+ }
+
+ @Test
+ public void testPutGetInt() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 4);
+ buffer.putInt(0, 1);
+ assertEquals(1, buffer.getInt(0));
+ }
+
+ @Test
+ public void testPutGetCoordinate() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 16);
+ buffer.putCoordinate(0, new Coordinate(1, 2));
+ CoordinateSequence coordinates = buffer.getCoordinate(0);
+ assertEquals(1, coordinates.size());
+ Coordinate coordinate = coordinates.getCoordinate(0);
+ assertEquals(1, coordinate.x, 1e-6);
+ assertEquals(2, coordinate.y, 1e-6);
+ }
+
+ @Test
+ public void testPutGetCoordinateXYZ() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 24);
+ buffer.setCoordinateType(CoordinateType.XYZ);
+ buffer.putCoordinate(0, new Coordinate(1, 2, 3));
+ CoordinateSequence coordinates = buffer.getCoordinate(0);
+ assertEquals(1, coordinates.size());
+ Coordinate coordinate = coordinates.getCoordinate(0);
+ assertEquals(1, coordinate.x, 1e-6);
+ assertEquals(2, coordinate.y, 1e-6);
+ assertEquals(3, coordinate.getZ(), 1e-6);
+ }
+
+ @Test
+ public void testPutGetCoordinateXYM() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 24);
+ buffer.setCoordinateType(CoordinateType.XYM);
+ buffer.putCoordinate(0, new CoordinateXYM(1, 2, 3));
+ CoordinateSequence coordinates = buffer.getCoordinate(0);
+ assertEquals(1, coordinates.size());
+ Coordinate coordinate = coordinates.getCoordinate(0);
+ assertEquals(1, coordinate.x, 1e-6);
+ assertEquals(2, coordinate.y, 1e-6);
+ assertEquals(3, coordinate.getM(), 1e-6);
+ }
+
+ @Test
+ public void testPutGetCoordinateXYZM() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 32);
+ buffer.setCoordinateType(CoordinateType.XYZM);
+ buffer.putCoordinate(0, new CoordinateXYZM(1, 2, 3, 4));
+ CoordinateSequence coordinates = buffer.getCoordinate(0);
+ assertEquals(1, coordinates.size());
+ Coordinate coordinate = coordinates.getCoordinate(0);
+ assertEquals(1, coordinate.x, 1e-6);
+ assertEquals(2, coordinate.y, 1e-6);
+ assertEquals(3, coordinate.getZ(), 1e-6);
+ assertEquals(4, coordinate.getM(), 1e-6);
+ }
+
+ @Test
+ public void testPutGetCoordinateWithUnmatchedCoordType() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 24);
+ buffer.setCoordinateType(CoordinateType.XYZ);
+ buffer.putCoordinate(0, new CoordinateXYM(1, 2, 3));
+ CoordinateSequence coordinates = buffer.getCoordinate(0);
+ assertEquals(1, coordinates.size());
+ Coordinate coordinate = coordinates.getCoordinate(0);
+ assertEquals(1, coordinate.x, 1e-6);
+ assertEquals(2, coordinate.y, 1e-6);
+ assertTrue(Double.isNaN(coordinate.getZ()));
+ }
+
+ @Test
+ public void testPutGetCoordinates() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 32);
+ CoordinateSequence coordinates =
+ new CoordinateArraySequence(new Coordinate[]{new Coordinate(1, 2), new Coordinate(3, 4)});
+ buffer.putCoordinates(0, coordinates);
+ CoordinateSequence coordinates2 = buffer.getCoordinates(0, 2);
+ assertEquals(1, coordinates2.getCoordinate(0).x, 1e-6);
+ assertEquals(2, coordinates2.getCoordinate(0).y, 1e-6);
+ assertEquals(3, coordinates2.getCoordinate(1).x, 1e-6);
+ assertEquals(4, coordinates2.getCoordinate(1).y, 1e-6);
+ }
+
+ @Test
+ public void testPutGetCoordinatesXYZ() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 48);
+ buffer.setCoordinateType(CoordinateType.XYZ);
+ CoordinateSequence coordinates =
+ new CoordinateArraySequence(
+ new Coordinate[]{new Coordinate(1, 2, 3), new Coordinate(4, 5, 6)});
+ buffer.putCoordinates(0, coordinates);
+ CoordinateSequence coordinates2 = buffer.getCoordinates(0, 2);
+ assertEquals(1, coordinates2.getCoordinate(0).x, 1e-6);
+ assertEquals(2, coordinates2.getCoordinate(0).y, 1e-6);
+ assertEquals(3, coordinates2.getCoordinate(0).getZ(), 1e-6);
+ assertEquals(4, coordinates2.getCoordinate(1).x, 1e-6);
+ assertEquals(5, coordinates2.getCoordinate(1).y, 1e-6);
+ assertEquals(6, coordinates2.getCoordinate(1).getZ(), 1e-6);
+ }
+
+ @Test
+ public void testPutGetCoordinatesXYM() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 48);
+ buffer.setCoordinateType(CoordinateType.XYM);
+ CoordinateSequence coordinates =
+ new CoordinateArraySequence(
+ new Coordinate[]{new CoordinateXYM(1, 2, 3), new CoordinateXYM(4, 5, 6)});
+ buffer.putCoordinates(0, coordinates);
+ CoordinateSequence coordinates2 = buffer.getCoordinates(0, 2);
+ assertEquals(1, coordinates2.getCoordinate(0).x, 1e-6);
+ assertEquals(2, coordinates2.getCoordinate(0).y, 1e-6);
+ assertEquals(3, coordinates2.getCoordinate(0).getM(), 1e-6);
+ assertEquals(4, coordinates2.getCoordinate(1).x, 1e-6);
+ assertEquals(5, coordinates2.getCoordinate(1).y, 1e-6);
+ assertEquals(6, coordinates2.getCoordinate(1).getM(), 1e-6);
+ }
+
+ @Test
+ public void testPutGetCoordinatesXYZM() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 64);
+ buffer.setCoordinateType(CoordinateType.XYZM);
+ CoordinateSequence coordinates =
+ new CoordinateArraySequence(
+ new Coordinate[]{new CoordinateXYZM(1, 2, 3, 4), new CoordinateXYZM(5, 6, 7, 8)});
+ buffer.putCoordinates(0, coordinates);
+ CoordinateSequence coordinates2 = buffer.getCoordinates(0, 2);
+ assertEquals(1, coordinates2.getCoordinate(0).x, 1e-6);
+ assertEquals(2, coordinates2.getCoordinate(0).y, 1e-6);
+ assertEquals(3, coordinates2.getCoordinate(0).getZ(), 1e-6);
+ assertEquals(4, coordinates2.getCoordinate(0).getM(), 1e-6);
+ assertEquals(5, coordinates2.getCoordinate(1).x, 1e-6);
+ assertEquals(6, coordinates2.getCoordinate(1).y, 1e-6);
+ assertEquals(7, coordinates2.getCoordinate(1).getZ(), 1e-6);
+ assertEquals(8, coordinates2.getCoordinate(1).getM(), 1e-6);
+ }
+
+ @Test
+ public void testPutGetCoordinatesWithUnmatchedCoordType() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 48);
+ buffer.setCoordinateType(CoordinateType.XYZ);
+ CoordinateSequence coordinates =
+ new CoordinateArraySequence(
+ new Coordinate[]{new CoordinateXYM(1, 2, 3), new CoordinateXYM(4, 5, 6)});
+ buffer.putCoordinates(0, coordinates);
+ CoordinateSequence coordinates2 = buffer.getCoordinates(0, 2);
+ assertEquals(1, coordinates2.getCoordinate(0).x, 1e-6);
+ assertEquals(2, coordinates2.getCoordinate(0).y, 1e-6);
+ assertTrue(Double.isNaN(coordinates2.getCoordinate(0).getZ()));
+ assertEquals(4, coordinates2.getCoordinate(1).x, 1e-6);
+ assertEquals(5, coordinates2.getCoordinate(1).y, 1e-6);
+ assertTrue(Double.isNaN(coordinates2.getCoordinate(1).getZ()));
+ }
+
+ @Test
+ public void testSlice() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 16);
+ buffer.putBytes(0, new byte[]{1, 2, 3, 4});
+ GeometryBuffer slice = buffer.slice(0);
+ assertEquals((byte) 1, slice.getByte(0));
+ assertEquals((byte) 2, slice.getByte(1));
+ assertEquals((byte) 3, slice.getByte(2));
+ assertEquals((byte) 4, slice.getByte(3));
+ slice = slice.slice(1);
+ assertEquals((byte) 2, slice.getByte(0));
+ assertEquals((byte) 4, slice.getByte(2));
+ slice = slice.slice(1);
+ assertEquals((byte) 3, slice.getByte(0));
+ assertEquals((byte) 4, slice.getByte(1));
+ assertEquals(14, slice.getLength());
+ }
+
+ @Test
+ public void testToByteArray() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 2);
+ buffer.putByte(0, (byte) 1);
+ buffer.putByte(1, (byte) 2);
+ byte[] bytes = buffer.toByteArray();
+ assertEquals(2, bytes.length);
+ assertEquals((byte) 1, bytes[0]);
+ assertEquals((byte) 2, bytes[1]);
+ }
+
+ @Test
+ public void testMixedDataTypes() {
+ GeometryBuffer buffer = GeometryBufferFactory.create(bufferType, 28);
+ buffer.putByte(0, (byte) 1);
+ buffer.putInt(4, 100);
+ buffer.putCoordinate(8, new Coordinate(10, 20));
+ buffer.putInt(24, 200);
+ assertEquals((byte) 1, buffer.getByte(0));
+ assertEquals(100, buffer.getInt(4));
+ assertEquals(10, buffer.getCoordinate(8).getCoordinate(0).x, 1e-6);
+ assertEquals(20, buffer.getCoordinate(8).getCoordinate(0).y, 1e-6);
+ assertEquals(200, buffer.getInt(24));
+ }
+
+ @Test
+ public void testWrap() {
+ byte[] bytes = new byte[]{1, 2, 3, 4};
+ GeometryBuffer buffer = GeometryBufferFactory.wrap(bufferType, bytes);
+ assertEquals(4, buffer.getLength());
+ assertEquals((byte) 1, buffer.getByte(0));
+ assertEquals((byte) 2, buffer.getByte(1));
+ assertEquals((byte) 3, buffer.getByte(2));
+ assertEquals((byte) 4, buffer.getByte(3));
+ }
+}
diff --git a/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryCollectionSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryCollectionSerdeTest.java
new file mode 100644
index 00000000..ce84e69a
--- /dev/null
+++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/GeometryCollectionSerdeTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+import org.junit.Test;
+import org.junit.Assert;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryCollection;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.geom.LineString;
+import org.locationtech.jts.geom.MultiLineString;
+import org.locationtech.jts.geom.MultiPoint;
+import org.locationtech.jts.geom.MultiPolygon;
+import org.locationtech.jts.geom.Point;
+import org.locationtech.jts.geom.Polygon;
+
+public class GeometryCollectionSerdeTest {
+ private static final GeometryFactory gf = new GeometryFactory();
+
+ @Test
+ public void testEmptyGeometryCollection() {
+ GeometryCollection geometryCollection = gf.createGeometryCollection();
+ geometryCollection.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(geometryCollection);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof GeometryCollection);
+ Assert.assertTrue(geom.isEmpty());
+ Assert.assertEquals(4326, geom.getSRID());
+ }
+
+ @Test
+ public void testGeometryCollection() {
+ Point point = gf.createPoint(new Coordinate(10, 20));
+ LineString lineString =
+ gf.createLineString(new Coordinate[]{new Coordinate(0, 0), new Coordinate(1, 1)});
+ Polygon polygon =
+ gf.createPolygon(
+ gf.createLinearRing(
+ new Coordinate[]{
+ new Coordinate(0, 0),
+ new Coordinate(0, 1),
+ new Coordinate(1, 1),
+ new Coordinate(1, 0),
+ new Coordinate(0, 0)
+ }),
+ null);
+ MultiPoint multiPoint =
+ gf.createMultiPointFromCoords(
+ new Coordinate[]{
+ new Coordinate(10, 20), new Coordinate(30, 40), new Coordinate(50, 60)
+ });
+ MultiLineString multiLineString =
+ gf.createMultiLineString(new LineString[]{lineString, lineString, lineString});
+ MultiPolygon multiPolygon = gf.createMultiPolygon(new Polygon[]{polygon, polygon});
+ GeometryCollection geometryCollection =
+ gf.createGeometryCollection(
+ new Geometry[]{
+ gf.createPoint(),
+ gf.createLineString(),
+ gf.createPolygon(),
+ point,
+ lineString,
+ polygon,
+ gf.createMultiPoint(),
+ gf.createMultiLineString(),
+ gf.createMultiPolygon(),
+ multiPoint,
+ multiLineString,
+ multiPolygon
+ });
+ geometryCollection.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(geometryCollection);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof GeometryCollection);
+ Assert.assertEquals(4326, geom.getSRID());
+ Assert.assertEquals(geometryCollection.getNumGeometries(), geom.getNumGeometries());
+ for (int k = 0; k < geom.getNumGeometries(); k++) {
+ Assert.assertEquals(geometryCollection.getGeometryN(k), geom.getGeometryN(k));
+ }
+ }
+
+ @Test
+ public void testNestedGeometryCollection() {
+ Point point = gf.createPoint(new Coordinate(10, 20));
+ LineString lineString =
+ gf.createLineString(new Coordinate[]{new Coordinate(0, 0), new Coordinate(1, 1)});
+ MultiLineString multiLineString =
+ gf.createMultiLineString(new LineString[]{lineString, lineString, lineString});
+ GeometryCollection geomCollection1 =
+ gf.createGeometryCollection(new Geometry[]{point, lineString, multiLineString});
+ GeometryCollection geomCollection2 =
+ gf.createGeometryCollection(
+ new Geometry[]{
+ point, geomCollection1, geomCollection1, multiLineString, geomCollection1
+ });
+ geomCollection2.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(geomCollection2);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof GeometryCollection);
+ Assert.assertEquals(4326, geom.getSRID());
+ Assert.assertEquals(geomCollection2.getNumGeometries(), geom.getNumGeometries());
+ for (int k = 0; k < geom.getNumGeometries(); k++) {
+ Assert.assertEquals(geomCollection2.getGeometryN(k), geom.getGeometryN(k));
+ }
+ }
+}
diff --git a/common/src/test/java/org/apache/sedona/common/geometrySerde/LineStringSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/LineStringSerdeTest.java
new file mode 100644
index 00000000..0679152f
--- /dev/null
+++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/LineStringSerdeTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+import org.junit.Test;
+import org.junit.Assert;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.CoordinateSequence;
+import org.locationtech.jts.geom.CoordinateXYM;
+import org.locationtech.jts.geom.CoordinateXYZM;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.geom.LineString;
+
+public class LineStringSerdeTest {
+ private static final GeometryFactory gf = new GeometryFactory();
+
+ @Test
+ public void testEmptyLineString() {
+ LineString lineString = gf.createLineString();
+ lineString.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(lineString);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof LineString);
+ Assert.assertTrue(geom.isEmpty());
+ Assert.assertEquals(4326, geom.getSRID());
+ }
+
+ @Test
+ public void testLineString() {
+ Coordinate[] coordinates =
+ new Coordinate[]{
+ new Coordinate(1.0, 2.0), new Coordinate(3.0, 4.0), new Coordinate(5.0, 6.0),
+ };
+ LineString lineString = gf.createLineString(coordinates);
+ lineString.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(lineString);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof LineString);
+ Assert.assertEquals(4326, geom.getSRID());
+ Assert.assertEquals(3, geom.getNumPoints());
+ CoordinateSequence coordSeq = ((LineString) geom).getCoordinateSequence();
+ Coordinate coord = coordSeq.getCoordinate(0);
+ Assert.assertEquals(1.0, coord.x, 1e-6);
+ Assert.assertEquals(2.0, coord.y, 1e-6);
+ Assert.assertTrue(Double.isNaN(coord.getZ()));
+ Assert.assertTrue(Double.isNaN(coord.getM()));
+ coord = coordSeq.getCoordinate(1);
+ Assert.assertEquals(3.0, coord.x, 1e-6);
+ Assert.assertEquals(4.0, coord.y, 1e-6);
+ coord = coordSeq.getCoordinate(2);
+ Assert.assertEquals(5.0, coord.x, 1e-6);
+ Assert.assertEquals(6.0, coord.y, 1e-6);
+ }
+
+ @Test
+ public void testLineStringXYZ() {
+ Coordinate[] coordinates =
+ new Coordinate[]{
+ new Coordinate(1.0, 2.0, 3.0), new Coordinate(4.0, 5.0, 6.0),
+ };
+ LineString lineString = gf.createLineString(coordinates);
+ lineString.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(lineString);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof LineString);
+ Assert.assertEquals(4326, geom.getSRID());
+ Assert.assertEquals(2, geom.getNumPoints());
+ CoordinateSequence coordSeq = ((LineString) geom).getCoordinateSequence();
+ Coordinate coord = coordSeq.getCoordinate(0);
+ Assert.assertEquals(1.0, coord.x, 1e-6);
+ Assert.assertEquals(2.0, coord.y, 1e-6);
+ Assert.assertEquals(3.0, coord.getZ(), 1e-6);
+ Assert.assertTrue(Double.isNaN(coord.getM()));
+ coord = coordSeq.getCoordinate(1);
+ Assert.assertEquals(4.0, coord.x, 1e-6);
+ Assert.assertEquals(5.0, coord.y, 1e-6);
+ Assert.assertEquals(6.0, coord.getZ(), 1e-6);
+ }
+
+ @Test
+ public void testLineStringXYM() {
+ Coordinate[] coordinates =
+ new Coordinate[]{
+ new CoordinateXYM(1.0, 2.0, 3.0), new CoordinateXYM(4.0, 5.0, 6.0),
+ };
+ LineString lineString = gf.createLineString(coordinates);
+ lineString.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(lineString);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof LineString);
+ Assert.assertEquals(4326, geom.getSRID());
+ Assert.assertEquals(2, geom.getNumPoints());
+ CoordinateSequence coordSeq = ((LineString) geom).getCoordinateSequence();
+ Coordinate coord = coordSeq.getCoordinate(0);
+ Assert.assertEquals(1.0, coord.x, 1e-6);
+ Assert.assertEquals(2.0, coord.y, 1e-6);
+ Assert.assertTrue(Double.isNaN(coord.getZ()));
+ Assert.assertEquals(3.0, coord.getM(), 1e-6);
+ coord = coordSeq.getCoordinate(1);
+ Assert.assertEquals(4.0, coord.x, 1e-6);
+ Assert.assertEquals(5.0, coord.y, 1e-6);
+ Assert.assertTrue(Double.isNaN(coord.getZ()));
+ Assert.assertEquals(6.0, coord.getM(), 1e-6);
+ }
+
+ @Test
+ public void testLineStringXYZM() {
+ Coordinate[] coordinates =
+ new Coordinate[]{
+ new CoordinateXYZM(1.0, 2.0, 3.0, 4.0), new CoordinateXYZM(5.0, 6.0, 7.0, 8.0),
+ };
+ LineString lineString = gf.createLineString(coordinates);
+ lineString.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(lineString);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof LineString);
+ Assert.assertEquals(4326, geom.getSRID());
+ Assert.assertEquals(2, geom.getNumPoints());
+ CoordinateSequence coordSeq = ((LineString) geom).getCoordinateSequence();
+ Coordinate coord = coordSeq.getCoordinate(0);
+ Assert.assertEquals(1.0, coord.x, 1e-6);
+ Assert.assertEquals(2.0, coord.y, 1e-6);
+ Assert.assertEquals(3.0, coord.getZ(), 1e-6);
+ Assert.assertEquals(4.0, coord.getM(), 1e-6);
+ coord = coordSeq.getCoordinate(1);
+ Assert.assertEquals(5.0, coord.x, 1e-6);
+ Assert.assertEquals(6.0, coord.y, 1e-6);
+ Assert.assertEquals(7.0, coord.getZ(), 1e-6);
+ Assert.assertEquals(8.0, coord.getM(), 1e-6);
+ }
+}
diff --git a/common/src/test/java/org/apache/sedona/common/geometrySerde/MultiLineStringSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/MultiLineStringSerdeTest.java
new file mode 100644
index 00000000..72e5637f
--- /dev/null
+++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/MultiLineStringSerdeTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+import org.junit.Test;
+import org.junit.Assert;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.geom.LineString;
+import org.locationtech.jts.geom.MultiLineString;
+
+public class MultiLineStringSerdeTest {
+ private static final GeometryFactory gf = new GeometryFactory();
+
+ @Test
+ public void testEmptyMultiLineString() {
+ MultiLineString multiLineString = gf.createMultiLineString();
+ multiLineString.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(multiLineString);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof MultiLineString);
+ Assert.assertTrue(geom.isEmpty());
+ Assert.assertEquals(4326, geom.getSRID());
+ }
+
+ @Test
+ public void testMultiLineString() {
+ MultiLineString multiLineString =
+ gf.createMultiLineString(
+ new LineString[]{
+ gf.createLineString(
+ new Coordinate[]{
+ new Coordinate(1, 2), new Coordinate(3, 4), new Coordinate(5, 6),
+ }),
+ gf.createLineString(),
+ gf.createLineString(
+ new Coordinate[]{
+ new Coordinate(7, 8), new Coordinate(9, 10), new Coordinate(11, 12),
+ }),
+ });
+ multiLineString.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(multiLineString);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof MultiLineString);
+ Assert.assertEquals(4326, geom.getSRID());
+ MultiLineString multiLineString2 = (MultiLineString) geom;
+ Assert.assertEquals(3, multiLineString2.getNumGeometries());
+ Assert.assertEquals(multiLineString, multiLineString2);
+ }
+}
diff --git a/common/src/test/java/org/apache/sedona/common/geometrySerde/MultiPointSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/MultiPointSerdeTest.java
new file mode 100644
index 00000000..9beb2590
--- /dev/null
+++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/MultiPointSerdeTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+import org.junit.Test;
+import org.junit.Assert;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.CoordinateXYM;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.geom.MultiPoint;
+import org.locationtech.jts.geom.Point;
+
+public class MultiPointSerdeTest {
+ private static final GeometryFactory gf = new GeometryFactory();
+
+ @Test
+ public void testEmptyMultiPoint() {
+ MultiPoint multiPoint = gf.createMultiPoint();
+ multiPoint.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(multiPoint);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof MultiPoint);
+ Assert.assertTrue(geom.isEmpty());
+ Assert.assertEquals(4326, geom.getSRID());
+ }
+
+ @Test
+ public void testMultiPoint() {
+ MultiPoint multiPoint =
+ gf.createMultiPointFromCoords(
+ new Coordinate[]{
+ new Coordinate(1, 2), new Coordinate(3, 4), new Coordinate(5, 6),
+ });
+ multiPoint.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(multiPoint);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof MultiPoint);
+ Assert.assertEquals(4326, geom.getSRID());
+ MultiPoint multiPoint2 = (MultiPoint) geom;
+ Assert.assertEquals(3, multiPoint2.getNumGeometries());
+ Assert.assertEquals(1, multiPoint2.getGeometryN(0).getCoordinate().x, 1e-6);
+ Assert.assertEquals(2, multiPoint2.getGeometryN(0).getCoordinate().y, 1e-6);
+ Assert.assertEquals(3, multiPoint2.getGeometryN(1).getCoordinate().x, 1e-6);
+ Assert.assertEquals(4, multiPoint2.getGeometryN(1).getCoordinate().y, 1e-6);
+ Assert.assertEquals(5, multiPoint2.getGeometryN(2).getCoordinate().x, 1e-6);
+ Assert.assertEquals(6, multiPoint2.getGeometryN(2).getCoordinate().y, 1e-6);
+ }
+
+ @Test
+ public void testMultiPointWithEmptyPoints() {
+ Point[] points =
+ new Point[]{
+ gf.createPoint(new Coordinate(1, 2)),
+ gf.createPoint(),
+ gf.createPoint(new Coordinate(3, 4))
+ };
+ MultiPoint multiPoint = gf.createMultiPoint(points);
+ byte[] bytes = GeometrySerializer.serialize(multiPoint);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof MultiPoint);
+ Assert.assertEquals(0, geom.getSRID());
+ MultiPoint multiPoint2 = (MultiPoint) geom;
+ Assert.assertEquals(3, multiPoint2.getNumGeometries());
+ Assert.assertEquals(1, multiPoint2.getGeometryN(0).getCoordinate().x, 1e-6);
+ Assert.assertEquals(2, multiPoint2.getGeometryN(0).getCoordinate().y, 1e-6);
+ Assert.assertTrue(multiPoint2.getGeometryN(1).isEmpty());
+ Assert.assertEquals(3, multiPoint2.getGeometryN(2).getCoordinate().x, 1e-6);
+ Assert.assertEquals(4, multiPoint2.getGeometryN(2).getCoordinate().y, 1e-6);
+ }
+
+ @Test
+ public void testMultiPointXYM() {
+ MultiPoint multiPoint =
+ gf.createMultiPointFromCoords(
+ new Coordinate[]{
+ new CoordinateXYM(1, 2, 3), new CoordinateXYM(4, 5, 6), new CoordinateXYM(7, 8, 9),
+ });
+ multiPoint.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(multiPoint);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof MultiPoint);
+ Assert.assertEquals(4326, geom.getSRID());
+ MultiPoint multiPoint2 = (MultiPoint) geom;
+ Assert.assertEquals(3, multiPoint2.getNumGeometries());
+ Assert.assertEquals(1, multiPoint2.getGeometryN(0).getCoordinate().x, 1e-6);
+ Assert.assertEquals(2, multiPoint2.getGeometryN(0).getCoordinate().y, 1e-6);
+ Assert.assertEquals(3, multiPoint2.getGeometryN(0).getCoordinate().getM(), 1e-6);
+ Assert.assertEquals(4, multiPoint2.getGeometryN(1).getCoordinate().x, 1e-6);
+ Assert.assertEquals(5, multiPoint2.getGeometryN(1).getCoordinate().y, 1e-6);
+ Assert.assertEquals(6, multiPoint2.getGeometryN(1).getCoordinate().getM(), 1e-6);
+ Assert.assertEquals(7, multiPoint2.getGeometryN(2).getCoordinate().x, 1e-6);
+ Assert.assertEquals(8, multiPoint2.getGeometryN(2).getCoordinate().y, 1e-6);
+ Assert.assertEquals(9, multiPoint2.getGeometryN(2).getCoordinate().getM(), 1e-6);
+ }
+}
diff --git a/common/src/test/java/org/apache/sedona/common/geometrySerde/MultiPolygonSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/MultiPolygonSerdeTest.java
new file mode 100644
index 00000000..1fed8a47
--- /dev/null
+++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/MultiPolygonSerdeTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+import org.junit.Test;
+import org.junit.Assert;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.geom.LinearRing;
+import org.locationtech.jts.geom.MultiPolygon;
+import org.locationtech.jts.geom.Polygon;
+
+public class MultiPolygonSerdeTest {
+ private static final GeometryFactory gf = new GeometryFactory();
+
+ @Test
+ public void testEmptyMultiPolygon() {
+ MultiPolygon multiPolygon = gf.createMultiPolygon();
+ multiPolygon.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(multiPolygon);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof MultiPolygon);
+ Assert.assertTrue(geom.isEmpty());
+ Assert.assertEquals(4326, geom.getSRID());
+ }
+
+ @Test
+ public void testMultiPolygon() {
+ LinearRing shell =
+ gf.createLinearRing(
+ new Coordinate[]{
+ new Coordinate(0, 0),
+ new Coordinate(0, 1),
+ new Coordinate(1, 1),
+ new Coordinate(1, 0),
+ new Coordinate(0, 0)
+ });
+ LinearRing hole1 =
+ gf.createLinearRing(
+ new Coordinate[]{
+ new Coordinate(0.1, 0.1),
+ new Coordinate(0.1, 0.2),
+ new Coordinate(0.2, 0.2),
+ new Coordinate(0.2, 0.1),
+ new Coordinate(0.1, 0.1)
+ });
+ LinearRing hole2 =
+ gf.createLinearRing(
+ new Coordinate[]{
+ new Coordinate(0.3, 0.3),
+ new Coordinate(0.3, 0.4),
+ new Coordinate(0.4, 0.4),
+ new Coordinate(0.4, 0.3),
+ new Coordinate(0.3, 0.3)
+ });
+ LinearRing[] holes = new LinearRing[]{hole1, hole2};
+ MultiPolygon multiPolygon =
+ gf.createMultiPolygon(
+ new Polygon[]{
+ gf.createPolygon(shell), gf.createPolygon(), gf.createPolygon(shell, holes)
+ });
+ multiPolygon.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(multiPolygon);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof MultiPolygon);
+ Assert.assertEquals(4326, geom.getSRID());
+ MultiPolygon mp = (MultiPolygon) geom;
+ Assert.assertEquals(3, mp.getNumGeometries());
+ Assert.assertEquals(multiPolygon, mp);
+ }
+}
diff --git a/common/src/test/java/org/apache/sedona/common/geometrySerde/PointSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/PointSerdeTest.java
new file mode 100644
index 00000000..ba7bab08
--- /dev/null
+++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/PointSerdeTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+import org.junit.Test;
+import org.junit.Assert;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.CoordinateXYM;
+import org.locationtech.jts.geom.CoordinateXYZM;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.geom.Point;
+
+public class PointSerdeTest {
+ private static final GeometryFactory gf = new GeometryFactory();
+
+ @Test
+ public void testEmptyPoint() {
+ byte[] bytes = GeometrySerializer.serialize(gf.createPoint());
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof Point);
+ Assert.assertTrue(geom.isEmpty());
+ Assert.assertEquals(0, geom.getSRID());
+ }
+
+ @Test
+ public void testEmptyPointWithSRID() {
+ Point point = gf.createPoint();
+ point.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(point);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof Point);
+ Assert.assertTrue(geom.isEmpty());
+ Assert.assertEquals(4326, geom.getSRID());
+ }
+
+ @Test
+ public void test2DPoint() {
+ Point point = gf.createPoint(new Coordinate(1.0, 2.0));
+ point.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(point);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof Point);
+ Assert.assertFalse(geom.isEmpty());
+ Assert.assertEquals(4326, geom.getSRID());
+ Assert.assertEquals(1.0, geom.getCoordinate().x, 1e-6);
+ Assert.assertEquals(2.0, geom.getCoordinate().y, 1e-6);
+ Assert.assertTrue(Double.isNaN(geom.getCoordinate().getZ()));
+ Assert.assertTrue(Double.isNaN(geom.getCoordinate().getM()));
+ }
+
+ @Test
+ public void testXYZPoint() {
+ Point point = gf.createPoint(new Coordinate(1.0, 2.0, 3.0));
+ point.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(point);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof Point);
+ Assert.assertFalse(geom.isEmpty());
+ Assert.assertEquals(4326, geom.getSRID());
+ Assert.assertEquals(1.0, geom.getCoordinate().x, 1e-6);
+ Assert.assertEquals(2.0, geom.getCoordinate().y, 1e-6);
+ Assert.assertEquals(3.0, geom.getCoordinate().getZ(), 1e-6);
+ Assert.assertTrue(Double.isNaN(geom.getCoordinate().getM()));
+ }
+
+ @Test
+ public void testXYMPoint() {
+ Point point = gf.createPoint(new CoordinateXYM(1.0, 2.0, 3.0));
+ byte[] bytes = GeometrySerializer.serialize(point);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof Point);
+ Assert.assertFalse(geom.isEmpty());
+ Assert.assertEquals(0, geom.getSRID());
+ Assert.assertEquals(1.0, geom.getCoordinate().x, 1e-6);
+ Assert.assertEquals(2.0, geom.getCoordinate().y, 1e-6);
+ Assert.assertTrue(Double.isNaN(geom.getCoordinate().getZ()));
+ Assert.assertEquals(3.0, geom.getCoordinate().getM(), 1e-6);
+ }
+
+ @Test
+ public void testXYZMPoint() {
+ Point point = gf.createPoint(new CoordinateXYZM(1.0, 2.0, 3.0, 4.0));
+ point.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(point);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof Point);
+ Assert.assertFalse(geom.isEmpty());
+ Assert.assertEquals(4326, geom.getSRID());
+ Assert.assertEquals(1.0, geom.getCoordinate().x, 1e-6);
+ Assert.assertEquals(2.0, geom.getCoordinate().y, 1e-6);
+ Assert.assertEquals(3.0, geom.getCoordinate().getZ(), 1e-6);
+ Assert.assertEquals(4.0, geom.getCoordinate().getM(), 1e-6);
+ }
+}
diff --git a/common/src/test/java/org/apache/sedona/common/geometrySerde/PolygonSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/PolygonSerdeTest.java
new file mode 100644
index 00000000..75ef4fc9
--- /dev/null
+++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/PolygonSerdeTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.common.geometrySerde;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.geom.LinearRing;
+import org.locationtech.jts.geom.Polygon;
+
+public class PolygonSerdeTest {
+ private static final GeometryFactory gf = new GeometryFactory();
+
+ @Test
+ public void testEmptyPolygon() {
+ Polygon polygon = gf.createPolygon();
+ polygon.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(polygon);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof Polygon);
+ Assert.assertTrue(geom.isEmpty());
+ Assert.assertEquals(4326, geom.getSRID());
+ }
+
+ @Test
+ public void testPolygonWithoutHoles() {
+ LinearRing shell =
+ gf.createLinearRing(
+ new Coordinate[]{
+ new Coordinate(0, 0),
+ new Coordinate(0, 1),
+ new Coordinate(1, 1),
+ new Coordinate(1, 0),
+ new Coordinate(0, 0)
+ });
+ Polygon polygon = gf.createPolygon(shell);
+ polygon.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(polygon);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof Polygon);
+ Assert.assertEquals(4326, geom.getSRID());
+ Assert.assertEquals(polygon, geom);
+ }
+
+ @Test
+ public void testPolygonWithHoles() {
+ LinearRing shell =
+ gf.createLinearRing(
+ new Coordinate[]{
+ new Coordinate(0, 0),
+ new Coordinate(0, 1),
+ new Coordinate(1, 1),
+ new Coordinate(1, 0),
+ new Coordinate(0, 0)
+ });
+ LinearRing hole1 =
+ gf.createLinearRing(
+ new Coordinate[]{
+ new Coordinate(0.1, 0.1),
+ new Coordinate(0.1, 0.2),
+ new Coordinate(0.2, 0.2),
+ new Coordinate(0.2, 0.1),
+ new Coordinate(0.1, 0.1)
+ });
+ LinearRing hole2 =
+ gf.createLinearRing(
+ new Coordinate[]{
+ new Coordinate(0.3, 0.3),
+ new Coordinate(0.3, 0.4),
+ new Coordinate(0.4, 0.4),
+ new Coordinate(0.4, 0.3),
+ new Coordinate(0.3, 0.3)
+ });
+ Polygon polygon = gf.createPolygon(shell, new LinearRing[]{hole1, hole2});
+ polygon.setSRID(4326);
+ byte[] bytes = GeometrySerializer.serialize(polygon);
+ Geometry geom = GeometrySerializer.deserialize(bytes);
+ Assert.assertTrue(geom instanceof Polygon);
+ Assert.assertEquals(4326, geom.getSRID());
+ Assert.assertEquals(polygon, geom);
+ }
+}
diff --git a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java b/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
index b826a875..e4aa6c0d 100755
--- a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
+++ b/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
@@ -24,17 +24,12 @@ import com.esotericsoftware.kryo.Registration;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.log4j.Logger;
import org.apache.sedona.common.geometryObjects.Circle;
-import org.apache.sedona.core.formatMapper.shapefileParser.parseUtils.shp.ShapeSerde;
+import org.apache.sedona.common.geometrySerde.GeometrySerializer;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryCollection;
-import org.locationtech.jts.geom.GeometryFactory;
import org.locationtech.jts.geom.LineString;
-import org.locationtech.jts.geom.MultiLineString;
-import org.locationtech.jts.geom.MultiPoint;
-import org.locationtech.jts.geom.MultiPolygon;
import org.locationtech.jts.geom.Point;
import org.locationtech.jts.geom.Polygon;
@@ -52,9 +47,6 @@ import java.io.Serializable;
public class GeometrySerde
extends Serializer implements Serializable
{
-
- private static final GeometryFactory geometryFactory = new GeometryFactory();
-
@Override
public void write(Kryo kryo, Output out, Object object)
{
@@ -66,20 +58,10 @@ public class GeometrySerde
writeUserData(kryo, out, circle);
}
else if (object instanceof Point || object instanceof LineString
- || object instanceof Polygon || object instanceof MultiPoint
- || object instanceof MultiLineString || object instanceof MultiPolygon) {
+ || object instanceof Polygon || object instanceof GeometryCollection) {
writeType(out, Type.SHAPE);
writeGeometry(kryo, out, (Geometry) object);
}
- else if (object instanceof GeometryCollection) {
- GeometryCollection collection = (GeometryCollection) object;
- writeType(out, Type.GEOMETRYCOLLECTION);
- out.writeInt(collection.getNumGeometries());
- for (int i = 0; i < collection.getNumGeometries(); i++) {
- writeGeometry(kryo, out, collection.getGeometryN(i));
- }
- writeUserData(kryo, out, collection);
- }
else if (object instanceof Envelope) {
Envelope envelope = (Envelope) object;
writeType(out, Type.ENVELOPE);
@@ -101,7 +83,8 @@ public class GeometrySerde
private void writeGeometry(Kryo kryo, Output out, Geometry geometry)
{
- byte[] data = ShapeSerde.serialize(geometry);
+ byte[] data = GeometrySerializer.serialize(geometry);
+ out.writeInt(data.length);
out.write(data, 0, data.length);
writeUserData(kryo, out, geometry);
}
@@ -132,16 +115,6 @@ public class GeometrySerde
circle.setUserData(userData);
return circle;
}
- case GEOMETRYCOLLECTION: {
- int numGeometries = input.readInt();
- Geometry[] geometries = new Geometry[numGeometries];
- for (int i = 0; i < numGeometries; i++) {
- geometries[i] = readGeometry(kryo, input);
- }
- GeometryCollection collection = geometryFactory.createGeometryCollection(geometries);
- collection.setUserData(readUserData(kryo, input));
- return collection;
- }
case ENVELOPE: {
double xMin = input.readDouble();
double xMax = input.readDouble();
@@ -167,7 +140,10 @@ public class GeometrySerde
private Geometry readGeometry(Kryo kryo, Input input)
{
- Geometry geometry = ShapeSerde.deserialize(input, geometryFactory);
+ int length = input.readInt();
+ byte[] bytes = new byte[length];
+ input.readBytes(bytes);
+ Geometry geometry = GeometrySerializer.deserialize(bytes);
geometry.setUserData(readUserData(kryo, input));
return geometry;
}
@@ -176,8 +152,7 @@ public class GeometrySerde
{
SHAPE(0),
CIRCLE(1),
- GEOMETRYCOLLECTION(2),
- ENVELOPE(3);
+ ENVELOPE(2);
private final int id;
diff --git a/python/sedona/sql/types.py b/python/sedona/sql/types.py
index cd00bdf0..f38d3f0d 100644
--- a/python/sedona/sql/types.py
+++ b/python/sedona/sql/types.py
@@ -15,10 +15,9 @@
# specific language governing permissions and limitations
# under the License.
-import struct
-
from pyspark.sql.types import UserDefinedType, BinaryType
-from shapely.wkb import dumps, loads
+
+from ..utils import geometry_serde
class GeometryType(UserDefinedType):
@@ -28,10 +27,11 @@ class GeometryType(UserDefinedType):
return BinaryType()
def serialize(self, obj):
- return dumps(obj)
+ return geometry_serde.serialize(obj)
def deserialize(self, datum):
- return loads(bytes(datum))
+ geom, offset = geometry_serde.deserialize(datum)
+ return geom
@classmethod
def module(cls):
diff --git a/python/sedona/utils/geometry_serde.py b/python/sedona/utils/geometry_serde.py
new file mode 100644
index 00000000..97affd59
--- /dev/null
+++ b/python/sedona/utils/geometry_serde.py
@@ -0,0 +1,490 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import struct
+import math
+from typing import List, Optional
+
+from shapely.coords import CoordinateSequence
+from shapely.geometry import Point
+from shapely.geometry import LineString
+from shapely.geometry import Polygon
+from shapely.geometry import LinearRing
+from shapely.geometry import MultiPoint
+from shapely.geometry import MultiLineString
+from shapely.geometry import MultiPolygon
+from shapely.geometry import GeometryCollection
+from shapely.wkt import loads as wkt_loads
+
+
+class GeometryTypeID:
+ """
+ Constants used to identify the geometry type in the serialized bytearray of geometry.
+ """
+ POINT = 1
+ LINESTRING = 2
+ POLYGON = 3
+ MULTIPOINT = 4
+ MULTILINESTRING = 5
+ MULTIPOLYGON = 6
+ GEOMETRYCOLLECTION = 7
+
+
+class CoordinateType:
+ """
+ Constants used to identify geometry dimensions in the serialized bytearray of geometry.
+ """
+ XY = 1
+ XYZ = 2
+ XYM = 3
+ XYZM = 4
+
+ BYTES_PER_COORDINATE = [16, 24, 24, 32]
+
+ @staticmethod
+ def type_of(coords: CoordinateSequence, has_z: bool) -> int:
+ if not coords:
+ return CoordinateType.XY
+ coord = coords[0]
+ len_coord = len(coord)
+ if len_coord == 2:
+ return CoordinateType.XY
+ elif len_coord == 3:
+ if has_z:
+ return CoordinateType.XYZ
+ else:
+ # This branch will never be reached since the current version of shapely
+ # does not support M dimension
+ return CoordinateType.XYM
+ elif len_coord == 4:
+ # This branch will never be reached since the current version of shapely
+ # does not support M dimension
+ return CoordinateType.XYZM
+ else:
+ raise ValueError("Invalid coordinate dimension: {}".format(coord))
+
+ @staticmethod
+ def bytes_per_coord(coord_type: int) -> int:
+ return CoordinateType.BYTES_PER_COORDINATE[coord_type - 1]
+
+
+class GeometryBuffer:
+ buffer: bytearray
+ coord_type: int
+ bytes_per_coord: int
+ num_coords: int
+ coords_offset: int
+ ints_offset: int
+
+ def __init__(self, buffer: bytearray, coord_type: int, coords_offset: int, num_coords: int):
+ self.buffer = buffer
+ self.coord_type = coord_type
+ self.bytes_per_coord = CoordinateType.bytes_per_coord(coord_type)
+ self.num_coords = num_coords
+ self.coords_offset = coords_offset
+ self.ints_offset = coords_offset + self.bytes_per_coord * num_coords
+
+ def read_linestring(self) -> LineString:
+ num_points = self.read_int()
+ return LineString(self.read_coordinates(num_points))
+
+ def read_linearring(self) -> LineString:
+ num_points = self.read_int()
+ return LinearRing(self.read_coordinates(num_points))
+
+ def read_polygon(self) -> Polygon:
+ num_rings = self.read_int()
+ if num_rings == 0:
+ return Polygon()
+ exterior = self.read_linearring()
+ interiors = []
+ for i in range(num_rings - 1):
+ interiors.append(self.read_linearring())
+ return Polygon(exterior, holes=interiors)
+
+ def write_linestring(self, line):
+ coords = line.coords
+ self.write_int(len(coords))
+ self.coords_offset = put_coordinates(self.buffer, self.coords_offset, self.coord_type, coords)
+
+ def write_polygon(self, polygon: Polygon):
+ exterior = polygon.exterior
+ if not exterior.coords:
+ self.write_int(0)
+ return
+ self.write_int(len(polygon.interiors) + 1)
+ self.write_linestring(exterior)
+ for interior in polygon.interiors:
+ self.write_linestring(interior)
+
+ def read_coordinates(self, num_coords: int) -> List[tuple]:
+ coords = get_coordinates(self.buffer, self.coords_offset, self.coord_type, num_coords)
+ self.coords_offset += num_coords * self.bytes_per_coord
+ return coords
+
+ def read_coordinate(self) -> tuple:
+ coord = get_coordinate(self.buffer, self.coords_offset, self.coord_type)
+ self.coords_offset += self.bytes_per_coord
+ return coord
+
+ def read_int(self) -> int:
+ value = struct.unpack_from("i", self.buffer, self.ints_offset)[0]
+ self.ints_offset += 4
+ return value
+
+ def write_int(self, value: int):
+ struct.pack_into("i", self.buffer, self.ints_offset, value)
+ self.ints_offset += 4
+
+
+def serialize(geom) -> Optional[bytearray]:
+ """
+ Serialize a shapely geometry object to the internal representation of GeometryUDT.
+ :param geom: shapely geometry object
+ :return: internal representation of GeometryUDT
+ """
+ if geom is None:
+ return None
+ geom_type = geom.geom_type
+ if geom_type == 'Point':
+ return serialize_point(geom)
+ elif geom_type == 'LineString':
+ return serialize_linestring(geom)
+ elif geom_type == 'Polygon':
+ return serialize_polygon(geom)
+ elif geom_type == 'MultiPoint':
+ return serialize_multi_point(geom)
+ elif geom_type == 'MultiLineString':
+ return serialize_multi_linestring(geom)
+ elif geom_type == 'MultiPolygon':
+ return serialize_multi_polygon(geom)
+ elif geom_type == 'GeometryCollection':
+ return serialize_geometry_collection(geom)
+ else:
+ raise ValueError("Unsupported geometry type: " + geom_type)
+
+
+def deserialize(buffer: bytearray):
+ """
+ Deserialize a shapely geometry object from the internal representation of GeometryUDT.
+ :param buffer: internal representation of GeometryUDT
+ :return: shapely geometry object
+ """
+ if buffer is None:
+ return None
+ preamble_byte = buffer[0]
+ geom_type = (preamble_byte >> 4) & 0x0F
+ coord_type = (preamble_byte >> 1) & 0x07
+ num_coords = struct.unpack_from('i', buffer, 4)[0]
+ geom_buffer = GeometryBuffer(buffer, coord_type, 8, num_coords)
+ if geom_type == GeometryTypeID.POINT:
+ geom = deserialize_point(geom_buffer)
+ elif geom_type == GeometryTypeID.LINESTRING:
+ geom = deserialize_linestring(geom_buffer)
+ elif geom_type == GeometryTypeID.POLYGON:
+ geom = deserialize_polygon(geom_buffer)
+ elif geom_type == GeometryTypeID.MULTIPOINT:
+ geom = deserialize_multi_point(geom_buffer)
+ elif geom_type == GeometryTypeID.MULTILINESTRING:
+ geom = deserialize_multi_linestring(geom_buffer)
+ elif geom_type == GeometryTypeID.MULTIPOLYGON:
+ geom = deserialize_multi_polygon(geom_buffer)
+ elif geom_type == GeometryTypeID.GEOMETRYCOLLECTION:
+ geom = deserialize_geometry_collection(geom_buffer)
+ else:
+ raise ValueError("Unsupported geometry type ID: {}".format(geom_type))
+ return geom, geom_buffer.ints_offset
+
+
+def create_buffer_for_geom(geom_type: int, coord_type: int, size: int, num_coords: int) -> bytearray:
+ buffer = bytearray(size)
+ preamble_byte = (geom_type << 4) | (coord_type << 1)
+ buffer[0] = preamble_byte
+ struct.pack_into('i', buffer, 4, num_coords)
+ return buffer
+
+
+def put_coordinates(buffer: bytearray, offset: int, coord_type: int, coords: CoordinateSequence):
+ for coord in coords:
+ offset = put_coordinate(buffer, offset, coord_type, coord)
+ return offset
+
+
+def put_coordinate(buffer: bytearray, offset: int, coord_type: int, coord: tuple):
+ x = coord[0]
+ y = coord[1]
+ z = coord[2] if len(coord) > 2 else math.nan
+ if coord_type == CoordinateType.XY:
+ struct.pack_into('d', buffer, offset, x)
+ struct.pack_into('d', buffer, offset + 8, y)
+ offset += 16
+ elif coord_type == CoordinateType.XYZ:
+ struct.pack_into('d', buffer, offset, x)
+ struct.pack_into('d', buffer, offset + 8, y)
+ struct.pack_into('d', buffer, offset + 16, z)
+ offset += 24
+ else:
+ # Shapely does not support M dimension for now
+ raise ValueError("Invalid coordinate type: {}".format(coord_type))
+ return offset
+
+
+def get_coordinates(buffer: bytearray, offset: int, coord_type: int, num_coords: int) -> List[tuple]:
+ coords = []
+ bytes_per_coord = CoordinateType.bytes_per_coord(coord_type)
+ for i in range(num_coords):
+ coord = get_coordinate(buffer, offset, coord_type)
+ coords.append(coord)
+ offset += bytes_per_coord
+ return coords
+
+
+def get_coordinate(buffer: bytearray, offset: int, coord_type: int) -> tuple:
+ x = struct.unpack_from('d', buffer, offset)[0]
+ y = struct.unpack_from('d', buffer, offset + 8)[0]
+ # Shapely does not support M dimension for now, so we'll simply ignore them
+ if coord_type == CoordinateType.XY or coord_type == CoordinateType.XYM:
+ return x, y
+ elif coord_type == CoordinateType.XYZ or coord_type == CoordinateType.XYZM:
+ z = struct.unpack_from('d', buffer, offset + 16)[0]
+ return x, y, z
+ else:
+ raise NotImplementedError("XYM or XYZM coordinates were not supported")
+
+
+def aligned_offset(offset):
+ return (offset + 7) & ~7
+
+
+def serialize_point(geom: Point) -> bytearray:
+ coords = geom.coords
+ if not coords:
+ return create_buffer_for_geom(GeometryTypeID.POINT, CoordinateType.XY, 8, 0)
+ coord_type = CoordinateType.type_of(coords, geom.has_z)
+ bytes_per_coord = CoordinateType.bytes_per_coord(coord_type)
+ size = 8 + bytes_per_coord
+ buffer = create_buffer_for_geom(GeometryTypeID.POINT, coord_type, size, 1)
+ put_coordinates(buffer, 8, coord_type, coords)
+ return buffer
+
+
+def deserialize_point(geom_buffer: GeometryBuffer) -> Point:
+ if geom_buffer.num_coords == 0:
+ # Here we don't call Point() directly since it would create an empty GeometryCollection
+ # in shapely 1.x. You'll find similar code for creating empty geometries in other
+ # deserialization functions.
+ return wkt_loads("POINT EMPTY")
+ coord = geom_buffer.read_coordinate()
+ return Point(coord)
+
+
+def serialize_multi_point(geom: MultiPoint) -> bytearray:
+ points = geom.geoms
+ num_points = len(points)
+ if num_points == 0:
+ return create_buffer_for_geom(GeometryTypeID.MULTIPOINT, CoordinateType.XY, 8, 0)
+ coord_type = CoordinateType.type_of(points[0].coords, geom.has_z)
+ bytes_per_coord = CoordinateType.bytes_per_coord(coord_type)
+ size = 8 + bytes_per_coord * num_points
+ buffer = create_buffer_for_geom(GeometryTypeID.MULTIPOINT, coord_type, size, num_points)
+ offset = 8
+ for point in points:
+ coords = point.coords
+ coord = coords[0] if coords else [(math.nan, math.nan)]
+ offset = put_coordinate(buffer, offset, coord_type, coord)
+ return buffer
+
+
+def deserialize_multi_point(geom_buffer: GeometryBuffer) -> MultiPoint:
+ if geom_buffer.num_coords == 0:
+ return wkt_loads("MULTIPOINT EMPTY")
+ coords = geom_buffer.read_coordinates(geom_buffer.num_coords)
+ points = []
+ for coord in coords:
+ if math.isnan(coord[0]):
+ # Shapely does not allow creating MultiPoint with empty components
+ pass
+ else:
+ points.append(Point(coord))
+ return MultiPoint(points)
+
+
+def serialize_linestring(geom: LineString) -> bytearray:
+ coords = geom.coords
+ coord_type = CoordinateType.type_of(coords, geom.has_z)
+ bytes_per_coord = CoordinateType.bytes_per_coord(coord_type)
+ size = 8 + len(coords) * bytes_per_coord
+ buffer = create_buffer_for_geom(GeometryTypeID.LINESTRING, coord_type, size, len(coords))
+ put_coordinates(buffer, 8, coord_type, coords)
+ return buffer
+
+
+def deserialize_linestring(geom_buffer: GeometryBuffer) -> LineString:
+ if geom_buffer.num_coords == 0:
+ return wkt_loads("LINESTRING EMPTY")
+ coords = geom_buffer.read_coordinates(geom_buffer.num_coords)
+ return LineString(coords)
+
+
+def serialize_multi_linestring(geom: MultiLineString) -> bytearray:
+ linestrings = geom.geoms
+ if not linestrings:
+ return create_buffer_for_geom(GeometryTypeID.MULTILINESTRING, CoordinateType.XY, 8, 0)
+ coord_type = CoordinateType.type_of(linestrings[0].coords, geom.has_z)
+ bytes_per_coord = CoordinateType.bytes_per_coord(coord_type)
+ num_coords = sum(len(ls.coords) for ls in linestrings)
+ num_linestrings_offset = 8 + num_coords * bytes_per_coord
+ size = num_linestrings_offset + 4 + 4 * len(linestrings)
+ buffer = create_buffer_for_geom(GeometryTypeID.MULTILINESTRING, coord_type, size, num_coords)
+ geom_buffer = GeometryBuffer(buffer, coord_type, 8, num_coords)
+ geom_buffer.write_int(len(linestrings))
+ for ls in linestrings:
+ geom_buffer.write_linestring(ls)
+ return buffer
+
+
+def deserialize_multi_linestring(geom_buffer: GeometryBuffer) -> MultiLineString:
+ if geom_buffer.num_coords == 0:
+ return wkt_loads("MULTILINESTRING EMPTY")
+ num_linestrings = geom_buffer.read_int()
+ linestrings = []
+ for k in range(0, num_linestrings):
+ linestring = geom_buffer.read_linestring()
+ if not linestring.is_empty:
+ linestrings.append(linestring)
+ return MultiLineString(linestrings)
+
+
+def serialize_polygon(geom: Polygon) -> bytearray:
+ exterior_coords = geom.exterior.coords
+ if not exterior_coords:
+ return create_buffer_for_geom(GeometryTypeID.POLYGON, CoordinateType.XY, 8, 0)
+ num_coords = len(exterior_coords)
+ for interior in geom.interiors:
+ num_coords += len(interior.coords)
+ bytes_per_coord = len(exterior_coords[0]) * 8
+ num_rings_offset = 8 + num_coords * bytes_per_coord
+ num_rings = len(geom.interiors) + 1
+ size = num_rings_offset + 4 + 4 * num_rings
+ coord_type = CoordinateType.type_of(exterior_coords, geom.has_z)
+ buffer = create_buffer_for_geom(GeometryTypeID.POLYGON, coord_type, size, num_coords)
+ geom_buffer = GeometryBuffer(buffer, coord_type, 8, num_coords)
+ geom_buffer.write_polygon(geom)
+ return buffer
+
+
+def deserialize_polygon(geom_buffer: GeometryBuffer) -> Polygon:
+ if geom_buffer.num_coords == 0:
+ return wkt_loads("POLYGON EMPTY")
+ return geom_buffer.read_polygon()
+
+
+def serialize_multi_polygon(geom: MultiPolygon) -> bytearray:
+ polygons = geom.geoms
+ if not polygons:
+ return create_buffer_for_geom(GeometryTypeID.MULTIPOLYGON, CoordinateType.XY, 8, 0)
+ coord_type = CoordinateType.type_of(polygons[0].exterior.coords, geom.has_z)
+ bytes_per_coord = CoordinateType.bytes_per_coord(coord_type)
+ num_coords = 0
+ num_rings = 0
+ for polygon in polygons:
+ num_exterior_coords = len(polygon.exterior.coords)
+ if num_exterior_coords == 0:
+ continue
+ num_coords += num_exterior_coords
+ num_rings += (1 + len(polygon.interiors))
+ for interior in polygon.interiors:
+ num_coords += len(interior.coords)
+ num_polygons_offset = 8 + num_coords * bytes_per_coord
+ size = num_polygons_offset + 4 + len(polygons) * 4 + num_rings * 4
+ buffer = create_buffer_for_geom(GeometryTypeID.MULTIPOLYGON, coord_type, size, num_coords)
+ geom_buffer = GeometryBuffer(buffer, coord_type, 8, num_coords)
+ geom_buffer.write_int(len(polygons))
+ for polygon in polygons:
+ geom_buffer.write_polygon(polygon)
+ return buffer
+
+
+def deserialize_multi_polygon(geom_buffer: GeometryBuffer) -> MultiPolygon:
+ if geom_buffer.num_coords == 0:
+ return wkt_loads("MULTIPOLYGON EMPTY")
+ num_polygons = geom_buffer.read_int()
+ polygons = []
+ for k in range(0, num_polygons):
+ polygon = geom_buffer.read_polygon()
+ if not polygon.is_empty:
+ polygons.append(polygon)
+ return MultiPolygon(polygons)
+
+
+def serialize_geometry_collection(geom: GeometryCollection) -> bytearray:
+ if not isinstance(geom, GeometryCollection):
+ # In shapely 1.x, Empty geometries such as empty points, empty polygons etc.
+ # have geom_type property evaluated to 'GeometryCollection'. Such geometries are not
+ # instances of GeometryCollection and do not have `geom` property.
+ return serialize_shapely_1_empty_geom(geom)
+ geometries = geom.geoms
+ if not geometries:
+ return create_buffer_for_geom(GeometryTypeID.GEOMETRYCOLLECTION, CoordinateType.XY, 8, 0)
+ num_geometries = len(geometries)
+ total_size = 8
+ buffers = []
+ for geom in geometries:
+ buf = serialize(geom)
+ buffers.append(buf)
+ total_size += aligned_offset(len(buf))
+ buffer = create_buffer_for_geom(GeometryTypeID.GEOMETRYCOLLECTION, CoordinateType.XY, total_size, num_geometries)
+ offset = 8
+ for buf in buffers:
+ buffer[offset:(offset + len(buf))] = buf
+ offset += aligned_offset(len(buf))
+ return buffer
+
+
+def serialize_shapely_1_empty_geom(geom) -> bytearray:
+ if isinstance(geom, Point):
+ geom_type = GeometryTypeID.POINT
+ elif isinstance(geom, LineString):
+ geom_type = GeometryTypeID.LINESTRING
+ elif isinstance(geom, Polygon):
+ geom_type = GeometryTypeID.POLYGON
+ elif isinstance(geom, MultiPoint):
+ geom_type = GeometryTypeID.MULTIPOINT
+ elif isinstance(geom, MultiLineString):
+ geom_type = GeometryTypeID.MULTILINESTRING
+ elif isinstance(geom, MultiPolygon):
+ geom_type = GeometryTypeID.MULTIPOLYGON
+ else:
+ raise ValueError("Invalid empty geometry collection object: {}".format(geom))
+ return create_buffer_for_geom(geom_type, CoordinateType.XY, 8, 0)
+
+
+def deserialize_geometry_collection(geom_buffer: GeometryBuffer) -> GeometryCollection:
+ num_geometries = geom_buffer.num_coords
+ if num_geometries == 0:
+ return wkt_loads("GEOMETRYCOLLECTION EMPTY")
+ geometries = []
+ geom_end_offset = 8
+ buffer = geom_buffer.buffer[8:]
+ for k in range(0, num_geometries):
+ geom, offset = deserialize(buffer)
+ geometries.append(geom)
+ offset = aligned_offset(offset)
+ buffer = buffer[offset:]
+ geom_end_offset += offset
+ geom_buffer.ints_offset = geom_end_offset
+ return GeometryCollection(geometries)
diff --git a/python/tests/serialization/test_deserializers.py b/python/tests/serialization/test_deserializers.py
index 0450a858..215337ba 100644
--- a/python/tests/serialization/test_deserializers.py
+++ b/python/tests/serialization/test_deserializers.py
@@ -17,7 +17,7 @@
import os
-from shapely.geometry import MultiPoint, Point, MultiLineString, LineString, Polygon, MultiPolygon
+from shapely.geometry import MultiPoint, Point, MultiLineString, LineString, Polygon, MultiPolygon, GeometryCollection
import geopandas as gpd
from tests import tests_resource
@@ -95,6 +95,21 @@ class TestGeometryConvert(TestBase):
((40, 40), (30, 30), (40, 20), (30, 10))
]).wkt
+ def test_geometry_collection_deserialization(self):
+ geom = self.spark.sql(
+ """SELECT st_geomFromWKT('GEOMETRYCOLLECTION (
+ MULTILINESTRING((1 2, 3 4), (5 6, 7 8)),
+ MULTILINESTRING((1 2, 3 4), (5 6, 7 8), (9 10, 11 12)),
+ POINT(10 20))') as geom"""
+ ).collect()[0][0]
+
+ assert type(geom) == GeometryCollection
+ assert geom.wkt == GeometryCollection([
+ MultiLineString([[(1, 2), (3, 4)], [(5, 6), (7, 8)]]),
+ MultiLineString([[(1, 2), (3, 4)], [(5, 6), (7, 8)], [(9, 10), (11, 12)]]),
+ Point(10, 20)
+ ]).wkt
+
def test_from_geopandas_convert(self):
gdf = gpd.read_file(os.path.join(tests_resource, "shapefiles/gis_osm_pois_free_1/"))
diff --git a/python/tests/utils/test_geometry_serde.py b/python/tests/utils/test_geometry_serde.py
new file mode 100644
index 00000000..5d12b621
--- /dev/null
+++ b/python/tests/utils/test_geometry_serde.py
@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from shapely.geometry.base import BaseGeometry
+from sedona.utils import geometry_serde
+
+from shapely.geometry import Point
+from shapely.geometry import LineString
+from shapely.geometry import Polygon
+from shapely.geometry import MultiPoint
+from shapely.geometry import MultiLineString
+from shapely.geometry import MultiPolygon
+from shapely.geometry import GeometryCollection
+from shapely.wkt import loads as wkt_loads
+
+
+class TestGeometrySerde:
+ def test_point(self):
+ points = [
+ wkt_loads("POINT EMPTY"),
+ Point(10, 20),
+ Point(10, 20, 30)
+ ]
+ self._test_serde_roundtrip(points)
+
+ def test_linestring(self):
+ linestrings = [
+ wkt_loads("LINESTRING EMPTY"),
+ LineString([(10, 20), (30, 40)]),
+ LineString([(10, 20), (30, 40), (50, 60)]),
+ LineString([(10, 20, 30), (30, 40, 50), (50, 60, 70)]),
+ ]
+ self._test_serde_roundtrip(linestrings)
+
+ def test_multi_point(self):
+ multi_points = [
+ wkt_loads("MULTIPOINT EMPTY"),
+ MultiPoint([(10, 20)]),
+ MultiPoint([(10, 20), (30, 40)]),
+ MultiPoint([(10, 20), (30, 40), (50, 60)]),
+ MultiPoint([(10, 20, 30), (30, 40, 50), (50, 60, 70)]),
+ ]
+ self._test_serde_roundtrip(multi_points)
+
+ def test_multi_linestring(self):
+ multi_linestrings = [
+ wkt_loads("MULTILINESTRING EMPTY"),
+ MultiLineString([[(10, 20), (30, 40)]]),
+ MultiLineString([[(10, 20), (30, 40)], [(50, 60), (70, 80)]]),
+ MultiLineString([[(10, 20, 30), (30, 40, 50)], [(50, 60, 70), (70, 80, 90)]]),
+ ]
+ self._test_serde_roundtrip(multi_linestrings)
+
+ def test_polygon(self):
+ ext = [(0, 0), (10, 0), (10, 10), (0, 10), (0, 0)]
+ int0 = [(1, 1), (1, 1.5), (1.5, 1.5), (1.5, 1), (1, 1)]
+ int1 = [(2, 2), (2, 2.5), (2.5, 2.5), (2.5, 2), (2, 2)]
+ polygons = [
+ wkt_loads("POLYGON EMPTY"),
+ Polygon(ext),
+ Polygon(ext, [int0]),
+ Polygon(ext, [int0, int1]),
+ ]
+ self._test_serde_roundtrip(polygons)
+
+ def test_multi_polygon(self):
+ ext = [(0, 0), (100, 0), (100, 100), (0, 100), (0, 0)]
+ int0 = [(10, 10), (10, 15), (15, 15), (15, 10), (10, 10)]
+ int1 = [(2, 2), (2, 2.5), (2.5, 2.5), (2.5, 2), (2, 2)]
+ multi_polygons = [
+ wkt_loads("MULTIPOLYGON EMPTY"),
+ MultiPolygon([Polygon(ext)]),
+ MultiPolygon([Polygon(ext), Polygon(ext, [int0])]),
+ MultiPolygon([Polygon(ext), Polygon(ext, [int0, int1])]),
+ MultiPolygon([Polygon(ext, [int1]), Polygon(ext), Polygon(ext, [int0, int1])]),
+ ]
+ self._test_serde_roundtrip(multi_polygons)
+
+ def test_geometry_collection(self):
+ geometry_collections = [
+ wkt_loads("GEOMETRYCOLLECTION EMPTY"),
+ GeometryCollection([Point(10, 20), LineString([(10, 20), (30, 40)]), Point(30, 40)]),
+ GeometryCollection([
+ MultiPoint([(10, 20), (30, 40)]),
+ MultiLineString([[(10, 20), (30, 40)], [(50, 60), (70, 80)]]),
+ MultiPolygon([
+ Polygon(
+ [(0, 0), (100, 0), (100, 100), (0, 100), (0, 0)],
+ [[(10, 10), (10, 15), (15, 15), (15, 10), (10, 10)]])
+ ]),
+ Point(100, 200)
+ ]),
+ GeometryCollection([
+ GeometryCollection([Point(10, 20), LineString([(10, 20), (30, 40)]), Point(30, 40)]),
+ GeometryCollection([
+ MultiPoint([(10, 20), (30, 40)]),
+ MultiLineString([[(10, 20), (30, 40)], [(50, 60), (70, 80)]]),
+ Point(10, 20)
+ ])
+ ])
+ ]
+ self._test_serde_roundtrip(geometry_collections)
+
+ @staticmethod
+ def _test_serde_roundtrip(geoms):
+ for geom in geoms:
+ geom_actual = TestGeometrySerde.serde_roundtrip(geom)
+ assert geom_actual.equals_exact(geom, 1e-6)
+ assert geom.wkt == geom_actual.wkt
+
+ @staticmethod
+ def serde_roundtrip(geom: BaseGeometry) -> BaseGeometry:
+ buffer = geometry_serde.serialize(geom)
+ geom2, offset = geometry_serde.deserialize(buffer)
+ return geom2
diff --git a/sql/src/main/scala/org/apache/sedona/sql/utils/GeometrySerializer.scala b/sql/src/main/scala/org/apache/sedona/sql/utils/GeometrySerializer.scala
index 587cda4b..8da851fa 100644
--- a/sql/src/main/scala/org/apache/sedona/sql/utils/GeometrySerializer.scala
+++ b/sql/src/main/scala/org/apache/sedona/sql/utils/GeometrySerializer.scala
@@ -18,10 +18,8 @@
*/
package org.apache.sedona.sql.utils
-import org.apache.sedona.common.utils.GeomUtils;
-import org.apache.spark.sql.catalyst.util.ArrayData
+import org.apache.sedona.common.geometrySerde
import org.locationtech.jts.geom.Geometry
-import org.locationtech.jts.io.{WKBReader, WKBWriter}
/**
* SerDe using the WKB reader and writer objects
@@ -35,8 +33,7 @@ object GeometrySerializer {
* @return Array of bites represents this geometry
*/
def serialize(geometry: Geometry): Array[Byte] = {
- val writer = new WKBWriter(GeomUtils.getDimension(geometry), 2, true)
- writer.write(geometry)
+ geometrySerde.GeometrySerializer.serialize(geometry)
}
/**
@@ -46,7 +43,6 @@ object GeometrySerializer {
* @return JTS geometry
*/
def deserialize(value: Array[Byte]): Geometry = {
- val reader = new WKBReader()
- reader.read(value)
+ geometrySerde.GeometrySerializer.deserialize(value)
}
}
diff --git a/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoDataSourceUtils.scala b/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoDataSourceUtils.scala
index c6ee8e3c..6c2e5f03 100644
--- a/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoDataSourceUtils.scala
+++ b/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoDataSourceUtils.scala
@@ -21,8 +21,30 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
import org.apache.spark.util.Utils
+import scala.util.Try
+
// Needed by Sedona to support Spark 3.0 - 3.3
object GeoDataSourceUtils {
+
+ val PARQUET_REBASE_MODE_IN_READ = firstAvailableConf(
+ "spark.sql.parquet.datetimeRebaseModeInRead",
+ "spark.sql.legacy.parquet.datetimeRebaseModeInRead")
+ val PARQUET_REBASE_MODE_IN_WRITE = firstAvailableConf(
+ "spark.sql.parquet.datetimeRebaseModeInWrite",
+ "spark.sql.legacy.parquet.datetimeRebaseModeInWrite")
+ val PARQUET_INT96_REBASE_MODE_IN_READ = firstAvailableConf(
+ "spark.sql.parquet.int96RebaseModeInRead",
+ "spark.sql.legacy.parquet.int96RebaseModeInRead",
+ "spark.sql.legacy.parquet.datetimeRebaseModeInRead")
+ val PARQUET_INT96_REBASE_MODE_IN_WRITE = firstAvailableConf(
+ "spark.sql.parquet.int96RebaseModeInWrite",
+ "spark.sql.legacy.parquet.int96RebaseModeInWrite",
+ "spark.sql.legacy.parquet.datetimeRebaseModeInWrite")
+
+ private def firstAvailableConf(confs: String *): String = {
+ confs.find(c => Try(SQLConf.get.getConfString(c)).isSuccess).get
+ }
+
def datetimeRebaseMode(
lookupFileMeta: String => String,
modeByConfig: String): LegacyBehaviorPolicy.Value = {
@@ -75,6 +97,18 @@ object GeoDataSourceUtils {
case LegacyBehaviorPolicy.CORRECTED => identity[Int]
}
+ def creteDateRebaseFuncInWrite(
+ rebaseMode: LegacyBehaviorPolicy.Value,
+ format: String): Int => Int = rebaseMode match {
+ case LegacyBehaviorPolicy.EXCEPTION => days: Int =>
+ if (days < RebaseDateTime.lastSwitchGregorianDay) {
+ throw DataSourceUtils.newRebaseExceptionInWrite(format)
+ }
+ days
+ case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianDays
+ case LegacyBehaviorPolicy.CORRECTED => identity[Int]
+ }
+
def creteTimestampRebaseFuncInRead(
rebaseMode: LegacyBehaviorPolicy.Value,
format: String): Long => Long = rebaseMode match {
@@ -86,4 +120,16 @@ object GeoDataSourceUtils {
case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseJulianToGregorianMicros
case LegacyBehaviorPolicy.CORRECTED => identity[Long]
}
+
+ def creteTimestampRebaseFuncInWrite(
+ rebaseMode: LegacyBehaviorPolicy.Value,
+ format: String): Long => Long = rebaseMode match {
+ case LegacyBehaviorPolicy.EXCEPTION => micros: Long =>
+ if (micros < RebaseDateTime.lastSwitchGregorianTs) {
+ throw DataSourceUtils.newRebaseExceptionInWrite(format)
+ }
+ micros
+ case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianMicros
+ case LegacyBehaviorPolicy.CORRECTED => identity[Long]
+ }
}
diff --git a/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetFileFormat.scala b/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetFileFormat.scala
index 93d0f5b1..8059fc4a 100644
--- a/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetFileFormat.scala
+++ b/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetFileFormat.scala
@@ -21,7 +21,10 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
import org.apache.parquet.filter2.compat.FilterCompat
import org.apache.parquet.filter2.predicate.FilterApi
import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS
+import org.apache.parquet.hadoop.ParquetOutputFormat.JobSummaryLevel
import org.apache.parquet.hadoop._
+import org.apache.parquet.hadoop.codec.CodecConfig
+import org.apache.parquet.hadoop.util.ContextUtil
import org.apache.spark.TaskContext
import org.apache.spark.internal.Logging
import org.apache.spark.sql._
@@ -37,7 +40,6 @@ import org.apache.spark.sql.types._
import org.apache.spark.util.SerializableConfiguration
import java.net.URI
-import java.util.NoSuchElementException
class GeoParquetFileFormat extends ParquetFileFormat with FileFormat with DataSourceRegister with Logging with Serializable {
@@ -54,6 +56,94 @@ class GeoParquetFileFormat extends ParquetFileFormat with FileFormat with DataSo
GeoParquetUtils.inferSchema(sparkSession, parameters, files)
}
+ override def prepareWrite(
+ sparkSession: SparkSession,
+ job: Job,
+ options: Map[String, String],
+ dataSchema: StructType): OutputWriterFactory = {
+ val parquetOptions = new ParquetOptions(options, sparkSession.sessionState.conf)
+
+ val conf = ContextUtil.getConfiguration(job)
+
+ val committerClass =
+ conf.getClass(
+ SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key,
+ classOf[ParquetOutputCommitter],
+ classOf[OutputCommitter])
+
+ if (conf.get(SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key) == null) {
+ logInfo("Using default output committer for Parquet: " +
+ classOf[ParquetOutputCommitter].getCanonicalName)
+ } else {
+ logInfo("Using user defined output committer for Parquet: " + committerClass.getCanonicalName)
+ }
+
+ conf.setClass(
+ SQLConf.OUTPUT_COMMITTER_CLASS.key,
+ committerClass,
+ classOf[OutputCommitter])
+
+ // We're not really using `ParquetOutputFormat[Row]` for writing data here, because we override
+ // it in `ParquetOutputWriter` to support appending and dynamic partitioning. The reason why
+ // we set it here is to setup the output committer class to `ParquetOutputCommitter`, which is
+ // bundled with `ParquetOutputFormat[Row]`.
+ job.setOutputFormatClass(classOf[ParquetOutputFormat[Row]])
+
+ ParquetOutputFormat.setWriteSupportClass(job, classOf[ParquetWriteSupport])
+
+ // This metadata is useful for keeping UDTs like Vector/Matrix.
+ ParquetWriteSupport.setSchema(dataSchema, conf)
+
+ // Sets flags for `ParquetWriteSupport`, which converts Catalyst schema to Parquet
+ // schema and writes actual rows to Parquet files.
+ conf.set(
+ SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key,
+ sparkSession.sessionState.conf.writeLegacyParquetFormat.toString)
+
+ conf.set(
+ SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key,
+ sparkSession.sessionState.conf.parquetOutputTimestampType.toString)
+
+ try {
+ val fieldIdWriteEnabled = SQLConf.get.getConfString("spark.sql.parquet.fieldId.write.enabled")
+ conf.set("spark.sql.parquet.fieldId.write.enabled", fieldIdWriteEnabled)
+ } catch {
+ case e: NoSuchElementException => ()
+ }
+
+ // Sets compression scheme
+ conf.set(ParquetOutputFormat.COMPRESSION, parquetOptions.compressionCodecClassName)
+
+ // SPARK-15719: Disables writing Parquet summary files by default.
+ if (conf.get(ParquetOutputFormat.JOB_SUMMARY_LEVEL) == null
+ && conf.get(ParquetOutputFormat.ENABLE_JOB_SUMMARY) == null) {
+ conf.setEnum(ParquetOutputFormat.JOB_SUMMARY_LEVEL, JobSummaryLevel.NONE)
+ }
+
+ if (ParquetOutputFormat.getJobSummaryLevel(conf) != JobSummaryLevel.NONE
+ && !classOf[ParquetOutputCommitter].isAssignableFrom(committerClass)) {
+ // output summary is requested, but the class is not a Parquet Committer
+ logWarning(s"Committer $committerClass is not a ParquetOutputCommitter and cannot" +
+ s" create job summaries. " +
+ s"Set Parquet option ${ParquetOutputFormat.JOB_SUMMARY_LEVEL} to NONE.")
+ }
+
+ conf.set(ParquetOutputFormat.WRITE_SUPPORT_CLASS, classOf[GeoParquetWriteSupport].getName)
+
+ new OutputWriterFactory {
+ override def newInstance(
+ path: String,
+ dataSchema: StructType,
+ context: TaskAttemptContext): OutputWriter = {
+ new ParquetOutputWriter(path, context)
+ }
+
+ override def getFileExtension(context: TaskAttemptContext): String = {
+ CodecConfig.from(context).getCodec.getExtension + ".parquet"
+ }
+ }
+ }
+
override def buildReaderWithPartitionValues(
sparkSession: SparkSession,
dataSchema: StructType,
@@ -160,33 +250,12 @@ class GeoParquetFileFormat extends ParquetFileFormat with FileFormat with DataSo
} else {
None
}
- var dateTimeRebaseModeConf:String = ""
- var int96RebaseModeConf:String = ""
- try {
- // Try Spark 3.2+ style
- dateTimeRebaseModeConf = SQLConf.get.getConfString("spark.sql.parquet.datetimeRebaseModeInRead")
- int96RebaseModeConf = SQLConf.get.getConfString("spark.sql.parquet.int96RebaseModeInRead")
- } catch {
- case e1:NoSuchElementException => {
- // Try Spark 3.1 style
- dateTimeRebaseModeConf = SQLConf.get.getConfString("spark.sql.legacy.parquet.datetimeRebaseModeInRead")
- try {
- int96RebaseModeConf = SQLConf.get.getConfString("spark.sql.legacy.parquet.int96RebaseModeInRead")
- }
- catch {
- case e2: NoSuchElementException => {
- // This is Spark 3.1 style. Assume int96 mode is same as dateTime
- int96RebaseModeConf = dateTimeRebaseModeConf
- }
- }
- }
- }
val datetimeRebaseMode = GeoDataSourceUtils.datetimeRebaseMode(
footerFileMetaData.getKeyValueMetaData.get,
- dateTimeRebaseModeConf)
+ SQLConf.get.getConfString(GeoDataSourceUtils.PARQUET_REBASE_MODE_IN_READ))
val int96RebaseMode = GeoDataSourceUtils.int96RebaseMode(
footerFileMetaData.getKeyValueMetaData.get,
- int96RebaseModeConf)
+ SQLConf.get.getConfString(GeoDataSourceUtils.PARQUET_INT96_REBASE_MODE_IN_READ))
val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0)
val hadoopAttemptContext =
diff --git a/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetReadSupport.scala b/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetReadSupport.scala
index 8b8f1f35..ff2dce75 100644
--- a/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetReadSupport.scala
+++ b/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetReadSupport.scala
@@ -27,6 +27,7 @@ import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
+import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT
import org.apache.spark.sql.types._
import java.time.ZoneId
@@ -119,7 +120,7 @@ class GeoParquetReadSupport (
val parquetRequestedSchema = readContext.getRequestedSchema
new GeoParquetRecordMaterializer(
parquetRequestedSchema,
- ParquetReadSupport.expandUDT(catalystRequestedSchema),
+ GeoParquetReadSupport.expandUDT(catalystRequestedSchema),
new GeoParquetToSparkSchemaConverter(conf),
convertTz,
datetimeRebaseMode,
@@ -379,6 +380,10 @@ object GeoParquetReadSupport extends Logging {
val expandedFields = t.fields.map(f => f.copy(dataType = expand(f.dataType)))
t.copy(fields = expandedFields)
+ // Don't expand GeometryUDT types. We'll treat geometry columns specially in
+ // GeoParquetRowConverter
+ case t: GeometryUDT => t
+
case t: UserDefinedType[_] =>
t.sqlType
diff --git a/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetRowConverter.scala b/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetRowConverter.scala
index 47e2a9d7..2d12073c 100644
--- a/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetRowConverter.scala
+++ b/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetRowConverter.scala
@@ -28,8 +28,10 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, CaseInsensitiveMap, DateTimeUtils, GenericArrayData}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
+import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
+import org.locationtech.jts.io.WKBReader
import java.math.{BigDecimal, BigInteger}
import java.time.{ZoneId, ZoneOffset}
@@ -67,7 +69,7 @@ import scala.collection.mutable.ArrayBuffer
* @param schemaConverter A utility converter used to convert Parquet types to Catalyst types.
* @param parquetType Parquet schema of Parquet records
* @param catalystType Spark SQL schema that corresponds to the Parquet record type. User-defined
- * types should have been expanded.
+ * types other than [[GeometryUDT]] should have been expanded.
* @param convertTz the optional time zone to convert to int96 data
* @param datetimeRebaseMode the mode of rebasing date/timestamp from Julian to Proleptic Gregorian
* calendar
@@ -96,7 +98,7 @@ private[parquet] class GeoParquetRowConverter(
""".stripMargin)
assert(
- !catalystType.existsRecursively(_.isInstanceOf[UserDefinedType[_]]),
+ !catalystType.existsRecursively(t => !t.isInstanceOf[GeometryUDT] && t.isInstanceOf[UserDefinedType[_]]),
s"""User-defined types in Catalyst schema should have already been expanded:
|${catalystType.prettyJson}
""".stripMargin)
@@ -199,6 +201,15 @@ private[parquet] class GeoParquetRowConverter(
case BooleanType | IntegerType | LongType | FloatType | DoubleType | BinaryType =>
new ParquetPrimitiveConverter(updater)
+ case GeometryUDT =>
+ new ParquetPrimitiveConverter(updater) {
+ override def addBinary(value: Binary): Unit = {
+ val wkbReader = new WKBReader()
+ val geom = wkbReader.read(value.getBytes)
+ updater.set(GeometryUDT.serialize(geom))
+ }
+ }
+
case ByteType =>
new ParquetPrimitiveConverter(updater) {
override def addInt(value: Int): Unit =
diff --git a/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetWriteSupport.scala b/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetWriteSupport.scala
new file mode 100644
index 00000000..9f14c443
--- /dev/null
+++ b/sql/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/GeoParquetWriteSupport.scala
@@ -0,0 +1,483 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.parquet
+
+import java.nio.{ByteBuffer, ByteOrder}
+import java.util
+import scala.collection.JavaConverters.mapAsJavaMapConverter
+import org.apache.hadoop.conf.Configuration
+import org.apache.parquet.hadoop.api.WriteSupport
+import org.apache.parquet.hadoop.api.WriteSupport.WriteContext
+import org.apache.parquet.io.api.{Binary, RecordConsumer}
+import org.apache.sedona.common.utils.GeomUtils
+import org.apache.spark.SPARK_VERSION_SHORT
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.SPARK_VERSION_METADATA_KEY
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
+import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT
+import org.apache.spark.sql.types._
+import org.locationtech.jts.io.WKBWriter
+
+/**
+ * A Parquet [[WriteSupport]] implementation that writes Catalyst [[InternalRow]]s as Parquet
+ * messages. This class can write Parquet data in two modes:
+ *
+ * - Standard mode: Parquet data are written in standard format defined in parquet-format spec.
+ * - Legacy mode: Parquet data are written in legacy format compatible with Spark 1.4 and prior.
+ *
+ * This behavior can be controlled by SQL option `spark.sql.parquet.writeLegacyFormat`. The value
+ * of this option is propagated to this class by the `init()` method and its Hadoop configuration
+ * argument.
+ */
+class GeoParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
+ // A `ValueWriter` is responsible for writing a field of an `InternalRow` to the record consumer.
+ // Here we are using `SpecializedGetters` rather than `InternalRow` so that we can directly access
+ // data in `ArrayData` without the help of `SpecificMutableRow`.
+ private type ValueWriter = (SpecializedGetters, Int) => Unit
+
+ // Schema of the `InternalRow`s to be written
+ private var schema: StructType = _
+
+ // `ValueWriter`s for all fields of the schema
+ private var rootFieldWriters: Array[ValueWriter] = _
+
+ // The Parquet `RecordConsumer` to which all `InternalRow`s are written
+ private var recordConsumer: RecordConsumer = _
+
+ // Whether to write data in legacy Parquet format compatible with Spark 1.4 and prior versions
+ private var writeLegacyParquetFormat: Boolean = _
+
+ // Which parquet timestamp type to use when writing.
+ private var outputTimestampType: SQLConf.ParquetOutputTimestampType.Value = _
+
+ // Reusable byte array used to write timestamps as Parquet INT96 values
+ private val timestampBuffer = new Array[Byte](12)
+
+ // Reusable byte array used to write decimal values
+ private val decimalBuffer =
+ new Array[Byte](Decimal.minBytesForPrecision(DecimalType.MAX_PRECISION))
+
+ private val datetimeRebaseMode = LegacyBehaviorPolicy.withName(
+ SQLConf.get.getConfString(GeoDataSourceUtils.PARQUET_REBASE_MODE_IN_WRITE))
+
+ private val dateRebaseFunc = GeoDataSourceUtils.creteDateRebaseFuncInWrite(
+ datetimeRebaseMode, "Parquet")
+
+ private val timestampRebaseFunc = GeoDataSourceUtils.creteTimestampRebaseFuncInWrite(
+ datetimeRebaseMode, "Parquet")
+
+ private val int96RebaseMode = LegacyBehaviorPolicy.withName(
+ SQLConf.get.getConfString(GeoDataSourceUtils.PARQUET_INT96_REBASE_MODE_IN_WRITE))
+
+ private val int96RebaseFunc = GeoDataSourceUtils.creteTimestampRebaseFuncInWrite(
+ int96RebaseMode, "Parquet INT96")
+
+ override def init(configuration: Configuration): WriteContext = {
+ val schemaString = configuration.get(ParquetWriteSupport.SPARK_ROW_SCHEMA)
+ this.schema = StructType.fromString(schemaString)
+ this.writeLegacyParquetFormat = {
+ // `SQLConf.PARQUET_WRITE_LEGACY_FORMAT` should always be explicitly set in ParquetRelation
+ assert(configuration.get(SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key) != null)
+ configuration.get(SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key).toBoolean
+ }
+
+ this.outputTimestampType = {
+ val key = SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key
+ assert(configuration.get(key) != null)
+ SQLConf.ParquetOutputTimestampType.withName(configuration.get(key))
+ }
+
+ this.rootFieldWriters = schema.map(_.dataType).map(makeWriter).toArray[ValueWriter]
+
+ val messageType = new SparkToParquetSchemaConverter(configuration).convert(schema)
+ val metadata = Map(
+ SPARK_VERSION_METADATA_KEY -> SPARK_VERSION_SHORT,
+ ParquetReadSupport.SPARK_METADATA_KEY -> schemaString
+ ) ++ {
+ if (datetimeRebaseMode == LegacyBehaviorPolicy.LEGACY) {
+ Some("org.apache.spark.legacyDateTime" -> "")
+ } else {
+ None
+ }
+ } ++ {
+ if (int96RebaseMode == LegacyBehaviorPolicy.LEGACY) {
+ Some("org.apache.spark.legacyINT96" -> "")
+ } else {
+ None
+ }
+ }
+
+ logInfo(
+ s"""Initialized Parquet WriteSupport with Catalyst schema:
+ |${schema.prettyJson}
+ |and corresponding Parquet message type:
+ |$messageType
+ """.stripMargin)
+
+ new WriteContext(messageType, metadata.asJava)
+ }
+
+ override def prepareForWrite(recordConsumer: RecordConsumer): Unit = {
+ this.recordConsumer = recordConsumer
+ }
+
+ override def write(row: InternalRow): Unit = {
+ consumeMessage {
+ writeFields(row, schema, rootFieldWriters)
+ }
+ }
+
+ private def writeFields(
+ row: InternalRow, schema: StructType, fieldWriters: Array[ValueWriter]): Unit = {
+ var i = 0
+ while (i < row.numFields) {
+ if (!row.isNullAt(i)) {
+ consumeField(schema(i).name, i) {
+ fieldWriters(i).apply(row, i)
+ }
+ }
+ i += 1
+ }
+ }
+
+ private def makeWriter(dataType: DataType): ValueWriter = {
+ dataType match {
+ case BooleanType =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ recordConsumer.addBoolean(row.getBoolean(ordinal))
+
+ case ByteType =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ recordConsumer.addInteger(row.getByte(ordinal))
+
+ case ShortType =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ recordConsumer.addInteger(row.getShort(ordinal))
+
+ case DateType =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ recordConsumer.addInteger(dateRebaseFunc(row.getInt(ordinal)))
+
+ case IntegerType =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ recordConsumer.addInteger(row.getInt(ordinal))
+
+ case LongType =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ recordConsumer.addLong(row.getLong(ordinal))
+
+ case FloatType =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ recordConsumer.addFloat(row.getFloat(ordinal))
+
+ case DoubleType =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ recordConsumer.addDouble(row.getDouble(ordinal))
+
+ case StringType =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ recordConsumer.addBinary(
+ Binary.fromReusedByteArray(row.getUTF8String(ordinal).getBytes))
+
+ case TimestampType =>
+ outputTimestampType match {
+ case SQLConf.ParquetOutputTimestampType.INT96 =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ val micros = int96RebaseFunc(row.getLong(ordinal))
+ val (julianDay, timeOfDayNanos) = DateTimeUtils.toJulianDay(micros)
+ val buf = ByteBuffer.wrap(timestampBuffer)
+ buf.order(ByteOrder.LITTLE_ENDIAN).putLong(timeOfDayNanos).putInt(julianDay)
+ recordConsumer.addBinary(Binary.fromReusedByteArray(timestampBuffer))
+
+ case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ val micros = row.getLong(ordinal)
+ recordConsumer.addLong(timestampRebaseFunc(micros))
+
+ case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MILLIS =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ val micros = row.getLong(ordinal)
+ val millis = GeoDateTimeUtils.microsToMillis(timestampRebaseFunc(micros))
+ recordConsumer.addLong(millis)
+ }
+
+ case BinaryType =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ recordConsumer.addBinary(Binary.fromReusedByteArray(row.getBinary(ordinal)))
+
+ case DecimalType.Fixed(precision, scale) =>
+ makeDecimalWriter(precision, scale)
+
+ case t: StructType =>
+ val fieldWriters = t.map(_.dataType).map(makeWriter).toArray[ValueWriter]
+ (row: SpecializedGetters, ordinal: Int) =>
+ consumeGroup {
+ writeFields(row.getStruct(ordinal, t.length), t, fieldWriters)
+ }
+
+ case t: ArrayType => makeArrayWriter(t)
+
+ case t: MapType => makeMapWriter(t)
+
+ case GeometryUDT =>
+ (row: SpecializedGetters, ordinal: Int) =>
+ val serializedGeometry = row.getBinary(ordinal)
+ val geom = GeometryUDT.deserialize(serializedGeometry)
+ val wkbWriter = new WKBWriter(GeomUtils.getDimension(geom))
+ recordConsumer.addBinary(Binary.fromReusedByteArray(wkbWriter.write(geom)))
+
+ case t: UserDefinedType[_] => makeWriter(t.sqlType)
+
+ // TODO Adds IntervalType support
+ case _ => sys.error(s"Unsupported data type $dataType.")
+ }
+ }
+
+ private def makeDecimalWriter(precision: Int, scale: Int): ValueWriter = {
+ assert(
+ precision <= DecimalType.MAX_PRECISION,
+ s"Decimal precision $precision exceeds max precision ${DecimalType.MAX_PRECISION}")
+
+ val numBytes = Decimal.minBytesForPrecision(precision)
+
+ val int32Writer =
+ (row: SpecializedGetters, ordinal: Int) => {
+ val unscaledLong = row.getDecimal(ordinal, precision, scale).toUnscaledLong
+ recordConsumer.addInteger(unscaledLong.toInt)
+ }
+
+ val int64Writer =
+ (row: SpecializedGetters, ordinal: Int) => {
+ val unscaledLong = row.getDecimal(ordinal, precision, scale).toUnscaledLong
+ recordConsumer.addLong(unscaledLong)
+ }
+
+ val binaryWriterUsingUnscaledLong =
+ (row: SpecializedGetters, ordinal: Int) => {
+ // When the precision is low enough (<= 18) to squeeze the decimal value into a `Long`, we
+ // can build a fixed-length byte array with length `numBytes` using the unscaled `Long`
+ // value and the `decimalBuffer` for better performance.
+ val unscaled = row.getDecimal(ordinal, precision, scale).toUnscaledLong
+ var i = 0
+ var shift = 8 * (numBytes - 1)
+
+ while (i < numBytes) {
+ decimalBuffer(i) = (unscaled >> shift).toByte
+ i += 1
+ shift -= 8
+ }
+
+ recordConsumer.addBinary(Binary.fromReusedByteArray(decimalBuffer, 0, numBytes))
+ }
+
+ val binaryWriterUsingUnscaledBytes =
+ (row: SpecializedGetters, ordinal: Int) => {
+ val decimal = row.getDecimal(ordinal, precision, scale)
+ val bytes = decimal.toJavaBigDecimal.unscaledValue().toByteArray
+ val fixedLengthBytes = if (bytes.length == numBytes) {
+ // If the length of the underlying byte array of the unscaled `BigInteger` happens to be
+ // `numBytes`, just reuse it, so that we don't bother copying it to `decimalBuffer`.
+ bytes
+ } else {
+ // Otherwise, the length must be less than `numBytes`. In this case we copy contents of
+ // the underlying bytes with padding sign bytes to `decimalBuffer` to form the result
+ // fixed-length byte array.
+ val signByte = if (bytes.head < 0) -1: Byte else 0: Byte
+ util.Arrays.fill(decimalBuffer, 0, numBytes - bytes.length, signByte)
+ System.arraycopy(bytes, 0, decimalBuffer, numBytes - bytes.length, bytes.length)
+ decimalBuffer
+ }
+
+ recordConsumer.addBinary(Binary.fromReusedByteArray(fixedLengthBytes, 0, numBytes))
+ }
+
+ writeLegacyParquetFormat match {
+ // Standard mode, 1 <= precision <= 9, writes as INT32
+ case false if precision <= Decimal.MAX_INT_DIGITS => int32Writer
+
+ // Standard mode, 10 <= precision <= 18, writes as INT64
+ case false if precision <= Decimal.MAX_LONG_DIGITS => int64Writer
+
+ // Legacy mode, 1 <= precision <= 18, writes as FIXED_LEN_BYTE_ARRAY
+ case true if precision <= Decimal.MAX_LONG_DIGITS => binaryWriterUsingUnscaledLong
+
+ // Either standard or legacy mode, 19 <= precision <= 38, writes as FIXED_LEN_BYTE_ARRAY
+ case _ => binaryWriterUsingUnscaledBytes
+ }
+ }
+
+ def makeArrayWriter(arrayType: ArrayType): ValueWriter = {
+ val elementWriter = makeWriter(arrayType.elementType)
+
+ def threeLevelArrayWriter(repeatedGroupName: String, elementFieldName: String): ValueWriter =
+ (row: SpecializedGetters, ordinal: Int) => {
+ val array = row.getArray(ordinal)
+ consumeGroup {
+ // Only creates the repeated field if the array is non-empty.
+ if (array.numElements() > 0) {
+ consumeField(repeatedGroupName, 0) {
+ var i = 0
+ while (i < array.numElements()) {
+ consumeGroup {
+ // Only creates the element field if the current array element is not null.
+ if (!array.isNullAt(i)) {
+ consumeField(elementFieldName, 0) {
+ elementWriter.apply(array, i)
+ }
+ }
+ }
+ i += 1
+ }
+ }
+ }
+ }
+ }
+
+ def twoLevelArrayWriter(repeatedFieldName: String): ValueWriter =
+ (row: SpecializedGetters, ordinal: Int) => {
+ val array = row.getArray(ordinal)
+ consumeGroup {
+ // Only creates the repeated field if the array is non-empty.
+ if (array.numElements() > 0) {
+ consumeField(repeatedFieldName, 0) {
+ var i = 0
+ while (i < array.numElements()) {
+ elementWriter.apply(array, i)
+ i += 1
+ }
+ }
+ }
+ }
+ }
+
+ (writeLegacyParquetFormat, arrayType.containsNull) match {
+ case (legacyMode @ false, _) =>
+ // Standard mode:
+ //
+ // <list-repetition> group <name> (LIST) {
+ // repeated group list {
+ // ^~~~ repeatedGroupName
+ // <element-repetition> <element-type> element;
+ // ^~~~~~~ elementFieldName
+ // }
+ // }
+ threeLevelArrayWriter(repeatedGroupName = "list", elementFieldName = "element")
+
+ case (legacyMode @ true, nullableElements @ true) =>
+ // Legacy mode, with nullable elements:
+ //
+ // <list-repetition> group <name> (LIST) {
+ // optional group bag {
+ // ^~~ repeatedGroupName
+ // repeated <element-type> array;
+ // ^~~~~ elementFieldName
+ // }
+ // }
+ threeLevelArrayWriter(repeatedGroupName = "bag", elementFieldName = "array")
+
+ case (legacyMode @ true, nullableElements @ false) =>
+ // Legacy mode, with non-nullable elements:
+ //
+ // <list-repetition> group <name> (LIST) {
+ // repeated <element-type> array;
+ // ^~~~~ repeatedFieldName
+ // }
+ twoLevelArrayWriter(repeatedFieldName = "array")
+ }
+ }
+
+ private def makeMapWriter(mapType: MapType): ValueWriter = {
+ val keyWriter = makeWriter(mapType.keyType)
+ val valueWriter = makeWriter(mapType.valueType)
+ val repeatedGroupName = if (writeLegacyParquetFormat) {
+ // Legacy mode:
+ //
+ // <map-repetition> group <name> (MAP) {
+ // repeated group map (MAP_KEY_VALUE) {
+ // ^~~ repeatedGroupName
+ // required <key-type> key;
+ // <value-repetition> <value-type> value;
+ // }
+ // }
+ "map"
+ } else {
+ // Standard mode:
+ //
+ // <map-repetition> group <name> (MAP) {
+ // repeated group key_value {
+ // ^~~~~~~~~ repeatedGroupName
+ // required <key-type> key;
+ // <value-repetition> <value-type> value;
+ // }
+ // }
+ "key_value"
+ }
+
+ (row: SpecializedGetters, ordinal: Int) => {
+ val map = row.getMap(ordinal)
+ val keyArray = map.keyArray()
+ val valueArray = map.valueArray()
+
+ consumeGroup {
+ // Only creates the repeated field if the map is non-empty.
+ if (map.numElements() > 0) {
+ consumeField(repeatedGroupName, 0) {
+ var i = 0
+ while (i < map.numElements()) {
+ consumeGroup {
+ consumeField("key", 0) {
+ keyWriter.apply(keyArray, i)
+ }
+
+ // Only creates the "value" field if the value if non-empty
+ if (!map.valueArray().isNullAt(i)) {
+ consumeField("value", 1) {
+ valueWriter.apply(valueArray, i)
+ }
+ }
+ }
+ i += 1
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private def consumeMessage(f: => Unit): Unit = {
+ recordConsumer.startMessage()
+ f
+ recordConsumer.endMessage()
+ }
+
+ private def consumeGroup(f: => Unit): Unit = {
+ recordConsumer.startGroup()
+ f
+ recordConsumer.endGroup()
+ }
+
+ private def consumeField(field: String, index: Int)(f: => Unit): Unit = {
+ recordConsumer.startField(field, index)
+ f
+ recordConsumer.endField(field, index)
+ }
+}