You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2023/05/10 06:08:29 UTC
[sedona] 01/01: Get rid of Sedona-Spark dependencies in Sedona-Flink
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch clean-up-flink-dependency
in repository https://gitbox.apache.org/repos/asf/sedona.git
commit 63e444605922736ca5bb7e4798ad8219485ab0d3
Author: Jia Yu <ji...@apache.org>
AuthorDate: Tue May 9 23:08:11 2023 -0700
Get rid of Sedona-Spark dependencies in Sedona-Flink
---
common/pom.xml | 4 +
.../common/geometrySerde}/GeometrySerde.java | 2 +-
.../common/geometrySerde}/SpatialIndexSerde.java | 3 +-
.../jts/index/quadtree/IndexSerde.java | 31 +--
.../locationtech/jts/index/strtree/IndexSerde.java | 31 +--
.../java/org/apache/sedona/common}/CircleTest.java | 2 +-
.../geometrySerde}/SpatialIndexSerdeTest.java | 3 +-
.../sedona/core/serde/SedonaKryoRegistrator.java | 4 +-
.../shapefileParser/shapes/GeometrySerdeTest.java | 2 +-
flink/pom.xml | 10 -
.../sedona/flink/SedonaFlinkRegistrator.java | 5 +-
.../sedona/flink/expressions/Constructors.java | 13 +-
.../sedona/flink/expressions/Predicates.java | 213 +++++----------------
pom.xml | 7 +-
.../expressions/geohash/GeoHashDecoder.scala | 78 --------
.../sedona/sql/functions/geohash/Fixtures.scala | 7 +-
.../sql/functions/geohash/TestGeoHashDecoder.scala | 6 +-
17 files changed, 118 insertions(+), 303 deletions(-)
diff --git a/common/pom.xml b/common/pom.xml
index ed685ea7..7a291375 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -73,6 +73,10 @@
<groupId>com.google.geometry</groupId>
<artifactId>s2-geometry</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.esotericsoftware</groupId>
+ <artifactId>kryo</artifactId>
+ </dependency>
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
diff --git a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerde.java
similarity index 99%
rename from core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
rename to common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerde.java
index e4aa6c0d..0b779e2f 100755
--- a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
+++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerde.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.sedona.core.geometryObjects;
+package org.apache.sedona.common.geometrySerde;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Registration;
diff --git a/core/src/main/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerde.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerde.java
similarity index 97%
rename from core/src/main/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerde.java
rename to common/src/main/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerde.java
index f4b5f6d2..efaf2e97 100644
--- a/core/src/main/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerde.java
+++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerde.java
@@ -17,13 +17,12 @@
* under the License.
*/
-package org.apache.sedona.core.geometryObjects;
+package org.apache.sedona.common.geometrySerde;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.log4j.Logger;
import org.locationtech.jts.index.quadtree.IndexSerde;
import org.locationtech.jts.index.quadtree.Quadtree;
import org.locationtech.jts.index.strtree.STRtree;
diff --git a/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java b/common/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
similarity index 79%
rename from core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
rename to common/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
index 5dd65566..46f8218c 100644
--- a/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
+++ b/common/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
@@ -1,22 +1,27 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.locationtech.jts.index.quadtree;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
+import org.apache.sedona.common.geometrySerde.GeometrySerde;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
diff --git a/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java b/common/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
similarity index 85%
rename from core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
rename to common/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
index bf3d308b..2501befb 100644
--- a/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
+++ b/common/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
@@ -1,22 +1,27 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.locationtech.jts.index.strtree;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
+import org.apache.sedona.common.geometrySerde.GeometrySerde;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
diff --git a/core/src/test/java/org/apache/sedona/core/geometryObjects/CircleTest.java b/common/src/test/java/org/apache/sedona/common/CircleTest.java
similarity index 99%
rename from core/src/test/java/org/apache/sedona/core/geometryObjects/CircleTest.java
rename to common/src/test/java/org/apache/sedona/common/CircleTest.java
index 58781dd4..f4bef207 100644
--- a/core/src/test/java/org/apache/sedona/core/geometryObjects/CircleTest.java
+++ b/common/src/test/java/org/apache/sedona/common/CircleTest.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.sedona.core.geometryObjects;
+package org.apache.sedona.common;
import org.apache.sedona.common.geometryObjects.Circle;
import org.apache.sedona.common.utils.GeomUtils;
diff --git a/core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerdeTest.java
similarity index 98%
rename from core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java
rename to common/src/test/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerdeTest.java
index acb08736..84dfc964 100644
--- a/core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java
+++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerdeTest.java
@@ -17,11 +17,12 @@
* under the License.
*/
-package org.apache.sedona.core.geometryObjects;
+package org.apache.sedona.common.geometrySerde;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
+import org.apache.sedona.common.geometrySerde.SpatialIndexSerde;
import org.junit.Test;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.Envelope;
diff --git a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java
index f3c62665..113fb434 100644
--- a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java
+++ b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java
@@ -22,8 +22,8 @@ package org.apache.sedona.core.serde;
import com.esotericsoftware.kryo.Kryo;
import org.apache.log4j.Logger;
import org.apache.sedona.common.geometryObjects.Circle;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
-import org.apache.sedona.core.geometryObjects.SpatialIndexSerde;
+import org.apache.sedona.common.geometrySerde.GeometrySerde;
+import org.apache.sedona.common.geometrySerde.SpatialIndexSerde;
import org.apache.spark.serializer.KryoRegistrator;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.GeometryCollection;
diff --git a/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java b/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java
index 0e8d88ab..924e9f13 100644
--- a/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java
+++ b/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java
@@ -24,7 +24,7 @@ import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import org.apache.sedona.common.utils.GeomUtils;
import org.apache.sedona.common.geometryObjects.Circle;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
+import org.apache.sedona.common.geometrySerde.GeometrySerde;
import org.junit.Test;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryCollection;
diff --git a/flink/pom.xml b/flink/pom.xml
index 9c52f3b9..1c52fa14 100644
--- a/flink/pom.xml
+++ b/flink/pom.xml
@@ -44,16 +44,6 @@
<artifactId>sedona-common</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-core-${spark.compat.version}_${scala.compat.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-sql-${spark.compat.version}_${scala.compat.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
diff --git a/flink/src/main/java/org/apache/sedona/flink/SedonaFlinkRegistrator.java b/flink/src/main/java/org/apache/sedona/flink/SedonaFlinkRegistrator.java
index 36a6e492..ef55cdd6 100644
--- a/flink/src/main/java/org/apache/sedona/flink/SedonaFlinkRegistrator.java
+++ b/flink/src/main/java/org/apache/sedona/flink/SedonaFlinkRegistrator.java
@@ -16,9 +16,8 @@ package org.apache.sedona.flink;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.sedona.common.geometryObjects.Circle;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
-import org.apache.sedona.core.geometryObjects.SpatialIndexSerde;
-import org.apache.sedona.core.spatialPartitioning.PartitioningUtils;
+import org.apache.sedona.common.geometrySerde.GeometrySerde;
+import org.apache.sedona.common.geometrySerde.SpatialIndexSerde;
import org.locationtech.jts.geom.*;
import org.locationtech.jts.index.quadtree.Quadtree;
import org.locationtech.jts.index.strtree.STRtree;
diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Constructors.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Constructors.java
index 08cc0ed9..36bbeba8 100644
--- a/flink/src/main/java/org/apache/sedona/flink/expressions/Constructors.java
+++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Constructors.java
@@ -18,7 +18,7 @@ import org.apache.flink.table.functions.ScalarFunction;
import org.apache.sedona.common.enums.FileDataSplitter;
import org.apache.sedona.common.enums.GeometryType;
import org.apache.sedona.common.utils.FormatUtils;
-import org.apache.spark.sql.sedona_sql.expressions.geohash.GeoHashDecoder;
+import org.apache.sedona.common.utils.GeoHashDecoder;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryFactory;
@@ -155,14 +155,17 @@ public class Constructors {
public static class ST_GeomFromGeoHash extends ScalarFunction {
@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class)
public Geometry eval(@DataTypeHint("String") String value,
- @DataTypeHint("Int") Integer precision) throws ParseException {
+ @DataTypeHint("Int") Integer precision)
+ throws ParseException, GeoHashDecoder.InvalidGeoHashException
+ {
// The default precision is the geohash length. Otherwise, use the precision given by the user
- scala.Option<Object> optionPrecision = scala.Option.apply(precision);
- return GeoHashDecoder.decode(value, optionPrecision);
+ return GeoHashDecoder.decode(value, precision);
}
@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class)
- public Geometry eval(@DataTypeHint("String") String value) throws ParseException {
+ public Geometry eval(@DataTypeHint("String") String value)
+ throws ParseException, GeoHashDecoder.InvalidGeoHashException
+ {
return eval(value, null);
}
}
diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Predicates.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Predicates.java
index e7f797d9..b29e6ad5 100644
--- a/flink/src/main/java/org/apache/sedona/flink/expressions/Predicates.java
+++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Predicates.java
@@ -15,258 +15,141 @@ package org.apache.sedona.flink.expressions;
import org.apache.flink.table.annotation.DataTypeHint;
import org.apache.flink.table.functions.ScalarFunction;
-import org.apache.sedona.common.utils.GeomUtils;
-import org.apache.sedona.core.spatialPartitioning.PartitioningUtils;
-import org.apache.sedona.common.utils.HalfOpenRectangle;
-import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
-import java.util.List;
-import java.util.Objects;
-
public class Predicates {
- public static class ST_Intersects extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_Intersects(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
+ public static class ST_Intersects extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_Intersects() {
+ public ST_Intersects()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.intersects(geom2);
}
-
- /**
- * Check spatial relation with duplicates removal
- * @param key
- * @param o1
- * @param o2
- * @return True if intersecting, false otherwise
- */
- @DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
- Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner.");
- Geometry geom1 = (Geometry) o1;
- Geometry geom2 = (Geometry) o2;
- HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key));
- return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.intersects(geom2);
- }
}
- public static class ST_Contains extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_Contains(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
+ public static class ST_Contains
+ extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_Contains() {
+ public ST_Contains()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.contains(geom2);
}
-
- /**
- * Check spatial relation with duplicates removal
- * @param key
- * @param o1
- * @param o2
- * @return True if o1 contains o2, false otherwise
- */
- @DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
- Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner.");
- Geometry geom1 = (Geometry) o1;
- Geometry geom2 = (Geometry) o2;
- HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key));
- return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.contains(geom2);
- }
}
- public static class ST_Within extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_Within(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
-
+ public static class ST_Within
+ extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_Within() {
+ public ST_Within()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.within(geom2);
}
-
- /**
- * Check spatial relation with duplicates removal
- * @param key
- * @param o1
- * @param o2
- * @return True if o1 is within o2, false otherwise
- */
- @DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
- Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner.");
- Geometry geom1 = (Geometry) o1;
- Geometry geom2 = (Geometry) o2;
- HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key));
- return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.within(geom2);
- }
}
- public static class ST_Covers extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_Covers(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
+ public static class ST_Covers
+ extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_Covers() {
+ public ST_Covers()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.covers(geom2);
}
-
- /**
- * Check spatial relation with duplicates removal
- * @param key
- * @param o1
- * @param o2
- * @return True if o1 covers o2, false otherwise
- */
- @DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
- Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner.");
- Geometry geom1 = (Geometry) o1;
- Geometry geom2 = (Geometry) o2;
- HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key));
- return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.covers(geom2);
- }
}
- public static class ST_CoveredBy extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_CoveredBy(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
+ public static class ST_CoveredBy
+ extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_CoveredBy() {
+ public ST_CoveredBy()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.coveredBy(geom2);
}
-
- /**
- * Check spatial relation with duplicates removal
- * @param key
- * @param o1
- * @param o2
- * @return True if o1 is covered by o2, false otherwise
- */
- @DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
- Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner.");
- Geometry geom1 = (Geometry) o1;
- Geometry geom2 = (Geometry) o2;
- HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key));
- return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.coveredBy(geom2);
- }
}
- public static class ST_Disjoint extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_Disjoint(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
+ public static class ST_Disjoint
+ extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_Disjoint() {
+ public ST_Disjoint()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.disjoint(geom2);
}
}
- public static class ST_OrderingEquals extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_OrderingEquals(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
+ public static class ST_OrderingEquals
+ extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_OrderingEquals() {
+ public ST_OrderingEquals()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.equalsExact(geom2);
diff --git a/pom.xml b/pom.xml
index a98c97df..7c7aa490 100644
--- a/pom.xml
+++ b/pom.xml
@@ -279,7 +279,12 @@
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
-
+ <dependency>
+ <groupId>com.esotericsoftware</groupId>
+ <artifactId>kryo</artifactId>
+ <version>4.0.2</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/geohash/GeoHashDecoder.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/geohash/GeoHashDecoder.scala
deleted file mode 100644
index 9e510cd0..00000000
--- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/geohash/GeoHashDecoder.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.spark.sql.sedona_sql.expressions.geohash
-
-import org.apache.sedona.common.utils.BBox
-import org.locationtech.jts.geom.{Geometry, GeometryFactory}
-
-import scala.collection.mutable
-
-class InvalidGeoHashException(message: String) extends Exception(message)
-
-object GeoHashDecoder {
- private val bits = Seq(16, 8, 4, 2, 1)
- private val base32 = "0123456789bcdefghjkmnpqrstuvwxyz"
- val geometryFactory = new GeometryFactory()
-
- def decode(geohash: String, precision: Option[Int] = None): Geometry = {
- decodeGeoHashBBox(geohash, precision).getBbox().toPolygon()
- }
-
- private def decodeGeoHashBBox(geohash: String, precision: Option[Int] = None): LatLon = {
- val latLon = LatLon(mutable.Seq(-180, 180), mutable.Seq(-90, 90))
- val geoHashLowered = geohash.toLowerCase()
- val geoHashLength = geohash.length
- val targetPrecision = precision match {
- case Some(value) => if (value < 0) throw new InvalidGeoHashException("Precision can not be negative")
- else math.min(geoHashLength, value)
- case None => geoHashLength
- }
- var isEven = true
-
- for (i <- Range(0, targetPrecision)){
- val c = geoHashLowered(i)
- val cd = base32.indexOf(c).toByte
- if (cd == -1){
- throw new InvalidGeoHashException(s"Invalid character '$c' found at index $i")
- }
-
- for (j <- Range(0, 5)){
- val mask = bits(j).toByte
- val index = if ((mask & cd) == 0) 1 else 0
- if (isEven){
- latLon.lons(index) = (latLon.lons.head + latLon.lons(1)) / 2
- }
- else {
- latLon.lats(index) = (latLon.lats.head + latLon.lats(1)) / 2
- }
- isEven = !isEven
- }
- }
- latLon
- }
-}
-
-private case class LatLon(var lons: mutable.Seq[Double], var lats: mutable.Seq[Double]){
- def getBbox(): BBox = new BBox(
- lons.head,
- lons(1),
- lats.head,
- lats(1)
- )
-}
diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/Fixtures.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/Fixtures.scala
index 1d729286..6470e37f 100644
--- a/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/Fixtures.scala
+++ b/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/Fixtures.scala
@@ -18,10 +18,9 @@
*/
package org.apache.sedona.sql.functions.geohash
-import org.apache.sedona.common.utils.GeometryGeoHashEncoder
-import org.apache.spark.sql.sedona_sql.expressions.geohash.GeoHashDecoder
+import org.apache.sedona.common.utils.{GeoHashDecoder, GeometryGeoHashEncoder}
import org.locationtech.jts.geom.Geometry
-import org.scalatest.prop.{TableDrivenPropertyChecks, TableFor2, TableFor3, TableFor4}
+import org.scalatest.prop.{TableDrivenPropertyChecks, TableFor3, TableFor4}
object Fixtures extends TableDrivenPropertyChecks {
@@ -67,7 +66,7 @@ object Fixtures extends TableDrivenPropertyChecks {
Option(GeometryGeoHashEncoder.calculate(geom, precision))
}
- def decodeGeoHash(geohash: String, precision: Option[Int]): Geometry =
+ def decodeGeoHash(geohash: String, precision: Int): Geometry =
GeoHashDecoder.decode(geohash, precision)
}
\ No newline at end of file
diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/TestGeoHashDecoder.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/TestGeoHashDecoder.scala
index 5cffc1f9..3740a72a 100644
--- a/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/TestGeoHashDecoder.scala
+++ b/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/TestGeoHashDecoder.scala
@@ -18,8 +18,8 @@
*/
package org.apache.sedona.sql.functions.geohash
+import org.apache.sedona.common.utils.GeoHashDecoder.InvalidGeoHashException
import org.apache.sedona.sql.functions.FunctionsHelper
-import org.apache.spark.sql.sedona_sql.expressions.geohash.InvalidGeoHashException
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers
import org.scalatest.prop.TableDrivenPropertyChecks
@@ -27,13 +27,13 @@ import org.scalatest.prop.TableDrivenPropertyChecks
class TestGeoHashDecoder extends AnyFunSuite with Matchers with TableDrivenPropertyChecks with FunctionsHelper{
for ((statement: String, geoHash: String, precision: Int, geometry: String) <- Fixtures.geometriesFromGeoHash) {
test("it should decode " + statement) {
- Fixtures.decodeGeoHash(geoHash, Some(precision)) shouldBe wktReader.read(geometry)
+ Fixtures.decodeGeoHash(geoHash, precision) shouldBe wktReader.read(geometry)
}
}
for ((statement: String, geoHash: String, precision: Int) <- Fixtures.invalidGeoHashes) {
test("it should raise InvalidGeoHashException when " + statement) {
- an[InvalidGeoHashException] shouldBe thrownBy(Fixtures.decodeGeoHash(geoHash, Some(precision)))
+ an[InvalidGeoHashException] shouldBe thrownBy(Fixtures.decodeGeoHash(geoHash, precision))
}
}
}