You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2023/05/10 06:08:28 UTC
[sedona] branch clean-up-flink-dependency created (now 63e44460)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a change to branch clean-up-flink-dependency
in repository https://gitbox.apache.org/repos/asf/sedona.git
at 63e44460 Get rid of Sedona-Spark dependencies in Sedona-Flink
This branch includes the following new commits:
new 63e44460 Get rid of Sedona-Spark dependencies in Sedona-Flink
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[sedona] 01/01: Get rid of Sedona-Spark dependencies in Sedona-Flink
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch clean-up-flink-dependency
in repository https://gitbox.apache.org/repos/asf/sedona.git
commit 63e444605922736ca5bb7e4798ad8219485ab0d3
Author: Jia Yu <ji...@apache.org>
AuthorDate: Tue May 9 23:08:11 2023 -0700
Get rid of Sedona-Spark dependencies in Sedona-Flink
---
common/pom.xml | 4 +
.../common/geometrySerde}/GeometrySerde.java | 2 +-
.../common/geometrySerde}/SpatialIndexSerde.java | 3 +-
.../jts/index/quadtree/IndexSerde.java | 31 +--
.../locationtech/jts/index/strtree/IndexSerde.java | 31 +--
.../java/org/apache/sedona/common}/CircleTest.java | 2 +-
.../geometrySerde}/SpatialIndexSerdeTest.java | 3 +-
.../sedona/core/serde/SedonaKryoRegistrator.java | 4 +-
.../shapefileParser/shapes/GeometrySerdeTest.java | 2 +-
flink/pom.xml | 10 -
.../sedona/flink/SedonaFlinkRegistrator.java | 5 +-
.../sedona/flink/expressions/Constructors.java | 13 +-
.../sedona/flink/expressions/Predicates.java | 213 +++++----------------
pom.xml | 7 +-
.../expressions/geohash/GeoHashDecoder.scala | 78 --------
.../sedona/sql/functions/geohash/Fixtures.scala | 7 +-
.../sql/functions/geohash/TestGeoHashDecoder.scala | 6 +-
17 files changed, 118 insertions(+), 303 deletions(-)
diff --git a/common/pom.xml b/common/pom.xml
index ed685ea7..7a291375 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -73,6 +73,10 @@
<groupId>com.google.geometry</groupId>
<artifactId>s2-geometry</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.esotericsoftware</groupId>
+ <artifactId>kryo</artifactId>
+ </dependency>
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
diff --git a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerde.java
similarity index 99%
rename from core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
rename to common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerde.java
index e4aa6c0d..0b779e2f 100755
--- a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java
+++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerde.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.sedona.core.geometryObjects;
+package org.apache.sedona.common.geometrySerde;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Registration;
diff --git a/core/src/main/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerde.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerde.java
similarity index 97%
rename from core/src/main/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerde.java
rename to common/src/main/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerde.java
index f4b5f6d2..efaf2e97 100644
--- a/core/src/main/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerde.java
+++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerde.java
@@ -17,13 +17,12 @@
* under the License.
*/
-package org.apache.sedona.core.geometryObjects;
+package org.apache.sedona.common.geometrySerde;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.log4j.Logger;
import org.locationtech.jts.index.quadtree.IndexSerde;
import org.locationtech.jts.index.quadtree.Quadtree;
import org.locationtech.jts.index.strtree.STRtree;
diff --git a/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java b/common/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
similarity index 79%
rename from core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
rename to common/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
index 5dd65566..46f8218c 100644
--- a/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
+++ b/common/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java
@@ -1,22 +1,27 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.locationtech.jts.index.quadtree;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
+import org.apache.sedona.common.geometrySerde.GeometrySerde;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
diff --git a/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java b/common/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
similarity index 85%
rename from core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
rename to common/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
index bf3d308b..2501befb 100644
--- a/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
+++ b/common/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java
@@ -1,22 +1,27 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.locationtech.jts.index.strtree;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
+import org.apache.sedona.common.geometrySerde.GeometrySerde;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
diff --git a/core/src/test/java/org/apache/sedona/core/geometryObjects/CircleTest.java b/common/src/test/java/org/apache/sedona/common/CircleTest.java
similarity index 99%
rename from core/src/test/java/org/apache/sedona/core/geometryObjects/CircleTest.java
rename to common/src/test/java/org/apache/sedona/common/CircleTest.java
index 58781dd4..f4bef207 100644
--- a/core/src/test/java/org/apache/sedona/core/geometryObjects/CircleTest.java
+++ b/common/src/test/java/org/apache/sedona/common/CircleTest.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.sedona.core.geometryObjects;
+package org.apache.sedona.common;
import org.apache.sedona.common.geometryObjects.Circle;
import org.apache.sedona.common.utils.GeomUtils;
diff --git a/core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerdeTest.java
similarity index 98%
rename from core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java
rename to common/src/test/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerdeTest.java
index acb08736..84dfc964 100644
--- a/core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java
+++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerdeTest.java
@@ -17,11 +17,12 @@
* under the License.
*/
-package org.apache.sedona.core.geometryObjects;
+package org.apache.sedona.common.geometrySerde;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
+import org.apache.sedona.common.geometrySerde.SpatialIndexSerde;
import org.junit.Test;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.Envelope;
diff --git a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java
index f3c62665..113fb434 100644
--- a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java
+++ b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java
@@ -22,8 +22,8 @@ package org.apache.sedona.core.serde;
import com.esotericsoftware.kryo.Kryo;
import org.apache.log4j.Logger;
import org.apache.sedona.common.geometryObjects.Circle;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
-import org.apache.sedona.core.geometryObjects.SpatialIndexSerde;
+import org.apache.sedona.common.geometrySerde.GeometrySerde;
+import org.apache.sedona.common.geometrySerde.SpatialIndexSerde;
import org.apache.spark.serializer.KryoRegistrator;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.GeometryCollection;
diff --git a/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java b/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java
index 0e8d88ab..924e9f13 100644
--- a/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java
+++ b/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java
@@ -24,7 +24,7 @@ import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import org.apache.sedona.common.utils.GeomUtils;
import org.apache.sedona.common.geometryObjects.Circle;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
+import org.apache.sedona.common.geometrySerde.GeometrySerde;
import org.junit.Test;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryCollection;
diff --git a/flink/pom.xml b/flink/pom.xml
index 9c52f3b9..1c52fa14 100644
--- a/flink/pom.xml
+++ b/flink/pom.xml
@@ -44,16 +44,6 @@
<artifactId>sedona-common</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-core-${spark.compat.version}_${scala.compat.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-sql-${spark.compat.version}_${scala.compat.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
diff --git a/flink/src/main/java/org/apache/sedona/flink/SedonaFlinkRegistrator.java b/flink/src/main/java/org/apache/sedona/flink/SedonaFlinkRegistrator.java
index 36a6e492..ef55cdd6 100644
--- a/flink/src/main/java/org/apache/sedona/flink/SedonaFlinkRegistrator.java
+++ b/flink/src/main/java/org/apache/sedona/flink/SedonaFlinkRegistrator.java
@@ -16,9 +16,8 @@ package org.apache.sedona.flink;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.sedona.common.geometryObjects.Circle;
-import org.apache.sedona.core.geometryObjects.GeometrySerde;
-import org.apache.sedona.core.geometryObjects.SpatialIndexSerde;
-import org.apache.sedona.core.spatialPartitioning.PartitioningUtils;
+import org.apache.sedona.common.geometrySerde.GeometrySerde;
+import org.apache.sedona.common.geometrySerde.SpatialIndexSerde;
import org.locationtech.jts.geom.*;
import org.locationtech.jts.index.quadtree.Quadtree;
import org.locationtech.jts.index.strtree.STRtree;
diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Constructors.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Constructors.java
index 08cc0ed9..36bbeba8 100644
--- a/flink/src/main/java/org/apache/sedona/flink/expressions/Constructors.java
+++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Constructors.java
@@ -18,7 +18,7 @@ import org.apache.flink.table.functions.ScalarFunction;
import org.apache.sedona.common.enums.FileDataSplitter;
import org.apache.sedona.common.enums.GeometryType;
import org.apache.sedona.common.utils.FormatUtils;
-import org.apache.spark.sql.sedona_sql.expressions.geohash.GeoHashDecoder;
+import org.apache.sedona.common.utils.GeoHashDecoder;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryFactory;
@@ -155,14 +155,17 @@ public class Constructors {
public static class ST_GeomFromGeoHash extends ScalarFunction {
@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class)
public Geometry eval(@DataTypeHint("String") String value,
- @DataTypeHint("Int") Integer precision) throws ParseException {
+ @DataTypeHint("Int") Integer precision)
+ throws ParseException, GeoHashDecoder.InvalidGeoHashException
+ {
// The default precision is the geohash length. Otherwise, use the precision given by the user
- scala.Option<Object> optionPrecision = scala.Option.apply(precision);
- return GeoHashDecoder.decode(value, optionPrecision);
+ return GeoHashDecoder.decode(value, precision);
}
@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class)
- public Geometry eval(@DataTypeHint("String") String value) throws ParseException {
+ public Geometry eval(@DataTypeHint("String") String value)
+ throws ParseException, GeoHashDecoder.InvalidGeoHashException
+ {
return eval(value, null);
}
}
diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Predicates.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Predicates.java
index e7f797d9..b29e6ad5 100644
--- a/flink/src/main/java/org/apache/sedona/flink/expressions/Predicates.java
+++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Predicates.java
@@ -15,258 +15,141 @@ package org.apache.sedona.flink.expressions;
import org.apache.flink.table.annotation.DataTypeHint;
import org.apache.flink.table.functions.ScalarFunction;
-import org.apache.sedona.common.utils.GeomUtils;
-import org.apache.sedona.core.spatialPartitioning.PartitioningUtils;
-import org.apache.sedona.common.utils.HalfOpenRectangle;
-import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
-import java.util.List;
-import java.util.Objects;
-
public class Predicates {
- public static class ST_Intersects extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_Intersects(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
+ public static class ST_Intersects extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_Intersects() {
+ public ST_Intersects()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.intersects(geom2);
}
-
- /**
- * Check spatial relation with duplicates removal
- * @param key
- * @param o1
- * @param o2
- * @return True if intersecting, false otherwise
- */
- @DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
- Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner.");
- Geometry geom1 = (Geometry) o1;
- Geometry geom2 = (Geometry) o2;
- HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key));
- return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.intersects(geom2);
- }
}
- public static class ST_Contains extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_Contains(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
+ public static class ST_Contains
+ extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_Contains() {
+ public ST_Contains()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.contains(geom2);
}
-
- /**
- * Check spatial relation with duplicates removal
- * @param key
- * @param o1
- * @param o2
- * @return True if o1 contains o2, false otherwise
- */
- @DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
- Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner.");
- Geometry geom1 = (Geometry) o1;
- Geometry geom2 = (Geometry) o2;
- HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key));
- return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.contains(geom2);
- }
}
- public static class ST_Within extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_Within(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
-
+ public static class ST_Within
+ extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_Within() {
+ public ST_Within()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.within(geom2);
}
-
- /**
- * Check spatial relation with duplicates removal
- * @param key
- * @param o1
- * @param o2
- * @return True if o1 is within o2, false otherwise
- */
- @DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
- Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner.");
- Geometry geom1 = (Geometry) o1;
- Geometry geom2 = (Geometry) o2;
- HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key));
- return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.within(geom2);
- }
}
- public static class ST_Covers extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_Covers(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
+ public static class ST_Covers
+ extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_Covers() {
+ public ST_Covers()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.covers(geom2);
}
-
- /**
- * Check spatial relation with duplicates removal
- * @param key
- * @param o1
- * @param o2
- * @return True if o1 covers o2, false otherwise
- */
- @DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
- Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner.");
- Geometry geom1 = (Geometry) o1;
- Geometry geom2 = (Geometry) o2;
- HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key));
- return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.covers(geom2);
- }
}
- public static class ST_CoveredBy extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_CoveredBy(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
+ public static class ST_CoveredBy
+ extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_CoveredBy() {
+ public ST_CoveredBy()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.coveredBy(geom2);
}
-
- /**
- * Check spatial relation with duplicates removal
- * @param key
- * @param o1
- * @param o2
- * @return True if o1 is covered by o2, false otherwise
- */
- @DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
- Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner.");
- Geometry geom1 = (Geometry) o1;
- Geometry geom2 = (Geometry) o2;
- HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key));
- return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.coveredBy(geom2);
- }
}
- public static class ST_Disjoint extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_Disjoint(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
+ public static class ST_Disjoint
+ extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_Disjoint() {
+ public ST_Disjoint()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.disjoint(geom2);
}
}
- public static class ST_OrderingEquals extends ScalarFunction {
- private List<Envelope> grids;
-
- /**
- * Constructor for duplicate removal
- */
- public ST_OrderingEquals(PartitioningUtils partitioner) {
- grids = partitioner.fetchLeafZones();
- }
+ public static class ST_OrderingEquals
+ extends ScalarFunction
+ {
/**
* Constructor for relation checking without duplicate removal
*/
- public ST_OrderingEquals() {
+ public ST_OrderingEquals()
+ {
}
@DataTypeHint("Boolean")
- public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) {
+ public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2)
+ {
Geometry geom1 = (Geometry) o1;
Geometry geom2 = (Geometry) o2;
return geom1.equalsExact(geom2);
diff --git a/pom.xml b/pom.xml
index a98c97df..7c7aa490 100644
--- a/pom.xml
+++ b/pom.xml
@@ -279,7 +279,12 @@
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
-
+ <dependency>
+ <groupId>com.esotericsoftware</groupId>
+ <artifactId>kryo</artifactId>
+ <version>4.0.2</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/geohash/GeoHashDecoder.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/geohash/GeoHashDecoder.scala
deleted file mode 100644
index 9e510cd0..00000000
--- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/geohash/GeoHashDecoder.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.spark.sql.sedona_sql.expressions.geohash
-
-import org.apache.sedona.common.utils.BBox
-import org.locationtech.jts.geom.{Geometry, GeometryFactory}
-
-import scala.collection.mutable
-
-class InvalidGeoHashException(message: String) extends Exception(message)
-
-object GeoHashDecoder {
- private val bits = Seq(16, 8, 4, 2, 1)
- private val base32 = "0123456789bcdefghjkmnpqrstuvwxyz"
- val geometryFactory = new GeometryFactory()
-
- def decode(geohash: String, precision: Option[Int] = None): Geometry = {
- decodeGeoHashBBox(geohash, precision).getBbox().toPolygon()
- }
-
- private def decodeGeoHashBBox(geohash: String, precision: Option[Int] = None): LatLon = {
- val latLon = LatLon(mutable.Seq(-180, 180), mutable.Seq(-90, 90))
- val geoHashLowered = geohash.toLowerCase()
- val geoHashLength = geohash.length
- val targetPrecision = precision match {
- case Some(value) => if (value < 0) throw new InvalidGeoHashException("Precision can not be negative")
- else math.min(geoHashLength, value)
- case None => geoHashLength
- }
- var isEven = true
-
- for (i <- Range(0, targetPrecision)){
- val c = geoHashLowered(i)
- val cd = base32.indexOf(c).toByte
- if (cd == -1){
- throw new InvalidGeoHashException(s"Invalid character '$c' found at index $i")
- }
-
- for (j <- Range(0, 5)){
- val mask = bits(j).toByte
- val index = if ((mask & cd) == 0) 1 else 0
- if (isEven){
- latLon.lons(index) = (latLon.lons.head + latLon.lons(1)) / 2
- }
- else {
- latLon.lats(index) = (latLon.lats.head + latLon.lats(1)) / 2
- }
- isEven = !isEven
- }
- }
- latLon
- }
-}
-
-private case class LatLon(var lons: mutable.Seq[Double], var lats: mutable.Seq[Double]){
- def getBbox(): BBox = new BBox(
- lons.head,
- lons(1),
- lats.head,
- lats(1)
- )
-}
diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/Fixtures.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/Fixtures.scala
index 1d729286..6470e37f 100644
--- a/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/Fixtures.scala
+++ b/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/Fixtures.scala
@@ -18,10 +18,9 @@
*/
package org.apache.sedona.sql.functions.geohash
-import org.apache.sedona.common.utils.GeometryGeoHashEncoder
-import org.apache.spark.sql.sedona_sql.expressions.geohash.GeoHashDecoder
+import org.apache.sedona.common.utils.{GeoHashDecoder, GeometryGeoHashEncoder}
import org.locationtech.jts.geom.Geometry
-import org.scalatest.prop.{TableDrivenPropertyChecks, TableFor2, TableFor3, TableFor4}
+import org.scalatest.prop.{TableDrivenPropertyChecks, TableFor3, TableFor4}
object Fixtures extends TableDrivenPropertyChecks {
@@ -67,7 +66,7 @@ object Fixtures extends TableDrivenPropertyChecks {
Option(GeometryGeoHashEncoder.calculate(geom, precision))
}
- def decodeGeoHash(geohash: String, precision: Option[Int]): Geometry =
+ def decodeGeoHash(geohash: String, precision: Int): Geometry =
GeoHashDecoder.decode(geohash, precision)
}
\ No newline at end of file
diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/TestGeoHashDecoder.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/TestGeoHashDecoder.scala
index 5cffc1f9..3740a72a 100644
--- a/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/TestGeoHashDecoder.scala
+++ b/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/TestGeoHashDecoder.scala
@@ -18,8 +18,8 @@
*/
package org.apache.sedona.sql.functions.geohash
+import org.apache.sedona.common.utils.GeoHashDecoder.InvalidGeoHashException
import org.apache.sedona.sql.functions.FunctionsHelper
-import org.apache.spark.sql.sedona_sql.expressions.geohash.InvalidGeoHashException
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers
import org.scalatest.prop.TableDrivenPropertyChecks
@@ -27,13 +27,13 @@ import org.scalatest.prop.TableDrivenPropertyChecks
class TestGeoHashDecoder extends AnyFunSuite with Matchers with TableDrivenPropertyChecks with FunctionsHelper{
for ((statement: String, geoHash: String, precision: Int, geometry: String) <- Fixtures.geometriesFromGeoHash) {
test("it should decode " + statement) {
- Fixtures.decodeGeoHash(geoHash, Some(precision)) shouldBe wktReader.read(geometry)
+ Fixtures.decodeGeoHash(geoHash, precision) shouldBe wktReader.read(geometry)
}
}
for ((statement: String, geoHash: String, precision: Int) <- Fixtures.invalidGeoHashes) {
test("it should raise InvalidGeoHashException when " + statement) {
- an[InvalidGeoHashException] shouldBe thrownBy(Fixtures.decodeGeoHash(geoHash, Some(precision)))
+ an[InvalidGeoHashException] shouldBe thrownBy(Fixtures.decodeGeoHash(geoHash, precision))
}
}
}