You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2023/09/02 07:28:08 UTC
[sedona] branch master updated: [SEDONA-381] Merge Python adapter to sql-common module (#995)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new 254aa04f9 [SEDONA-381] Merge Python adapter to sql-common module (#995)
254aa04f9 is described below
commit 254aa04f95747ff29c42d6284d165a357b2a6332
Author: Jia Yu <ji...@apache.org>
AuthorDate: Sat Sep 2 00:28:03 2023 -0700
[SEDONA-381] Merge Python adapter to sql-common module (#995)
---
pom.xml | 2 -
python-adapter/.gitignore | 6 -
python-adapter/pom.xml | 152 ---------------------
.../src/test/resources/log4j2.properties | 31 -----
.../sedona/python/wrapper/GeometrySample.scala | 58 --------
.../sedona/python/wrapper/PythonTestSpec.scala | 30 ----
.../apache/sedona/python/wrapper/SparkUtil.scala | 45 ------
.../python/wrapper/TestToPythonSerialization.scala | 99 --------------
spark-shaded/pom.xml | 5 -
.../python/wrapper}/SerializationException.scala | 0
.../python/wrapper}/adapters/EnvelopeAdapter.scala | 0
.../python/wrapper}/adapters/GeomSerializer.scala | 0
.../python/wrapper}/adapters/GeometryAdapter.scala | 0
.../wrapper}/adapters/JoinParamsAdapter.scala | 0
.../python/wrapper}/adapters/PythonConverter.scala | 8 +-
.../adapters/PythonRddToJavaRDDAdapter.scala | 0
.../wrapper}/adapters/RawJvmIndexRDDAdapter.scala | 0
.../adapters/SpatialObjectLoaderAdapter.scala | 0
.../wrapper}/translation/CircleSerializer.scala | 2 +-
.../translation/FlatPairRddConverter.scala | 2 +-
.../translation/GeometryRddConverter.scala | 2 +-
.../translation/GeometrySeqToPythonConverter.scala | 2 +-
.../wrapper}/translation/GeometrySerializer.scala | 2 +-
.../translation/ListPairRddConverter.scala | 2 +-
.../translation/PythonGeometrySerializer.scala | 4 +-
.../translation/PythonRDDToJavaConverter.scala | 2 +-
.../translation/PythonRddToJavaRDDConverter.scala | 0
.../translation/RDDToPythonConverter.scala | 0
.../wrapper}/utils/PythonAdapterWrapper.scala | 0
.../sedona/python/wrapper}/utils/implicits.scala | 4 +-
.../python/wrapper/TestToPythonSerialization.scala | 143 +++++++++++++++++++
.../org/apache/sedona/sql/TestBaseScala.scala | 2 +
32 files changed, 157 insertions(+), 446 deletions(-)
diff --git a/pom.xml b/pom.xml
index f7e414328..86211314e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -658,7 +658,6 @@
<module>core</module>
<module>sql</module>
<module>viz</module>
- <module>python-adapter</module>
<module>spark-shaded</module>
</modules>
</profile>
@@ -681,7 +680,6 @@
<module>core</module>
<module>sql</module>
<module>viz</module>
- <module>python-adapter</module>
<module>spark-shaded</module>
<module>flink</module>
<module>flink-shaded</module>
diff --git a/python-adapter/.gitignore b/python-adapter/.gitignore
deleted file mode 100644
index 67d8d314a..000000000
--- a/python-adapter/.gitignore
+++ /dev/null
@@ -1,6 +0,0 @@
-/target/
-bin
-/.settings
-/.classpath
-/.project
-*.iml
\ No newline at end of file
diff --git a/python-adapter/pom.xml b/python-adapter/pom.xml
deleted file mode 100644
index b534f8a2c..000000000
--- a/python-adapter/pom.xml
+++ /dev/null
@@ -1,152 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>sedona-parent</artifactId>
- <groupId>org.apache.sedona</groupId>
- <version>1.5.0-SNAPSHOT</version>
- </parent>
- <artifactId>sedona-python-adapter-${spark.compat.version}_${scala.compat.version}</artifactId>
-
- <name>${project.groupId}:${project.artifactId}</name>
- <description>A cluster computing system for processing large-scale spatial data: Python Adapter.</description>
- <url>http://sedona.apache.org/</url>
- <packaging>jar</packaging>
-
- <properties>
- <maven.deploy.skip>false</maven.deploy.skip>
- <geotools.scope>provided</geotools.scope>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-common</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-core-${spark.compat.version}_${scala.compat.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-sql-${spark.compat.version}_${scala.compat.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.compat.version}</artifactId>
- <version>${spark.version}</version>
- <scope>provided</scope>
- <exclusions>
- <!-- Make sure Hadoop's guava version is used for older versions of Spark tests -->
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <!-- Exclude log4j 1 for older versions of Spark-->
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <!-- Exclude log4j-slf4j2-impl for Spark 3.4 -->
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.compat.version}</artifactId>
- <version>${spark.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.locationtech.jts</groupId>
- <artifactId>jts-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-main</artifactId>
- </dependency>
- <dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-epsg-hsql</artifactId>
- </dependency>
- <dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-coverage</artifactId>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>${scala.version}</version>
- </dependency>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
- <artifactId>scala-collection-compat_${scala.compat.version}</artifactId>
- <version>${scala-collection-compat.version}</version>
- </dependency>
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.compat.version}</artifactId>
- <version>${scalatest.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <sourceDirectory>src/main/scala</sourceDirectory>
- <plugins>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/python-adapter/src/test/resources/log4j2.properties b/python-adapter/src/test/resources/log4j2.properties
deleted file mode 100644
index 5f8985946..000000000
--- a/python-adapter/src/test/resources/log4j2.properties
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Set everything to be logged to the file target/unit-tests.log
-rootLogger.level = info
-rootLogger.appenderRef.file.ref = File
-
-appender.file.type = File
-appender.file.name = File
-appender.file.fileName = target/unit-tests.log
-appender.file.append = true
-appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
-
-# Ignore messages below warning level from Jetty, because it's a bit verbose
-logger.jetty.name = org.sparkproject.jetty
-logger.jetty.level = warn
diff --git a/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/GeometrySample.scala b/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/GeometrySample.scala
deleted file mode 100644
index 8e90a4443..000000000
--- a/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/GeometrySample.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.python.wrapper
-
-import org.apache.sedona.common.geometryObjects.Circle
-import org.locationtech.jts.geom.Geometry
-
-import java.io.{FileInputStream, InputStream}
-import scala.io.Source
-
-trait GeometrySample extends PythonTestSpec {
- self: TestToPythonSerialization =>
-
- val resourceFolder = System.getProperty("user.dir") + "/../core/src/test/resources/"
- private[python] val samplePoints: List[Geometry] =
- loadGeometriesFromResources(resourceFolder + "python/samplePoints")
-
- private[python] val sampleCircles: List[Geometry] = samplePoints.map(new Circle(_, 1.0))
-
- private[python] val sampleLines: List[Geometry] =
- loadGeometriesFromResources(resourceFolder + "python/sampleLines")
-
- private[python] val samplePolygons: List[Geometry] =
- loadGeometriesFromResources(resourceFolder + "python/samplePolygons")
-
- private def loadGeometriesFromResources(fileName: String): List[Geometry] = {
- val resourceFileText = loadResourceFile(fileName)
- loadFromWktStrings(resourceFileText)
- }
-
- private def loadFromWktStrings(geometries: List[String]): List[Geometry] = {
- geometries.map(
- geometryWKT => wktReader.read(geometryWKT)
- )
- }
-
- private def loadResourceFile(fileName: String): List[String] = {
- val stream: InputStream = new FileInputStream(fileName)
- Source.fromInputStream(stream).getLines.toList
- }
-}
diff --git a/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/PythonTestSpec.scala b/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/PythonTestSpec.scala
deleted file mode 100644
index f00644a7d..000000000
--- a/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/PythonTestSpec.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.python.wrapper
-
-import org.apache.sedona.python.wrapper.translation.PythonGeometrySerializer
-import org.locationtech.jts.geom.GeometryFactory
-import org.locationtech.jts.io.WKTReader
-
-trait PythonTestSpec {
- private[python] lazy val geometryFactory = new GeometryFactory()
- private[python] lazy val pythonGeometrySerializer = new PythonGeometrySerializer()
- private[python] lazy val wktReader = new WKTReader(geometryFactory)
-}
diff --git a/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/SparkUtil.scala b/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/SparkUtil.scala
deleted file mode 100644
index 537325fac..000000000
--- a/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/SparkUtil.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.python.wrapper
-
-import org.apache.log4j.{Level, Logger}
-import org.apache.sedona.core.monitoring.Listener
-import org.apache.sedona.core.serde.SedonaKryoRegistrator
-import org.apache.spark.serializer.KryoSerializer
-import org.apache.spark.{SparkConf, SparkContext}
-import org.scalatest.FunSuite
-
-abstract class SparkUtil extends FunSuite {
- lazy val sc = SparkUtil.sc
-}
-
-object SparkUtil {
- private lazy val sc = {
- val conf = new SparkConf().setAppName("scalaTest").setMaster("local[2]")
- conf.set("spark.serializer", classOf[KryoSerializer].getName)
- conf.set("spark.kryo.registrator", classOf[SedonaKryoRegistrator].getName)
-
- val sc = new SparkContext(conf)
- sc.addSparkListener(new Listener)
- Logger.getLogger("org").setLevel(Level.WARN)
- Logger.getLogger("akka").setLevel(Level.WARN)
- sc
- }
-}
diff --git a/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala b/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala
deleted file mode 100644
index b390acfe1..000000000
--- a/python-adapter/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sedona.python.wrapper
-
-import org.apache.sedona.python.wrapper.adapters.PythonConverter
-import org.apache.sedona.python.wrapper.translation.{FlatPairRddConverter, GeometryRddConverter, ListPairRddConverter, PythonRDDToJavaConverter}
-import org.apache.spark.api.java.JavaPairRDD
-import org.scalatest.Matchers
-import org.apache.sedona.python.wrapper.utils.implicits._
-
-import scala.jdk.CollectionConverters._
-
-
-class TestToPythonSerialization extends SparkUtil with GeometrySample with Matchers {
-
- test("Test Serialize To Python JavaRDD[Geometry]") {
- val convertedToPythonRDD = GeometryRddConverter(pointSpatialRDD, pythonGeometrySerializer).translateToPython
- val convertedToPythonArrays = convertedToPythonRDD.collect().toArray().toList.map(arr => arr match {
- case a: Array[Byte] => a.toList
- })
- convertedToPythonArrays should contain theSameElementsAs expectedPointArray
- }
-
- test("Test between Circle RDD and Python RDD") {
- val rddPython = PythonConverter.translateSpatialRDDToPython(circleSpatialRDD)
- val rddJava = PythonConverter.translatePythonRDDToJava(rddPython)
- rddJava.collect() should contain theSameElementsAs sampleCircles
- }
-
- test("Test between Point RDD and Python RDD") {
- val rddPython = PythonConverter.translateSpatialRDDToPython(pointSpatialRDD)
- val rddJava = PythonConverter.translatePythonRDDToJava(rddPython)
- rddJava.collect() should contain theSameElementsAs samplePoints
- }
-
- test("Should serialize to Python JavaRDD[Geometry, Geometry]") {
- val translatedToPythonSpatialPairRdd = FlatPairRddConverter(
- JavaPairRDD.fromRDD(spatialPairRDD), pythonGeometrySerializer).translateToPython
-
- translatedToPythonSpatialPairRdd.collect().toArray().toList.map(arr => arr match {
- case a: Array[Byte] => a.toList
- }) should contain theSameElementsAs expectedPairRDDPythonArray
- }
-
- test("Should serialize to Python JavaRDD[Geometry, List[Geometry]]") {
- val translatedToPythonList = ListPairRddConverter(
- JavaPairRDD.fromRDD(spatialPairRDDWithList), pythonGeometrySerializer).translateToPython
- val existingValues = translatedToPythonList.collect().toArray().toList.map {
- case a: Array[Byte] => a.toList
- }
- existingValues should contain theSameElementsAs expectedPairRDDWithListPythonArray
- }
-
- private val pointSpatialRDD = sc.parallelize(samplePoints).toJavaRDD()
-
- private val circleSpatialRDD = sc.parallelize(sampleCircles).toJavaRDD()
-
- private val spatialPairRDD = sc.parallelize(
- samplePoints.zip(samplePolygons).map(
- geometries => (geometries._1, geometries._2)
- )
- )
-
- private val spatialPairRDDWithList = sc.parallelize(
- samplePolygons.map(
- polygon => (polygon, samplePoints.slice(0, 2).asJava)
- )
- )
-
- private val expectedPointArray: List[List[Byte]] = samplePoints.map(point =>
- 0.toByteArray().toList ++ pythonGeometrySerializer.serialize(point).toList ++ 0.toByteArray().toList)
-
- private val expectedPairRDDPythonArray: List[List[Byte]] = samplePoints.zip(samplePolygons).map(
- geometries => 2.toByteArray().toList ++ pythonGeometrySerializer.serialize(geometries._1).toList
- ++ 1.toByteArray().toList ++ pythonGeometrySerializer.serialize(geometries._2).toList
- )
-
- private val expectedPairRDDWithListPythonArray: List[List[Byte]] = samplePolygons.map(
- samplePolygon => 1.toByteArray().toList ++ pythonGeometrySerializer.serialize(samplePolygon).toList ++ 2.toByteArray() ++
- samplePoints.slice(0, 2).flatMap(samplePoint => pythonGeometrySerializer.serialize(samplePoint)))
-
-}
diff --git a/spark-shaded/pom.xml b/spark-shaded/pom.xml
index 550f5ca0a..e491295cd 100644
--- a/spark-shaded/pom.xml
+++ b/spark-shaded/pom.xml
@@ -59,11 +59,6 @@
<artifactId>sedona-sql-${spark.compat.version}_${scala.compat.version}</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-python-adapter-${spark.compat.version}_${scala.compat.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
<!-- Define this to be shaded since it is provided in other modules -->
<dependency>
<groupId>edu.ucar</groupId>
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/SerializationException.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/SerializationException.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/SerializationException.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/SerializationException.scala
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/EnvelopeAdapter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/EnvelopeAdapter.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/EnvelopeAdapter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/EnvelopeAdapter.scala
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/GeomSerializer.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/GeomSerializer.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/GeomSerializer.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/GeomSerializer.scala
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/GeometryAdapter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/GeometryAdapter.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/GeometryAdapter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/GeometryAdapter.scala
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/JoinParamsAdapter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/JoinParamsAdapter.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/JoinParamsAdapter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/JoinParamsAdapter.scala
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/PythonConverter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/PythonConverter.scala
similarity index 91%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/PythonConverter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/PythonConverter.scala
index 63073bc04..08a037e8a 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/PythonConverter.scala
+++ b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/PythonConverter.scala
@@ -19,13 +19,7 @@
package org.apache.sedona.python.wrapper.adapters
-import org.apache.sedona.python.wrapper.translation.{
- FlatPairRddConverter,
- GeometryRddConverter,
- GeometrySeqToPythonConverter,
- ListPairRddConverter,
- PythonRDDToJavaConverter
-}
+import org.apache.sedona.python.wrapper.translation._
import org.apache.spark.api.java.{JavaPairRDD, JavaRDD}
import org.locationtech.jts.geom.Geometry
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/PythonRddToJavaRDDAdapter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/PythonRddToJavaRDDAdapter.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/PythonRddToJavaRDDAdapter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/PythonRddToJavaRDDAdapter.scala
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/RawJvmIndexRDDAdapter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/RawJvmIndexRDDAdapter.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/RawJvmIndexRDDAdapter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/RawJvmIndexRDDAdapter.scala
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/SpatialObjectLoaderAdapter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/SpatialObjectLoaderAdapter.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/adapters/SpatialObjectLoaderAdapter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/adapters/SpatialObjectLoaderAdapter.scala
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/CircleSerializer.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/CircleSerializer.scala
similarity index 93%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/CircleSerializer.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/CircleSerializer.scala
index ea9c4ce8d..ec40efb7d 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/CircleSerializer.scala
+++ b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/CircleSerializer.scala
@@ -20,7 +20,7 @@
package org.apache.sedona.python.wrapper.translation
import org.apache.sedona.common.geometryObjects.Circle
-import org.apache.sedona.python.wrapper.utils.implicits._
+import org.apache.sedona.python.wrapper.utils.implicits.{DoubleImplicit, GeometryEnhancer, IntImplicit}
import org.locationtech.jts.io.WKBWriter
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/FlatPairRddConverter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/FlatPairRddConverter.scala
similarity index 96%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/FlatPairRddConverter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/FlatPairRddConverter.scala
index c303e9d95..a456b1319 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/FlatPairRddConverter.scala
+++ b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/FlatPairRddConverter.scala
@@ -19,7 +19,7 @@
package org.apache.sedona.python.wrapper.translation
-import org.apache.sedona.python.wrapper.utils.implicits._
+import org.apache.sedona.python.wrapper.utils.implicits.IntImplicit
import org.apache.spark.api.java.{JavaPairRDD, JavaRDD}
import org.locationtech.jts.geom.Geometry
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometryRddConverter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/GeometryRddConverter.scala
similarity index 95%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometryRddConverter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/GeometryRddConverter.scala
index 1071cf3ac..326b5027d 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometryRddConverter.scala
+++ b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/GeometryRddConverter.scala
@@ -19,7 +19,7 @@
package org.apache.sedona.python.wrapper.translation
-import org.apache.sedona.python.wrapper.utils.implicits._
+import org.apache.sedona.python.wrapper.utils.implicits.IntImplicit
import org.apache.spark.api.java.JavaRDD
import org.locationtech.jts.geom.Geometry
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometrySeqToPythonConverter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/GeometrySeqToPythonConverter.scala
similarity index 94%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometrySeqToPythonConverter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/GeometrySeqToPythonConverter.scala
index 237b58895..a27cdde07 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometrySeqToPythonConverter.scala
+++ b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/GeometrySeqToPythonConverter.scala
@@ -19,7 +19,7 @@
package org.apache.sedona.python.wrapper.translation
-import org.apache.sedona.python.wrapper.utils.implicits._
+import org.apache.sedona.python.wrapper.utils.implicits.IntImplicit
import org.locationtech.jts.geom.Geometry
case class GeometrySeqToPythonConverter(spatialData: Seq[Geometry],
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometrySerializer.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/GeometrySerializer.scala
similarity index 94%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometrySerializer.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/GeometrySerializer.scala
index 662a4b815..bb1eb1ea2 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/GeometrySerializer.scala
+++ b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/GeometrySerializer.scala
@@ -19,7 +19,7 @@
package org.apache.sedona.python.wrapper.translation
-import org.apache.sedona.python.wrapper.utils.implicits._
+import org.apache.sedona.python.wrapper.utils.implicits.{GeometryEnhancer, IntImplicit}
import org.locationtech.jts.geom.Geometry
import org.locationtech.jts.io.WKBWriter
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/ListPairRddConverter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/ListPairRddConverter.scala
similarity index 96%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/ListPairRddConverter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/ListPairRddConverter.scala
index 993d3bcc8..7bc3d2631 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/ListPairRddConverter.scala
+++ b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/ListPairRddConverter.scala
@@ -19,7 +19,7 @@
package org.apache.sedona.python.wrapper.translation
-import org.apache.sedona.python.wrapper.utils.implicits._
+import org.apache.sedona.python.wrapper.utils.implicits.IntImplicit
import org.apache.spark.api.java.{JavaPairRDD, JavaRDD}
import org.locationtech.jts.geom.Geometry
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonGeometrySerializer.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonGeometrySerializer.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala
index 722923fd8..c88b2483f 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonGeometrySerializer.scala
+++ b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonGeometrySerializer.scala
@@ -19,13 +19,13 @@
package org.apache.sedona.python.wrapper.translation
-import java.nio.ByteBuffer
-
import org.apache.sedona.common.geometryObjects.Circle
import org.apache.sedona.python.wrapper.SerializationException
import org.locationtech.jts.geom.Geometry
import org.locationtech.jts.io.WKBReader
+import java.nio.ByteBuffer
+
private[python] class PythonGeometrySerializer extends Serializable {
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonRDDToJavaConverter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonRDDToJavaConverter.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonRDDToJavaConverter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonRDDToJavaConverter.scala
index 3dd211574..50dac332f 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonRDDToJavaConverter.scala
+++ b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonRDDToJavaConverter.scala
@@ -19,12 +19,12 @@
package org.apache.sedona.python.wrapper.translation
-import java.io.{ByteArrayInputStream, DataInputStream}
import org.apache.sedona.common.geometryObjects.Circle
import org.apache.sedona.python.wrapper.SerializationException
import org.apache.spark.api.java.JavaRDD
import org.locationtech.jts.geom.Geometry
+import java.io.{ByteArrayInputStream, DataInputStream}
import java.nio.{ByteBuffer, ByteOrder}
case class PythonRDDToJavaConverter(javaRDD: JavaRDD[Array[Byte]], geometrySerializer: PythonGeometrySerializer) {
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonRddToJavaRDDConverter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonRddToJavaRDDConverter.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/PythonRddToJavaRDDConverter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/PythonRddToJavaRDDConverter.scala
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/RDDToPythonConverter.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/RDDToPythonConverter.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/translation/RDDToPythonConverter.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/translation/RDDToPythonConverter.scala
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/utils/PythonAdapterWrapper.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/utils/PythonAdapterWrapper.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/utils/PythonAdapterWrapper.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/utils/PythonAdapterWrapper.scala
diff --git a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/utils/implicits.scala b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/utils/implicits.scala
similarity index 100%
rename from python-adapter/src/main/scala/org.apache.sedona.python.wrapper/utils/implicits.scala
rename to sql/common/src/main/scala/org/apache/sedona/python/wrapper/utils/implicits.scala
index 36d62a638..0f877c7ce 100644
--- a/python-adapter/src/main/scala/org.apache.sedona.python.wrapper/utils/implicits.scala
+++ b/sql/common/src/main/scala/org/apache/sedona/python/wrapper/utils/implicits.scala
@@ -19,11 +19,11 @@
package org.apache.sedona.python.wrapper.utils
+import org.locationtech.jts.geom.Geometry
+
import java.nio.charset.StandardCharsets
import java.nio.{ByteBuffer, ByteOrder}
-import org.locationtech.jts.geom.Geometry
-
object implicits {
implicit class IntImplicit(value: Int) {
diff --git a/sql/common/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala b/sql/common/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala
new file mode 100644
index 000000000..393e02313
--- /dev/null
+++ b/sql/common/src/test/scala/org/apache/sedona/python/wrapper/TestToPythonSerialization.scala
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sedona.python.wrapper
+
+import org.apache.sedona.common.geometryObjects.Circle
+import org.apache.sedona.python.wrapper.adapters.PythonConverter
+import org.apache.sedona.python.wrapper.translation.{FlatPairRddConverter, GeometryRddConverter, ListPairRddConverter, PythonGeometrySerializer}
+import org.apache.sedona.python.wrapper.utils.implicits._
+import org.apache.sedona.sql.TestBaseScala
+import org.apache.spark.api.java.JavaPairRDD
+import org.locationtech.jts.geom.{Geometry, GeometryFactory}
+import org.locationtech.jts.io.WKTReader
+import org.scalatest.matchers.must.Matchers.contain
+import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
+
+import java.io.{FileInputStream, InputStream}
+import scala.io.Source
+import scala.jdk.CollectionConverters._
+
+
+class TestToPythonSerialization extends TestBaseScala {
+
+ private lazy val geometryFactory = new GeometryFactory()
+
+ private lazy val pythonGeometrySerializer = new PythonGeometrySerializer()
+
+ private lazy val wktReader = new WKTReader(geometryFactory)
+
+ private val samplePoints: List[Geometry] =
+ loadGeometriesFromResources(resourceFolder + "python/samplePoints")
+
+ private val sampleCircles: List[Geometry] = samplePoints.map(new Circle(_, 1.0))
+
+ private val sampleLines: List[Geometry] =
+ loadGeometriesFromResources(resourceFolder + "python/sampleLines")
+
+ private val samplePolygons: List[Geometry] =
+ loadGeometriesFromResources(resourceFolder + "python/samplePolygons")
+
+ private def loadGeometriesFromResources(fileName: String): List[Geometry] = {
+ val resourceFileText = loadResourceFile(fileName)
+ loadFromWktStrings(resourceFileText)
+ }
+
+ private def loadFromWktStrings(geometries: List[String]): List[Geometry] = {
+ geometries.map(
+ geometryWKT => wktReader.read(geometryWKT)
+ )
+ }
+
+ private def loadResourceFile(fileName: String): List[String] = {
+ val stream: InputStream = new FileInputStream(fileName)
+ Source.fromInputStream(stream).getLines.toList
+ }
+
+ val pointSpatialRDD = sc.parallelize(samplePoints).toJavaRDD()
+
+ val circleSpatialRDD = sc.parallelize(sampleCircles).toJavaRDD()
+
+ val spatialPairRDD = sc.parallelize(
+ samplePoints.zip(samplePolygons).map(
+ geometries => (geometries._1, geometries._2)
+ )
+ )
+
+ val spatialPairRDDWithList = sc.parallelize(
+ samplePolygons.map(
+ polygon => (polygon, samplePoints.slice(0, 2).asJava)
+ )
+ )
+
+ val expectedPointArray: List[List[Byte]] = samplePoints.map(point =>
+ 0.toByteArray().toList ++ pythonGeometrySerializer.serialize(point).toList ++ 0.toByteArray().toList)
+
+ val expectedPairRDDPythonArray: List[List[Byte]] = samplePoints.zip(samplePolygons).map(
+ geometries => 2.toByteArray().toList ++ pythonGeometrySerializer.serialize(geometries._1).toList
+ ++ 1.toByteArray().toList ++ pythonGeometrySerializer.serialize(geometries._2).toList
+ )
+
+ val expectedPairRDDWithListPythonArray: List[List[Byte]] = samplePolygons.map(
+ samplePolygon => 1.toByteArray().toList ++ pythonGeometrySerializer.serialize(samplePolygon).toList ++ 2.toByteArray() ++
+ samplePoints.slice(0, 2).flatMap(samplePoint => pythonGeometrySerializer.serialize(samplePoint)))
+
+ describe("Sedona Python Wrapper Test") {
+ it("Test Serialize To Python JavaRDD[Geometry]") {
+ val convertedToPythonRDD = GeometryRddConverter(pointSpatialRDD, pythonGeometrySerializer).translateToPython
+ val convertedToPythonArrays = convertedToPythonRDD.collect().toArray().toList.map(arr => arr match {
+ case a: Array[Byte] => a.toList
+ })
+ convertedToPythonArrays should contain theSameElementsAs expectedPointArray
+ }
+
+ it("Test between Circle RDD and Python RDD") {
+ val rddPython = PythonConverter.translateSpatialRDDToPython(circleSpatialRDD)
+ val rddJava = PythonConverter.translatePythonRDDToJava(rddPython)
+ rddJava.collect() should contain theSameElementsAs sampleCircles
+ }
+
+ it("Test between Point RDD and Python RDD") {
+ val rddPython = PythonConverter.translateSpatialRDDToPython(pointSpatialRDD)
+ val rddJava = PythonConverter.translatePythonRDDToJava(rddPython)
+ rddJava.collect() should contain theSameElementsAs samplePoints
+ }
+
+ it("Should serialize to Python JavaRDD[Geometry, Geometry]") {
+ val translatedToPythonSpatialPairRdd = FlatPairRddConverter(
+ JavaPairRDD.fromRDD(spatialPairRDD), pythonGeometrySerializer).translateToPython
+
+ translatedToPythonSpatialPairRdd.collect().toArray().toList.map(arr => arr match {
+ case a: Array[Byte] => a.toList
+ }) should contain theSameElementsAs expectedPairRDDPythonArray
+ }
+
+ it("Should serialize to Python JavaRDD[Geometry, List[Geometry]]") {
+ val translatedToPythonList = ListPairRddConverter(
+ JavaPairRDD.fromRDD(spatialPairRDDWithList), pythonGeometrySerializer).translateToPython
+ val existingValues = translatedToPythonList.collect().toArray().toList.map {
+ case a: Array[Byte] => a.toList
+ }
+ existingValues should contain theSameElementsAs expectedPairRDDWithListPythonArray
+ }
+ }
+
+
+
+}
diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala
index 09a6f50ee..41e140c57 100644
--- a/sql/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala
+++ b/sql/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala
@@ -43,6 +43,8 @@ trait TestBaseScala extends FunSpec with BeforeAndAfterAll {
.config("spark.kryoserializer.buffer.max", "64m")
.getOrCreate()
+ val sc = sparkSession.sparkContext
+
val resourceFolder = System.getProperty("user.dir") + "/../../core/src/test/resources/"
val mixedWkbGeometryInputLocation = resourceFolder + "county_small_wkb.tsv"
val mixedWktGeometryInputLocation = resourceFolder + "county_small.tsv"